From d8d733f15f963d63ef2e658d8a6c5971302e0d97 Mon Sep 17 00:00:00 2001 From: Thiago Pacheco Carneiro Date: Wed, 16 Sep 2026 17:36:36 -0300 Subject: [PATCH 1/4] Detect missing store before .zmetadata fallback DetectVersion defaulted to V2 when a path had no store markers at all (no zarr.json and no .zgroup), so opening a non-existent path fell through to the V2 reader and surfaced as a confusing .zmetadata parse error. Return a clear "not an MDIO store or path does not exist" error from DetectVersion (its only caller is from_zmetadata) and propagate detection failures there instead of silently defaulting to V2. --- mdio/dataset.h | 9 ++++++--- mdio/dataset_test.cc | 16 ++++++++++++++++ mdio/zarr/zarr_driver.h | 15 ++++++++++----- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/mdio/dataset.h b/mdio/dataset.h index f9b5710..5b71754 100644 --- a/mdio/dataset.h +++ b/mdio/dataset.h @@ -136,10 +136,13 @@ from_zmetadata(const std::string& dataset_path, [promise = std::move(promise), dataset_path, kvs = ready_kvs.value()]( tensorstore::ReadyFuture version_ready) { - zarr::ZarrVersion version = zarr::ZarrVersion::kV2; - if (version_ready.result().ok()) { - version = version_ready.value(); + // A version detection failure means the path has no store + // markers; propagate it instead of falling back to V2. + if (!version_ready.result().ok()) { + promise.SetResult(version_ready.result().status()); + return; } + zarr::ZarrVersion version = version_ready.value(); auto result_future = zarr::ReadDatasetMetadata( version, dataset_path, tensorstore::MakeReadyFuture(kvs)); diff --git a/mdio/dataset_test.cc b/mdio/dataset_test.cc index 5301674..0aae6e2 100644 --- a/mdio/dataset_test.cc +++ b/mdio/dataset_test.cc @@ -1363,6 +1363,22 @@ TEST(Dataset, openNonExistent) { << "Opened a non-existent dataset without error!"; } +// Opening a path with no store markers (zarr.json and .zgroup both absent) +// must report the missing store itself, not a .zmetadata parse error from +// the V2 fallback. +TEST(Dataset, openNonExistentPathReportsMissingStore) { + std::string path = "zarrs/missing_store_test"; + auto datasetRes = mdio::Dataset::Open(path, mdio::constants::kOpen); + ASSERT_FALSE(datasetRes.status().ok()) + << "Opened a non-existent dataset without error!"; + + EXPECT_THAT(datasetRes.status().message(), + testing::HasSubstr("not an MDIO store")); + EXPECT_THAT(datasetRes.status().message(), testing::HasSubstr(path)); + EXPECT_THAT(datasetRes.status().message(), + testing::Not(testing::HasSubstr(".zmetadata"))); +} + TEST(Dataset, kCreateOverExisting) { auto json_vars = GetToyExample(); diff --git a/mdio/zarr/zarr_driver.h b/mdio/zarr/zarr_driver.h index b63af13..74ebe0d 100644 --- a/mdio/zarr/zarr_driver.h +++ b/mdio/zarr/zarr_driver.h @@ -91,8 +91,9 @@ inline Result ParseVersion(const nlohmann::json& version_spec) { * @brief Detects the Zarr version from an existing store by examining metadata * files. * @param kvstore The KvStore to examine. - * @return Future The detected version or an error if unable to - * determine. + * @return Future The detected version, or an error if the store + * has no version markers at all (neither `zarr.json` nor `.zgroup`), which + * means the path is not an MDIO store or does not exist. */ inline Future DetectVersion(const tensorstore::KvStore& kvstore) { // Check for zarr.json first (V3 indicator) @@ -112,14 +113,18 @@ inline Future DetectVersion(const tensorstore::KvStore& kvstore) { // Check for .zgroup or .zmetadata (V2 indicators) auto v2_check = tensorstore::kvstore::Read(kvstore, ".zgroup"); v2_check.ExecuteWhenReady( - [promise = std::move(promise)]( + [promise = std::move(promise), + store_path = kvstore.path]( tensorstore::ReadyFuture v2_result) mutable { if (v2_result.result().ok() && v2_result.value().has_value()) { promise.SetResult(ZarrVersion::kV2); } else { - // Default to V2 if we can't determine - promise.SetResult(ZarrVersion::kV2); + // No version markers at all. Defaulting to V2 here would + // surface downstream as a confusing .zmetadata parse error. + promise.SetResult(absl::InvalidArgumentError( + "not an MDIO store or path does not exist: " + + store_path)); } }); }); From bc73835d355a7e03c7c0d1bcd8e424337bcab24c Mon Sep 17 00:00:00 2001 From: Thiago Pacheco Carneiro Date: Wed, 16 Sep 2026 18:07:12 -0300 Subject: [PATCH 2/4] Tolerate missing dataset metadata on open with a warning mdio-python 1.x to_mdio writes no name/apiVersion/createdOn in the root metadata. The path-based Dataset::Open already opens such stores (the schema's required fields only gate the create path via Construct), but it did so silently; it now emits an ABSL_LOG(WARNING) naming the missing fields. Stores carrying the v0 marker (api_version) are skipped so the dedicated v0 rejection stays the first message. The create path keeps rejecting creation specs without the fields, and the writer is unchanged. --- mdio/CMakeLists.txt | 9 +++- mdio/dataset.h | 54 ++++++++++++++++++++ mdio/dataset_test.cc | 119 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) diff --git a/mdio/CMakeLists.txt b/mdio/CMakeLists.txt index 08ea17f..bfc058a 100644 --- a/mdio/CMakeLists.txt +++ b/mdio/CMakeLists.txt @@ -49,6 +49,9 @@ target_include_directories(mdio INTERFACE ) # Link dependencies +# NOTE: Abseil libraries (e.g. absl::log used by dataset.h) are provided +# transitively through EXT_TENSORSTORE::tensorstore, matching the existing +# convention for absl/strings usage in dataset_factory.h. target_link_libraries(mdio INTERFACE EXT_TENSORSTORE::tensorstore nlohmann_json_schema_validator::nlohmann_json_schema_validator @@ -291,7 +294,7 @@ mdio_cc_test( SRCS dataset_test.cc COPTS - ${mdio_DEFAULT_COPTS} + ${mdio_DEFAULT_COPTS} LINKOPTS ${mdio_DEFAULT_LINKOPTS} DEPS @@ -307,6 +310,10 @@ mdio_cc_test( tensorstore::index_space_index_transform tensorstore::util_status_testutil nlohmann_json_schema_validator + absl::log + absl::log_initialize + absl::log_sink + absl::log_internal_log_sink_set ) mdio_cc_test( diff --git a/mdio/dataset.h b/mdio/dataset.h index 5b71754..6cd02f2 100644 --- a/mdio/dataset.h +++ b/mdio/dataset.h @@ -17,6 +17,7 @@ #define MDIO_API_VERSION "1.0.0" #include +#include #include #include #include @@ -26,12 +27,15 @@ #include #include #include +#include #include #include #include #include #include +#include "absl/log/log.h" +#include "absl/strings/str_join.h" #include "mdio/dataset_factory.h" #include "mdio/header_variable.h" #include "mdio/variable.h" @@ -157,6 +161,51 @@ from_zmetadata(const std::string& dataset_path, return pair.future; } + +/** + * @brief Dataset metadata fields that are purely informational. + * + * mdio-python 1.x `to_mdio` writes none of them, so the read path tolerates + * their absence (with a warning) instead of rejecting the store. The create + * path (`from_json` -> `Construct` -> `validate_dataset`) keeps requiring + * them, and stores written by mdio-cpp always carry them. + */ +inline constexpr std::array kInformationalDatasetFields = + {"name", "apiVersion", "createdOn"}; + +/** + * @brief Warns when an opened store is missing informational dataset metadata. + * + * The fields above do not affect how data is read, so an open store missing + * them is still usable; this only reports the gap. Stores carrying the v0 + * marker (`api_version`) are skipped -- they are rejected downstream with a + * dedicated v0 error and would only produce a misleading warning here. + * + * @param metadata The dataset metadata read from the store. + * @param dataset_path The path the store was opened from. + */ +inline void WarnOnMissingDatasetMetadata(const ::nlohmann::json& metadata, + const std::string& dataset_path) { + if (metadata.contains("api_version")) { + // v0 store; rejected downstream with its own error message. + return; + } + std::vector missing; + for (const auto field : kInformationalDatasetFields) { + if (!metadata.contains(field)) { + missing.push_back(field); + } + } + if (missing.empty()) { + return; + } + ABSL_LOG(WARNING) + << "Dataset '" << dataset_path + << "' is missing dataset metadata field(s): " + << absl::StrJoin(missing, ", ") + << ". Continuing without them; these fields are informational and are " + "not written by mdio-python 1.x."; +} } // namespace internal using coordinate_map = @@ -940,6 +989,11 @@ class Dataset { mdio::internal::from_zmetadata(dataset_path, context).result()); auto [dataset_metadata, json_vars] = params_from_zmetadata; + // The read path tolerates stores without the informational dataset + // metadata fields (e.g. written by mdio-python 1.x); warn about the gap. + // Creation specs keep going through `Construct`'s strict validation. + internal::WarnOnMissingDatasetMetadata(dataset_metadata, dataset_path); + return mdio::Dataset::Open(dataset_metadata, json_vars, std::forward