Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion mdio/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ target_include_directories(mdio INTERFACE
)

# Link dependencies
# NOTE: Abseil libraries (e.g. absl::log used by dataset.h) are provided
# transitively through EXT_TENSORSTORE::tensorstore, matching the existing
# convention for absl/strings usage in dataset_factory.h.
target_link_libraries(mdio INTERFACE
EXT_TENSORSTORE::tensorstore
nlohmann_json_schema_validator::nlohmann_json_schema_validator
Expand Down Expand Up @@ -291,7 +294,7 @@ mdio_cc_test(
SRCS
dataset_test.cc
COPTS
${mdio_DEFAULT_COPTS}
${mdio_DEFAULT_COPTS}
LINKOPTS
${mdio_DEFAULT_LINKOPTS}
DEPS
Expand All @@ -307,6 +310,10 @@ mdio_cc_test(
tensorstore::index_space_index_transform
tensorstore::util_status_testutil
nlohmann_json_schema_validator
absl::log
absl::log_initialize
absl::log_sink
absl::log_internal_log_sink_set
)

mdio_cc_test(
Expand Down
63 changes: 60 additions & 3 deletions mdio/dataset.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#define MDIO_API_VERSION "1.0.0"

#include <algorithm>
#include <array>
#include <cstddef>
#include <fstream>
#include <limits>
Expand All @@ -26,12 +27,15 @@
#include <set>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "absl/log/log.h"
#include "absl/strings/str_join.h"
#include "mdio/dataset_factory.h"
#include "mdio/header_variable.h"
#include "mdio/variable.h"
Expand Down Expand Up @@ -136,10 +140,13 @@ from_zmetadata(const std::string& dataset_path,
[promise = std::move(promise), dataset_path,
kvs = ready_kvs.value()](
tensorstore::ReadyFuture<zarr::ZarrVersion> version_ready) {
zarr::ZarrVersion version = zarr::ZarrVersion::kV2;
if (version_ready.result().ok()) {
version = version_ready.value();
// A version detection failure means the path has no store
// markers; propagate it instead of falling back to V2.
if (!version_ready.result().ok()) {
promise.SetResult(version_ready.result().status());
return;
}
zarr::ZarrVersion version = version_ready.value();
auto result_future = zarr::ReadDatasetMetadata(
version, dataset_path,
tensorstore::MakeReadyFuture<tensorstore::KvStore>(kvs));
Expand All @@ -154,6 +161,51 @@ from_zmetadata(const std::string& dataset_path,

return pair.future;
}

/**
* @brief Dataset metadata fields that are purely informational.
*
* They do not affect how data is read, so the read path tolerates their
* absence (with a warning) instead of rejecting the store. The create
* path (`from_json` -> `Construct` -> `validate_dataset`) keeps requiring
* them, and stores written by mdio-cpp always carry them.
*/
inline constexpr std::array<std::string_view, 3> kInformationalDatasetFields = {
"name", "apiVersion", "createdOn"};

/**
* @brief Warns when an opened store is missing informational dataset metadata.
*
* The fields above do not affect how data is read, so an open store missing
* them is still usable; this only reports the gap. Stores carrying the v0
* marker (`api_version`) are skipped -- they are rejected downstream with a
* dedicated v0 error and would only produce a misleading warning here.
*
* @param metadata The dataset metadata read from the store.
* @param dataset_path The path the store was opened from.
*/
inline void WarnOnMissingDatasetMetadata(const ::nlohmann::json& metadata,
const std::string& dataset_path) {
if (metadata.contains("api_version")) {
// v0 store; rejected downstream with its own error message.
return;
}
std::vector<std::string_view> missing;
for (const auto field : kInformationalDatasetFields) {
if (!metadata.contains(field)) {
missing.push_back(field);
}
}
if (missing.empty()) {
return;
}
ABSL_LOG(WARNING)
<< "Dataset '" << dataset_path
<< "' is missing dataset metadata field(s): "
<< absl::StrJoin(missing, ", ")
<< ". Continuing without them; these fields are informational and do "
"not affect how the dataset is read.";
}
} // namespace internal

using coordinate_map =
Expand Down Expand Up @@ -937,6 +989,11 @@ class Dataset {
mdio::internal::from_zmetadata(dataset_path, context).result());
auto [dataset_metadata, json_vars] = params_from_zmetadata;

// The read path tolerates stores without the informational dataset
// metadata fields; warn about the gap. Creation specs keep going
// through `Construct`'s strict validation.
internal::WarnOnMissingDatasetMetadata(dataset_metadata, dataset_path);

return mdio::Dataset::Open(dataset_metadata, json_vars,
std::forward<Option>(options)...);
}
Expand Down
135 changes: 135 additions & 0 deletions mdio/dataset_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
#include <utility>
#include <vector>

#include "absl/log/initialize.h"
#include "absl/log/internal/log_sink_set.h"
#include "absl/log/log_sink.h"
#include "mdio/dataset_factory.h"
#include "mdio/zarr/zarr.h"
#include "tensorstore/driver/driver.h"
Expand All @@ -43,6 +46,14 @@
// clang-format on
namespace {

// Initialize Abseil logging before tests run: before InitializeLog() is
// called, log messages are directed only to stderr and registered LogSinks
// capture nothing (same pattern as gcs_test.cc).
struct AbslLogInit {
AbslLogInit() { absl::InitializeLog(); }
};
static AbslLogInit absl_log_init;

/**
* @brief Returns a string representation of the Zarr version for naming.
*/
Expand Down Expand Up @@ -1363,6 +1374,130 @@ TEST(Dataset, openNonExistent) {
<< "Opened a non-existent dataset without error!";
}

// Opening a path with no store markers (zarr.json and .zgroup both absent)
// must report the missing store itself, not a .zmetadata parse error from
// the V2 fallback.
TEST(Dataset, openNonExistentPathReportsMissingStore) {
std::string path = "zarrs/missing_store_test";
auto datasetRes = mdio::Dataset::Open(path, mdio::constants::kOpen);
ASSERT_FALSE(datasetRes.status().ok())
<< "Opened a non-existent dataset without error!";

EXPECT_THAT(datasetRes.status().message(),
testing::HasSubstr("not an MDIO store"));
EXPECT_THAT(datasetRes.status().message(), testing::HasSubstr(path));
EXPECT_THAT(datasetRes.status().message(),
testing::Not(testing::HasSubstr(".zmetadata")));
}

// Rewrites the root zarr.json of a v3 store the way mdio-python 1.x leaves it
// after a fresh `to_mdio` write: `attributes` is an empty object and none of
// the dataset metadata fields (name/apiVersion/createdOn) are present.
void RewriteRootAsPythonWritten(const std::string& path) {
nlohmann::json root = {{"zarr_format", 3},
{"node_type", "group"},
{"attributes", nlohmann::json::object()}};
std::ofstream root_file(path + "/zarr.json");
root_file << root.dump(4);
}

// Captures absl log messages while registered, so tests can assert on
// warnings emitted by the library.
class CapturingLogSink : public absl::LogSink {
public:
void Send(const absl::LogEntry& entry) override {
if (entry.log_severity() >= absl::LogSeverity::kWarning) {
warnings_.emplace_back(entry.text_message());
}
}

// True when some captured WARNING-or-worse message contains `needle`.
bool HasWarningContaining(const std::string& needle) const {
for (const auto& message : warnings_) {
if (message.find(needle) != std::string::npos) {
return true;
}
}
return false;
}

private:
std::vector<std::string> warnings_;
};

// mdio-python 1.x `to_mdio` writes no name/apiVersion/createdOn in the root
// metadata. Opening such a store must succeed (lenient read path) and warn
// about the missing informational fields.
TEST(Dataset, openV3WithoutDatasetMetadata) {
std::string path = "zarrs/py_written_v3";
std::filesystem::remove_all(path);

auto json_vars = GetToyExample();
auto created =
mdio::Dataset::from_json(json_vars, path, mdio::zarr::ZarrVersion::kV3,
mdio::constants::kCreateClean);
ASSERT_TRUE(created.status().ok()) << created.status();

RewriteRootAsPythonWritten(path);

CapturingLogSink sink;
absl::log_internal::AddLogSink(&sink);
auto reopened = mdio::Dataset::Open(path, mdio::constants::kOpen).result();
absl::log_internal::RemoveLogSink(&sink);

ASSERT_TRUE(reopened.status().ok()) << reopened.status();
EXPECT_TRUE(sink.HasWarningContaining("missing dataset metadata"))
<< "expected a warning about the missing dataset metadata fields";
EXPECT_TRUE(sink.HasWarningContaining(path));
for (const auto& field : {"name", "apiVersion", "createdOn"}) {
EXPECT_TRUE(sink.HasWarningContaining(field))
<< "warning should name the missing field '" << field << "'";
}

std::filesystem::remove_all(path);
}

// A store written by mdio-cpp carries the full dataset metadata; opening it
// must keep succeeding without the missing-metadata warning.
TEST(Dataset, openV3WithDatasetMetadata) {
std::string path = "zarrs/cpp_written_v3";
std::filesystem::remove_all(path);

auto json_vars = GetToyExample();
auto created =
mdio::Dataset::from_json(json_vars, path, mdio::zarr::ZarrVersion::kV3,
mdio::constants::kCreateClean);
ASSERT_TRUE(created.status().ok()) << created.status();

CapturingLogSink sink;
absl::log_internal::AddLogSink(&sink);
auto reopened = mdio::Dataset::Open(path, mdio::constants::kOpen).result();
absl::log_internal::RemoveLogSink(&sink);

ASSERT_TRUE(reopened.status().ok()) << reopened.status();
EXPECT_FALSE(sink.HasWarningContaining("missing dataset metadata"))
<< "store has full metadata; no warning expected";

std::filesystem::remove_all(path);
}

// The create path stays strict: a creation spec without the dataset metadata
// fields is rejected by `Construct`'s schema validation, as before.
TEST(Dataset, createSpecWithoutDatasetMetadataIsRejected) {
auto json_vars = GetToyExample();
json_vars["metadata"].erase("name");
json_vars["metadata"].erase("apiVersion");
json_vars["metadata"].erase("createdOn");

auto created = mdio::Dataset::from_json(
json_vars, "zarrs/creation_no_metadata", mdio::zarr::ZarrVersion::kV3,
mdio::constants::kCreateClean);
ASSERT_FALSE(created.status().ok())
<< "creation spec without dataset metadata must be rejected";
EXPECT_THAT(created.status().message(),
testing::HasSubstr("required property 'name' not found"));
}

TEST(Dataset, kCreateOverExisting) {
auto json_vars = GetToyExample();

Expand Down
13 changes: 8 additions & 5 deletions mdio/zarr/zarr_driver.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ inline Result<ZarrVersion> ParseVersion(const nlohmann::json& version_spec) {
* @brief Detects the Zarr version from an existing store by examining metadata
* files.
* @param kvstore The KvStore to examine.
* @return Future<ZarrVersion> The detected version or an error if unable to
* determine.
* @return Future<ZarrVersion> The detected version, or an error if the store
* has no version markers at all (neither `zarr.json` nor `.zgroup`), which
* means the path is not an MDIO store or does not exist.
*/
inline Future<ZarrVersion> DetectVersion(const tensorstore::KvStore& kvstore) {
// Check for zarr.json first (V3 indicator)
Expand All @@ -112,14 +113,16 @@ inline Future<ZarrVersion> DetectVersion(const tensorstore::KvStore& kvstore) {
// Check for .zgroup or .zmetadata (V2 indicators)
auto v2_check = tensorstore::kvstore::Read(kvstore, ".zgroup");
v2_check.ExecuteWhenReady(
[promise = std::move(promise)](
[promise = std::move(promise), store_path = kvstore.path](
tensorstore::ReadyFuture<tensorstore::kvstore::ReadResult>
v2_result) mutable {
if (v2_result.result().ok() && v2_result.value().has_value()) {
promise.SetResult(ZarrVersion::kV2);
} else {
// Default to V2 if we can't determine
promise.SetResult(ZarrVersion::kV2);
// No version markers at all. Defaulting to V2 here would
// surface downstream as a confusing .zmetadata parse error.
promise.SetResult(absl::InvalidArgumentError(
"not an MDIO store or path does not exist: " + store_path));
}
});
});
Expand Down
Loading