From fc99f8f0cc767eb2a41e3c5636bb3a0d0611f64b Mon Sep 17 00:00:00 2001 From: Thiago Pacheco Carneiro Date: Wed, 16 Sep 2026 18:42:46 -0300 Subject: [PATCH 1/3] Add Dataset::to_json for creation-schema round-trip Serialize the logical schema (variable names, data types, dimensions, chunk grids, compressors, long names, coordinates and stored user attributes) into the creation JSON consumed by from_json(). The output is validated against the dataset creation schema before being returned. Key translation decisions: - dataType comes from the zarr dtype metadata (V2 numpy-style strings reverse-mapped, V3 MDIO names, struct fields translated per field), not the top-level spec dtype, which is the byte view for structs. - statsV1/unitsV1/attributes are read from the stored array metadata (V2 .zattrs, V3 zarr.json) because the in-memory UserAttributes flatten unitsV1 objects, a form the creation schema rejects. - Dimension coordinates use the object form so the create path registers the dimension; other variables use the string form. - Datasets with header variables are rejected loudly: the creation schema has no representation for them, so the round-trip would silently drop them (creation-side support is a prerequisite). Tests: round-trip (V2+V3) with sampling and full creation-JSON equality, explicit schema validation, loud header-variable rejection; struct-array and header-variable round-trip preservation are prerequisite-gated (GTEST_SKIP). --- mdio/dataset.h | 441 +++++++++++++++++++++++++++++++++++++++++++ mdio/dataset_test.cc | 183 ++++++++++++++++++ 2 files changed, 624 insertions(+) diff --git a/mdio/dataset.h b/mdio/dataset.h index f9b5710..87411bf 100644 --- a/mdio/dataset.h +++ b/mdio/dataset.h @@ -154,6 +154,382 @@ from_zmetadata(const std::string& dataset_path, return pair.future; } + +// ========================================================================== +// Creation-JSON serialization helpers (Dataset::to_json) +// ========================================================================== + +/** + * @brief Reverse of zarr::v2::ToZarrDtype: maps a numpy-style dtype string to + * the MDIO scalar name. + * + * One-byte dtypes accept both the "<" and "|" byte-order prefixes (numpy + * emits "|" for types that carry no endianness). + * @param dtype The numpy-style dtype string (e.g. " FromZarrV2Dtype(const std::string& dtype) { + if (dtype == "/.zattrs" and V3 stores in + * "/zarr.json" under "attributes". The stored form is authoritative + * for "statsV1"/"unitsV1"/"attributes" because the in-memory UserAttributes + * flatten unitsV1 objects (e.g. {"speed": "m/s"} becomes "m/s"), a form the + * creation schema rejects. + * @param var The Variable to read from. + * @param version The Zarr version of the underlying store. + * @return The stored attributes object; a missing attribute file yields an + * empty object. Read and parse errors are propagated. + */ +inline Result ReadStoredAttributes( + const Variable<>& var, const zarr::ZarrVersion version) { + const tensorstore::KvStore kvstore = var.get_store().kvstore(); + // The store's kvstore path keeps its trailing slash on local file systems; + // joining it with a leading-slash key would form an invalid "//" path. + const char* attribute_file = + version == zarr::ZarrVersion::kV3 ? "zarr.json" : ".zattrs"; + const std::string separator = + kvstore.path.empty() || kvstore.path.back() == '/' ? "" : "/"; + auto read_result = + tensorstore::kvstore::Read(kvstore, separator + attribute_file).result(); + if (!read_result.ok()) { + return read_result.status(); + } + if (!read_result->has_value()) { + return nlohmann::json::object(); + } + const auto parsed = + nlohmann::json::parse(std::string(read_result->value), nullptr, false); + if (parsed.is_discarded()) { + return absl::InvalidArgumentError( + "Failed to parse stored attributes of variable '" + + var.get_variable_name() + "'."); + } + if (version == zarr::ZarrVersion::kV3) { + if (!parsed.contains("attributes")) { + return nlohmann::json::object(); + } + return parsed.at("attributes"); + } + return parsed; +} + +/** + * @brief Builds the creation-schema struct fields from a V2 numpy-style dtype + * array (e.g. [["cdp-x", " StructFieldsFromV2Dtype( + const nlohmann::json& dtype, const std::string& variable_name) { + nlohmann::json fields = nlohmann::json::array(); + for (const nlohmann::json& pair : dtype) { + if (!pair.is_array() || pair.size() != 2 || !pair.at(0).is_string() || + !pair.at(1).is_string()) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a malformed V2 struct dtype entry."); + } + MDIO_ASSIGN_OR_RETURN(const std::string format, + FromZarrV2Dtype(pair.at(1).get())); + fields.push_back({{"name", pair.at(0)}, {"format", format}}); + } + return nlohmann::json{{"fields", fields}}; +} + +/** + * @brief Builds the creation-schema struct fields from a V3 data_type + * descriptor ({"name": "struct", "configuration": {"fields": [...]}}). + */ +inline Result StructFieldsFromV3DataType( + const nlohmann::json& data_type, const std::string& variable_name) { + if (!data_type.is_object() || !data_type.contains("configuration") || + !data_type.at("configuration").contains("fields")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has an unsupported V3 data_type layout."); + } + nlohmann::json fields = nlohmann::json::array(); + for (const nlohmann::json& field : + data_type.at("configuration").at("fields")) { + if (!field.contains("name") || !field.contains("data_type")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a struct field without a name or data_type."); + } + fields.push_back( + {{"name", field.at("name")}, {"format", field.at("data_type")}}); + } + return nlohmann::json{{"fields", fields}}; +} + +/** + * @brief Builds the creation-schema "dataType" from a Variable's zarr spec. + * + * Scalars map through the zarr dtype metadata (V2 numpy-style strings, V3 + * MDIO names). Structured arrays translate their per-field layout into the + * schema's {"fields": [{"name", "format"}]} form. The top-level spec "dtype" + * is deliberately not used: for structured arrays it is the byte view + * ("byte"), which the creation schema rejects. + * @param spec The Variable's zarr spec (from Variable::spec()). + * @param version The Zarr version of the underlying store. + * @param variable_name The Variable name, for error context. + * @return The creation-schema dataType (a scalar name or a fields object), or + * InvalidArgumentError for an unsupported layout. + */ +inline Result DataTypeFromSpec( + const nlohmann::json& spec, const zarr::ZarrVersion version, + const std::string& variable_name) { + const nlohmann::json& metadata = spec.at("metadata"); + if (version == zarr::ZarrVersion::kV3) { + const nlohmann::json& data_type = metadata.at("data_type"); + if (data_type.is_string()) { + return data_type; + } + return StructFieldsFromV3DataType(data_type, variable_name); + } + const nlohmann::json& dtype = metadata.at("dtype"); + if (dtype.is_string()) { + return FromZarrV2Dtype(dtype.get()); + } + if (dtype.is_array()) { + return StructFieldsFromV2Dtype(dtype, variable_name); + } + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has an unsupported V2 dtype."); +} + +/** + * @brief Builds the creation-schema Blosc compressor from a zarr Blosc + * configuration. + * + * The V2 (numcodecs) integer shuffle form is translated to the schema's + * string enum; the V3 form already uses the string enum and passes through. + * @param config The Blosc configuration (cname, clevel, shuffle, blocksize, + * optional typesize). + * @param variable_name The Variable name, for error context. + * @return The creation-schema compressor object, or InvalidArgumentError when + * a required key is missing. + */ +inline Result BloscCompressorFromConfig( + const nlohmann::json& config, const std::string& variable_name) { + for (const auto* key : {"cname", "clevel", "shuffle", "blocksize"}) { + if (!config.contains(key)) { + return absl::InvalidArgumentError("Blosc configuration of variable '" + + variable_name + "' is missing key '" + + key + "'."); + } + } + nlohmann::json compressor = { + {"name", "blosc"}, + {"cname", config.at("cname")}, + {"clevel", config.at("clevel")}, + {"shuffle", blosc_shuffle_to_string(config.at("shuffle"))}, + {"blocksize", config.at("blocksize")}}; + if (config.contains("typesize")) { + compressor["typesize"] = config.at("typesize"); + } + return compressor; +} + +/** + * @brief Builds the creation-schema "compressor" from a Variable's zarr spec. + * + * V2 stores express compression as metadata.compressor (numcodecs form, null + * when uncompressed); V3 stores as a metadata.codecs pipeline ([bytes] alone + * means uncompressed, [bytes, blosc] means Blosc). Only Blosc is supported by + * the creation path (transform_compressor), so any other codec is reported + * instead of being silently dropped. + * @param spec The Variable's zarr spec (from Variable::spec()). + * @param version The Zarr version of the underlying store. + * @param variable_name The Variable name, for error context. + * @return The creation-schema compressor object, null when the Variable is + * uncompressed, or InvalidArgumentError for an unsupported codec. + */ +inline Result CompressorFromSpec( + const nlohmann::json& spec, const zarr::ZarrVersion version, + const std::string& variable_name) { + const nlohmann::json& metadata = spec.at("metadata"); + if (version == zarr::ZarrVersion::kV3) { + if (!metadata.contains("codecs")) { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has no codecs pipeline."); + } + for (const nlohmann::json& codec : metadata.at("codecs")) { + if (!codec.is_object() || !codec.contains("name")) { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has a malformed codec."); + } + if (codec.at("name") == "bytes") { + continue; // Endianness codec, always present. + } + if (codec.at("name") != "blosc") { + return absl::InvalidArgumentError( + "Variable '" + variable_name + "' uses codec '" + + codec.at("name").get() + + "', which the creation schema does not support."); + } + if (!codec.contains("configuration")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a blosc codec without configuration."); + } + return BloscCompressorFromConfig(codec.at("configuration"), + variable_name); + } + return nlohmann::json(); // [bytes] only: uncompressed. + } + if (!metadata.contains("compressor") || metadata.at("compressor").is_null()) { + return nlohmann::json(); // Uncompressed. + } + const nlohmann::json& compressor = metadata.at("compressor"); + if (!compressor.contains("id") || compressor.at("id") != "blosc") { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' uses a compressor the creation " + "schema does not support."); + } + return BloscCompressorFromConfig(compressor, variable_name); +} + +/** + * @brief Reads the chunk grid name from a Variable's zarr spec. + * + * V3 specs carry it in metadata.chunk_grid.name; V2 "chunks" arrays are + * regular grids by definition, so "regular" is returned for them. + */ +inline std::string ChunkGridNameFromSpec(const nlohmann::json& spec, + const zarr::ZarrVersion version) { + if (version == zarr::ZarrVersion::kV3) { + const nlohmann::json& metadata = spec.at("metadata"); + if (metadata.contains("chunk_grid") && + metadata.at("chunk_grid").contains("name")) { + return metadata.at("chunk_grid").at("name").get(); + } + } + return "regular"; +} + +/** + * @brief Builds the creation-schema "dimensions" for a Variable. + * + * Dimension coordinates (rank 1, name matching the dimension label) use the + * object form [{"name", "size"}] so the create path registers the dimension; + * every other variable uses the string form. The trailing unlabeled byte axis + * of structured arrays is not part of the logical schema and is dropped. + * @param var The Variable to serialize. + * @return The creation-schema dimensions entry, or InvalidArgumentError when + * a dimension is unlabeled in a way the schema cannot express. + */ +inline Result DimensionsFromVariable(const Variable<>& var) { + const std::string& name = var.get_variable_name(); + auto dimensions_view = var.dimensions(); + std::vector labels(dimensions_view.labels().begin(), + dimensions_view.labels().end()); + std::vector shape(dimensions_view.shape().begin(), + dimensions_view.shape().end()); + // Drop the trailing byte axis of structured arrays (unlabeled). + while (!labels.empty() && labels.back().empty()) { + labels.pop_back(); + shape.pop_back(); + } + for (const std::string_view& label : labels) { + if (label.empty()) { + return absl::InvalidArgumentError( + "Variable '" + name + + "' has an unlabeled dimension that is not " + "the trailing byte axis; the creation schema cannot express it."); + } + } + if (labels.size() == 1 && labels.front() == name) { + // Dimension coordinate: the object form registers name and size. + nlohmann::json dimension = {{"name", name}, {"size", shape.front()}}; + return nlohmann::json::array({dimension}); + } + return nlohmann::json(std::vector(labels.begin(), labels.end())); +} + +/** + * @brief Serializes one Variable into a creation-schema variable entry. + * + * The zarr spec supplies the dataType, dimensions and chunk grid; the stored + * attributes ("statsV1"/"unitsV1"/"attributes") are read from the kvstore + * because the in-memory forms flatten unitsV1 objects (see + * ReadStoredAttributes). + * @param var The Variable to serialize. + * @param coordinates The non-dimension coordinate names of the Variable. + * @param defaults Whether to include default values in the spec + * serialization. + * @return The creation-schema variable entry, or an error. + */ +inline Result VariableToCreationJson( + const Variable<>& var, const std::vector& coordinates, + const IncludeDefaults defaults) { + const std::string& name = var.get_variable_name(); + + auto spec_result = var.spec(); + if (!spec_result.ok()) { + return spec_result.status(); + } + auto json_result = spec_result.value().ToJson(defaults); + if (!json_result.ok()) { + return json_result.status(); + } + const nlohmann::json spec = json_result.value(); + const zarr::ZarrVersion version = zarr::GetVersionFromSpec(spec); + + nlohmann::json entry; + entry["name"] = name; + MDIO_ASSIGN_OR_RETURN(entry["dataType"], + DataTypeFromSpec(spec, version, name)); + MDIO_ASSIGN_OR_RETURN(entry["dimensions"], DimensionsFromVariable(var)); + + if (!var.get_long_name().empty()) { + entry["longName"] = var.get_long_name(); + } + + MDIO_ASSIGN_OR_RETURN(const nlohmann::json compressor, + CompressorFromSpec(spec, version, name)); + if (!compressor.is_null()) { + entry["compressor"] = compressor; + } + + nlohmann::json metadata_block; + MDIO_ASSIGN_OR_RETURN(const auto chunk_shape, var.get_chunk_shape()); + metadata_block["chunkGrid"] = { + {"name", ChunkGridNameFromSpec(spec, version)}, + {"configuration", {{"chunkShape", chunk_shape}}}}; + MDIO_ASSIGN_OR_RETURN(const nlohmann::json stored_attributes, + ReadStoredAttributes(var, version)); + for (const auto* attribute_key : {"statsV1", "unitsV1", "attributes"}) { + if (stored_attributes.contains(attribute_key) && + !stored_attributes.at(attribute_key).is_null()) { + metadata_block[attribute_key] = stored_attributes.at(attribute_key); + } + } + entry["metadata"] = metadata_block; + + if (!coordinates.empty()) { + entry["coordinates"] = coordinates; + } + return entry; +} } // namespace internal using coordinate_map = @@ -1400,6 +1776,71 @@ class Dataset { */ const nlohmann::json& getMetadata() const { return metadata; } + /** + * @brief Serializes the Dataset into the creation JSON consumed by + * from_json(). + * @details \b Usage + * @code + * MDIO_ASSIGN_OR_RETURN(auto creation_json, dataset.to_json()); + * auto round_tripped = mdio::Dataset::from_json( + * creation_json, "path/to/copy", mdio::zarr::ZarrVersion::kV3, + * mdio::constants::kCreateClean); + * @endcode + * The output carries the logical schema: variable names, data types, + * dimensions, chunk grids, compressors, long names, coordinates and the + * stored user attributes ("statsV1"/"unitsV1"/"attributes"). It is + * validated against the dataset creation schema before being returned, so + * a successful result can be fed back to from_json(). + * + * Datasets with header variables are rejected: the creation schema has no + * representation for them, so serializing them would produce a JSON that + * silently drops them when re-created. + * @param defaults Whether to include default values in the underlying + * spec serialization. + * @return The creation JSON on success, or an error. + */ + Result to_json( + IncludeDefaults defaults = IncludeDefaults{}) const { + if (!header_variables.get_iterable_accessor().empty()) { + return absl::InvalidArgumentError( + "to_json() does not support datasets with header variables: the " + "creation schema consumed by from_json() has no representation for " + "them, so the round-trip would silently drop them. Creation-side " + "header variable support is a prerequisite."); + } + + nlohmann::json out; + out["metadata"] = metadata; + out["variables"] = nlohmann::json::array(); + for (const auto& key : variables.get_iterable_accessor()) { + auto var_result = variables.at(key); + if (!var_result.ok()) { + return var_result.status(); + } + std::vector var_coordinates; + auto coordinate_it = coordinates.find(key); + if (coordinate_it != coordinates.end()) { + var_coordinates = coordinate_it->second; + } + MDIO_ASSIGN_OR_RETURN(const nlohmann::json entry, + internal::VariableToCreationJson( + var_result.value(), var_coordinates, defaults)); + out["variables"].push_back(entry); + } + + // The output must be consumable by from_json(); validate a copy because + // validate_dataset normalizes legacy compressor keys in place. + nlohmann::json validated = out; + absl::Status validation = validate_dataset(validated); + if (!validation.ok()) { + return absl::Status( + absl::StatusCode::kInvalidArgument, + "to_json() produced a spec that fails creation validation: " + + std::string(validation.message())); + } + return out; + } + /// variables contained in the dataset VariableCollection variables; diff --git a/mdio/dataset_test.cc b/mdio/dataset_test.cc index 5301674..0482fc7 100644 --- a/mdio/dataset_test.cc +++ b/mdio/dataset_test.cc @@ -1691,4 +1691,187 @@ TEST(Dataset, selRangeWithSameStartStop) { testing::HasSubstr("Start and stop values must be different")); } +// ============================================================================ +// Dataset::to_json — creation-schema serialization +// ============================================================================ + +/** + * @brief Returns the variable entry with the given name from a creation JSON, + * or a null JSON when absent. + */ +nlohmann::json FindVariableEntry(const nlohmann::json& creation_json, + const std::string& name) { + for (const auto& entry : creation_json.at("variables")) { + if (entry.at("name") == name) { + return entry; + } + } + return nlohmann::json(); +} + +TEST_P(DatasetVersionTest, toJsonRoundTrip) { + auto json_vars = GetToyExample(); + auto datasetRes = mdio::Dataset::from_json(json_vars, base_path_, version_, + mdio::constants::kCreateClean); + ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status(); + auto ds = datasetRes.value(); + + auto jsonRes = ds.to_json(); + ASSERT_TRUE(jsonRes.status().ok()) << jsonRes.status(); + nlohmann::json creation_json = jsonRes.value(); + + // --- Sampling of the serialized logical schema. --- + EXPECT_EQ(creation_json.at("metadata").at("name"), "campos_3d"); + EXPECT_TRUE( + creation_json.at("metadata").at("attributes").contains("textHeader")); + + ASSERT_EQ(creation_json.at("variables").size(), 9); + + const nlohmann::json image = FindVariableEntry(creation_json, "image"); + ASSERT_FALSE(image.is_null()); + EXPECT_EQ(image.at("dataType"), "float32"); + EXPECT_EQ(image.at("dimensions"), + nlohmann::json({"inline", "crossline", "depth"})); + EXPECT_EQ( + image.at("metadata").at("chunkGrid").at("configuration").at("chunkShape"), + nlohmann::json({128, 128, 128})); + EXPECT_EQ(image.at("compressor").at("cname"), "zstd"); + EXPECT_EQ(image.at("metadata").at("statsV1").at("count"), 100); + EXPECT_EQ(image.at("metadata").at("attributes").at("fizz"), "buzz"); + EXPECT_EQ(image.at("coordinates"), nlohmann::json({"cdp-x", "cdp-y"})); + + const nlohmann::json velocity = FindVariableEntry(creation_json, "velocity"); + ASSERT_FALSE(velocity.is_null()); + EXPECT_EQ(velocity.at("dataType"), "float16"); + // unitsV1 must keep the stored object form; the in-memory flattened form + // ("m/s") is rejected by the creation schema. + EXPECT_EQ(velocity.at("metadata").at("unitsV1"), + nlohmann::json({{"speed", "m/s"}})); + + const nlohmann::json image_inline = + FindVariableEntry(creation_json, "image_inline"); + ASSERT_FALSE(image_inline.is_null()); + EXPECT_EQ(image_inline.at("longName"), + "inline optimized version of 3d_stack"); + + const nlohmann::json image_headers = + FindVariableEntry(creation_json, "image_headers"); + ASSERT_FALSE(image_headers.is_null()); + EXPECT_EQ(image_headers.at("dimensions"), + nlohmann::json({"inline", "crossline"})); + const nlohmann::json fields = image_headers.at("dataType").at("fields"); + ASSERT_EQ(fields.size(), 4); + EXPECT_EQ(fields.at(0).at("name"), "cdp-x"); + EXPECT_EQ(fields.at(0).at("format"), "int32"); + EXPECT_EQ(fields.at(3).at("name"), "some_scalar"); + EXPECT_EQ(fields.at(3).at("format"), "float16"); + + const nlohmann::json inline_var = FindVariableEntry(creation_json, "inline"); + ASSERT_FALSE(inline_var.is_null()); + // Dimension coordinates use the object form so the create path registers + // the dimension. + const nlohmann::json inline_dimension = {{"name", "inline"}, {"size", 256}}; + EXPECT_EQ(inline_var.at("dimensions"), + nlohmann::json::array({inline_dimension})); + + const nlohmann::json depth = FindVariableEntry(creation_json, "depth"); + ASSERT_FALSE(depth.is_null()); + EXPECT_EQ(depth.at("metadata").at("unitsV1"), + nlohmann::json({{"length", "m"}})); + + // --- Round-trip: re-create the dataset from the serialized form. --- + const std::string copy_path = base_path_ + "_to_json_roundtrip"; + std::filesystem::remove_all(copy_path); + { + auto copyRes = mdio::Dataset::from_json(creation_json, copy_path, version_, + mdio::constants::kCreateClean); + ASSERT_TRUE(copyRes.status().ok()) << copyRes.status(); + auto copy = copyRes.value(); + + // get_iterable_accessor() is sorted; get_keys() order is unspecified. + EXPECT_EQ(ds.variables.get_iterable_accessor(), + copy.variables.get_iterable_accessor()); + + // The copy serializes to the same creation JSON: the logical schema + // survives the round-trip unchanged. + auto copyJsonRes = copy.to_json(); + ASSERT_TRUE(copyJsonRes.status().ok()) << copyJsonRes.status(); + EXPECT_EQ(copyJsonRes.value(), creation_json); + } + // The copy is destroyed here so the kvstore releases its files before the + // directory cleanup (NFS silly-rename otherwise keeps them busy). + std::filesystem::remove_all(copy_path); +} + +TEST_P(DatasetVersionTest, toJsonOutputPassesSchemaValidation) { + auto json_vars = GetToyExample(); + auto datasetRes = mdio::Dataset::from_json(json_vars, base_path_, version_, + mdio::constants::kCreateClean); + ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status(); + + auto jsonRes = datasetRes.value().to_json(); + ASSERT_TRUE(jsonRes.status().ok()) << jsonRes.status(); + + // to_json() validates internally; pin the contract explicitly against the + // creation schema, the same check from_json() runs. + nlohmann::json creation_json = jsonRes.value(); + auto validation = validate_schema(creation_json); + EXPECT_TRUE(validation.ok()) << validation; +} + +TEST_P(DatasetVersionTest, toJsonStructArrayRoundTrip) { + GTEST_SKIP() + << "Prerequisite-gated (api-gap-plan M1): the structured-array write " + "dialect ('struct' vs 'structured', zarr-python #2134) and the spec " + "dtype derivation for structured variables must land first."; +} + +TEST(Dataset, toJsonRejectsHeaderVariablesLoudly) { + // Header variables can only be built through Dataset::Open with a + // header-only spec; from_json() has no creation representation for them. + const nlohmann::json metadata = { + {"name", "header_only"}, + {"apiVersion", "1.0.0"}, + {"createdOn", "2023-12-12T15:02:06.413469-06:00"}}; + const nlohmann::json header_spec = nlohmann::json::parse(R"({ + "_mdio_header_only": true, + "driver": "zarr3", + "kvstore": { + "driver": "file", + "path": "zarrs/to_json_header_only/segy_file_header" + }, + "_mdio_array_metadata": { + "shape": [], + "data_type": { + "name": "fixed_length_utf32", + "configuration": {"length_bytes": 4} + }, + "attributes": { + "textHeader": "C01 EXAMPLE" + } + } + })"); + + auto dsRes = + mdio::Dataset::Open(metadata, {header_spec}, mdio::constants::kOpen); + ASSERT_TRUE(dsRes.status().ok()) << dsRes.status(); + auto ds = dsRes.value(); + ASSERT_EQ(ds.header_variables.get_iterable_accessor().size(), 1); + + auto jsonRes = ds.to_json(); + ASSERT_FALSE(jsonRes.status().ok()); + EXPECT_EQ(jsonRes.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(jsonRes.status().message(), + testing::HasSubstr("header variables")); +} + +TEST(Dataset, toJsonHeaderVariableRoundTripPreservation) { + GTEST_SKIP() + << "Prerequisite-gated (api-gap-plan M1): from_json()/Construct " + "rejects header-variable entries because the creation schema has no " + "representation for them. Creation-side header variable support " + "must land before the round-trip can preserve them; until then " + "to_json() fails loudly instead of silently dropping them."; +} + } // namespace From 75607b4b418fb0a46d62a496f42dde89fff20a69 Mon Sep 17 00:00:00 2001 From: Thiago Pacheco Carneiro Date: Wed, 16 Sep 2026 18:46:57 -0300 Subject: [PATCH 2/3] Ungate struct-array round-trip coverage (gate premise refuted) --- mdio/dataset_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mdio/dataset_test.cc b/mdio/dataset_test.cc index 0482fc7..68bc558 100644 --- a/mdio/dataset_test.cc +++ b/mdio/dataset_test.cc @@ -1819,12 +1819,12 @@ TEST_P(DatasetVersionTest, toJsonOutputPassesSchemaValidation) { EXPECT_TRUE(validation.ok()) << validation; } -TEST_P(DatasetVersionTest, toJsonStructArrayRoundTrip) { - GTEST_SKIP() - << "Prerequisite-gated (api-gap-plan M1): the structured-array write " - "dialect ('struct' vs 'structured', zarr-python #2134) and the spec " - "dtype derivation for structured variables must land first."; -} +// Struct-array round-trip is covered by toJsonRoundTrip above (field-level +// assertions + creation-JSON equality). The plan's original gate premise +// ("spec derivation yields 'byte'") was refuted by the implementation: +// to_json derives dataType from the zarr dtype metadata, not the top-level +// spec. The remaining struct limitation is cross-tool (zarr-python write +// dialect, Issue 01) — not mdio-cpp round-trip. TEST(Dataset, toJsonRejectsHeaderVariablesLoudly) { // Header variables can only be built through Dataset::Open with a From b64d956f4f1cf4248a364d24deb874749ffbc5a7 Mon Sep 17 00:00:00 2001 From: Thiago Pacheco Carneiro Date: Fri, 18 Sep 2026 22:52:25 -0300 Subject: [PATCH 3/3] Move to_json serialization helpers to dataset_factory.h Relocate the creation-JSON serialization helpers next to their parsing inverses (to_zarr_dtype, blosc resolution helpers) in dataset_factory.h, inside mdio::internal. Pure relocation: no signature, body, or call-site changes; dataset.h already includes dataset_factory.h. --- mdio/dataset.h | 376 ---------------------------------------- mdio/dataset_factory.h | 383 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 383 insertions(+), 376 deletions(-) diff --git a/mdio/dataset.h b/mdio/dataset.h index 87411bf..b329814 100644 --- a/mdio/dataset.h +++ b/mdio/dataset.h @@ -154,382 +154,6 @@ from_zmetadata(const std::string& dataset_path, return pair.future; } - -// ========================================================================== -// Creation-JSON serialization helpers (Dataset::to_json) -// ========================================================================== - -/** - * @brief Reverse of zarr::v2::ToZarrDtype: maps a numpy-style dtype string to - * the MDIO scalar name. - * - * One-byte dtypes accept both the "<" and "|" byte-order prefixes (numpy - * emits "|" for types that carry no endianness). - * @param dtype The numpy-style dtype string (e.g. " FromZarrV2Dtype(const std::string& dtype) { - if (dtype == "/.zattrs" and V3 stores in - * "/zarr.json" under "attributes". The stored form is authoritative - * for "statsV1"/"unitsV1"/"attributes" because the in-memory UserAttributes - * flatten unitsV1 objects (e.g. {"speed": "m/s"} becomes "m/s"), a form the - * creation schema rejects. - * @param var The Variable to read from. - * @param version The Zarr version of the underlying store. - * @return The stored attributes object; a missing attribute file yields an - * empty object. Read and parse errors are propagated. - */ -inline Result ReadStoredAttributes( - const Variable<>& var, const zarr::ZarrVersion version) { - const tensorstore::KvStore kvstore = var.get_store().kvstore(); - // The store's kvstore path keeps its trailing slash on local file systems; - // joining it with a leading-slash key would form an invalid "//" path. - const char* attribute_file = - version == zarr::ZarrVersion::kV3 ? "zarr.json" : ".zattrs"; - const std::string separator = - kvstore.path.empty() || kvstore.path.back() == '/' ? "" : "/"; - auto read_result = - tensorstore::kvstore::Read(kvstore, separator + attribute_file).result(); - if (!read_result.ok()) { - return read_result.status(); - } - if (!read_result->has_value()) { - return nlohmann::json::object(); - } - const auto parsed = - nlohmann::json::parse(std::string(read_result->value), nullptr, false); - if (parsed.is_discarded()) { - return absl::InvalidArgumentError( - "Failed to parse stored attributes of variable '" + - var.get_variable_name() + "'."); - } - if (version == zarr::ZarrVersion::kV3) { - if (!parsed.contains("attributes")) { - return nlohmann::json::object(); - } - return parsed.at("attributes"); - } - return parsed; -} - -/** - * @brief Builds the creation-schema struct fields from a V2 numpy-style dtype - * array (e.g. [["cdp-x", " StructFieldsFromV2Dtype( - const nlohmann::json& dtype, const std::string& variable_name) { - nlohmann::json fields = nlohmann::json::array(); - for (const nlohmann::json& pair : dtype) { - if (!pair.is_array() || pair.size() != 2 || !pair.at(0).is_string() || - !pair.at(1).is_string()) { - return absl::InvalidArgumentError( - "Variable '" + variable_name + - "' has a malformed V2 struct dtype entry."); - } - MDIO_ASSIGN_OR_RETURN(const std::string format, - FromZarrV2Dtype(pair.at(1).get())); - fields.push_back({{"name", pair.at(0)}, {"format", format}}); - } - return nlohmann::json{{"fields", fields}}; -} - -/** - * @brief Builds the creation-schema struct fields from a V3 data_type - * descriptor ({"name": "struct", "configuration": {"fields": [...]}}). - */ -inline Result StructFieldsFromV3DataType( - const nlohmann::json& data_type, const std::string& variable_name) { - if (!data_type.is_object() || !data_type.contains("configuration") || - !data_type.at("configuration").contains("fields")) { - return absl::InvalidArgumentError( - "Variable '" + variable_name + - "' has an unsupported V3 data_type layout."); - } - nlohmann::json fields = nlohmann::json::array(); - for (const nlohmann::json& field : - data_type.at("configuration").at("fields")) { - if (!field.contains("name") || !field.contains("data_type")) { - return absl::InvalidArgumentError( - "Variable '" + variable_name + - "' has a struct field without a name or data_type."); - } - fields.push_back( - {{"name", field.at("name")}, {"format", field.at("data_type")}}); - } - return nlohmann::json{{"fields", fields}}; -} - -/** - * @brief Builds the creation-schema "dataType" from a Variable's zarr spec. - * - * Scalars map through the zarr dtype metadata (V2 numpy-style strings, V3 - * MDIO names). Structured arrays translate their per-field layout into the - * schema's {"fields": [{"name", "format"}]} form. The top-level spec "dtype" - * is deliberately not used: for structured arrays it is the byte view - * ("byte"), which the creation schema rejects. - * @param spec The Variable's zarr spec (from Variable::spec()). - * @param version The Zarr version of the underlying store. - * @param variable_name The Variable name, for error context. - * @return The creation-schema dataType (a scalar name or a fields object), or - * InvalidArgumentError for an unsupported layout. - */ -inline Result DataTypeFromSpec( - const nlohmann::json& spec, const zarr::ZarrVersion version, - const std::string& variable_name) { - const nlohmann::json& metadata = spec.at("metadata"); - if (version == zarr::ZarrVersion::kV3) { - const nlohmann::json& data_type = metadata.at("data_type"); - if (data_type.is_string()) { - return data_type; - } - return StructFieldsFromV3DataType(data_type, variable_name); - } - const nlohmann::json& dtype = metadata.at("dtype"); - if (dtype.is_string()) { - return FromZarrV2Dtype(dtype.get()); - } - if (dtype.is_array()) { - return StructFieldsFromV2Dtype(dtype, variable_name); - } - return absl::InvalidArgumentError("Variable '" + variable_name + - "' has an unsupported V2 dtype."); -} - -/** - * @brief Builds the creation-schema Blosc compressor from a zarr Blosc - * configuration. - * - * The V2 (numcodecs) integer shuffle form is translated to the schema's - * string enum; the V3 form already uses the string enum and passes through. - * @param config The Blosc configuration (cname, clevel, shuffle, blocksize, - * optional typesize). - * @param variable_name The Variable name, for error context. - * @return The creation-schema compressor object, or InvalidArgumentError when - * a required key is missing. - */ -inline Result BloscCompressorFromConfig( - const nlohmann::json& config, const std::string& variable_name) { - for (const auto* key : {"cname", "clevel", "shuffle", "blocksize"}) { - if (!config.contains(key)) { - return absl::InvalidArgumentError("Blosc configuration of variable '" + - variable_name + "' is missing key '" + - key + "'."); - } - } - nlohmann::json compressor = { - {"name", "blosc"}, - {"cname", config.at("cname")}, - {"clevel", config.at("clevel")}, - {"shuffle", blosc_shuffle_to_string(config.at("shuffle"))}, - {"blocksize", config.at("blocksize")}}; - if (config.contains("typesize")) { - compressor["typesize"] = config.at("typesize"); - } - return compressor; -} - -/** - * @brief Builds the creation-schema "compressor" from a Variable's zarr spec. - * - * V2 stores express compression as metadata.compressor (numcodecs form, null - * when uncompressed); V3 stores as a metadata.codecs pipeline ([bytes] alone - * means uncompressed, [bytes, blosc] means Blosc). Only Blosc is supported by - * the creation path (transform_compressor), so any other codec is reported - * instead of being silently dropped. - * @param spec The Variable's zarr spec (from Variable::spec()). - * @param version The Zarr version of the underlying store. - * @param variable_name The Variable name, for error context. - * @return The creation-schema compressor object, null when the Variable is - * uncompressed, or InvalidArgumentError for an unsupported codec. - */ -inline Result CompressorFromSpec( - const nlohmann::json& spec, const zarr::ZarrVersion version, - const std::string& variable_name) { - const nlohmann::json& metadata = spec.at("metadata"); - if (version == zarr::ZarrVersion::kV3) { - if (!metadata.contains("codecs")) { - return absl::InvalidArgumentError("Variable '" + variable_name + - "' has no codecs pipeline."); - } - for (const nlohmann::json& codec : metadata.at("codecs")) { - if (!codec.is_object() || !codec.contains("name")) { - return absl::InvalidArgumentError("Variable '" + variable_name + - "' has a malformed codec."); - } - if (codec.at("name") == "bytes") { - continue; // Endianness codec, always present. - } - if (codec.at("name") != "blosc") { - return absl::InvalidArgumentError( - "Variable '" + variable_name + "' uses codec '" + - codec.at("name").get() + - "', which the creation schema does not support."); - } - if (!codec.contains("configuration")) { - return absl::InvalidArgumentError( - "Variable '" + variable_name + - "' has a blosc codec without configuration."); - } - return BloscCompressorFromConfig(codec.at("configuration"), - variable_name); - } - return nlohmann::json(); // [bytes] only: uncompressed. - } - if (!metadata.contains("compressor") || metadata.at("compressor").is_null()) { - return nlohmann::json(); // Uncompressed. - } - const nlohmann::json& compressor = metadata.at("compressor"); - if (!compressor.contains("id") || compressor.at("id") != "blosc") { - return absl::InvalidArgumentError("Variable '" + variable_name + - "' uses a compressor the creation " - "schema does not support."); - } - return BloscCompressorFromConfig(compressor, variable_name); -} - -/** - * @brief Reads the chunk grid name from a Variable's zarr spec. - * - * V3 specs carry it in metadata.chunk_grid.name; V2 "chunks" arrays are - * regular grids by definition, so "regular" is returned for them. - */ -inline std::string ChunkGridNameFromSpec(const nlohmann::json& spec, - const zarr::ZarrVersion version) { - if (version == zarr::ZarrVersion::kV3) { - const nlohmann::json& metadata = spec.at("metadata"); - if (metadata.contains("chunk_grid") && - metadata.at("chunk_grid").contains("name")) { - return metadata.at("chunk_grid").at("name").get(); - } - } - return "regular"; -} - -/** - * @brief Builds the creation-schema "dimensions" for a Variable. - * - * Dimension coordinates (rank 1, name matching the dimension label) use the - * object form [{"name", "size"}] so the create path registers the dimension; - * every other variable uses the string form. The trailing unlabeled byte axis - * of structured arrays is not part of the logical schema and is dropped. - * @param var The Variable to serialize. - * @return The creation-schema dimensions entry, or InvalidArgumentError when - * a dimension is unlabeled in a way the schema cannot express. - */ -inline Result DimensionsFromVariable(const Variable<>& var) { - const std::string& name = var.get_variable_name(); - auto dimensions_view = var.dimensions(); - std::vector labels(dimensions_view.labels().begin(), - dimensions_view.labels().end()); - std::vector shape(dimensions_view.shape().begin(), - dimensions_view.shape().end()); - // Drop the trailing byte axis of structured arrays (unlabeled). - while (!labels.empty() && labels.back().empty()) { - labels.pop_back(); - shape.pop_back(); - } - for (const std::string_view& label : labels) { - if (label.empty()) { - return absl::InvalidArgumentError( - "Variable '" + name + - "' has an unlabeled dimension that is not " - "the trailing byte axis; the creation schema cannot express it."); - } - } - if (labels.size() == 1 && labels.front() == name) { - // Dimension coordinate: the object form registers name and size. - nlohmann::json dimension = {{"name", name}, {"size", shape.front()}}; - return nlohmann::json::array({dimension}); - } - return nlohmann::json(std::vector(labels.begin(), labels.end())); -} - -/** - * @brief Serializes one Variable into a creation-schema variable entry. - * - * The zarr spec supplies the dataType, dimensions and chunk grid; the stored - * attributes ("statsV1"/"unitsV1"/"attributes") are read from the kvstore - * because the in-memory forms flatten unitsV1 objects (see - * ReadStoredAttributes). - * @param var The Variable to serialize. - * @param coordinates The non-dimension coordinate names of the Variable. - * @param defaults Whether to include default values in the spec - * serialization. - * @return The creation-schema variable entry, or an error. - */ -inline Result VariableToCreationJson( - const Variable<>& var, const std::vector& coordinates, - const IncludeDefaults defaults) { - const std::string& name = var.get_variable_name(); - - auto spec_result = var.spec(); - if (!spec_result.ok()) { - return spec_result.status(); - } - auto json_result = spec_result.value().ToJson(defaults); - if (!json_result.ok()) { - return json_result.status(); - } - const nlohmann::json spec = json_result.value(); - const zarr::ZarrVersion version = zarr::GetVersionFromSpec(spec); - - nlohmann::json entry; - entry["name"] = name; - MDIO_ASSIGN_OR_RETURN(entry["dataType"], - DataTypeFromSpec(spec, version, name)); - MDIO_ASSIGN_OR_RETURN(entry["dimensions"], DimensionsFromVariable(var)); - - if (!var.get_long_name().empty()) { - entry["longName"] = var.get_long_name(); - } - - MDIO_ASSIGN_OR_RETURN(const nlohmann::json compressor, - CompressorFromSpec(spec, version, name)); - if (!compressor.is_null()) { - entry["compressor"] = compressor; - } - - nlohmann::json metadata_block; - MDIO_ASSIGN_OR_RETURN(const auto chunk_shape, var.get_chunk_shape()); - metadata_block["chunkGrid"] = { - {"name", ChunkGridNameFromSpec(spec, version)}, - {"configuration", {{"chunkShape", chunk_shape}}}}; - MDIO_ASSIGN_OR_RETURN(const nlohmann::json stored_attributes, - ReadStoredAttributes(var, version)); - for (const auto* attribute_key : {"statsV1", "unitsV1", "attributes"}) { - if (stored_attributes.contains(attribute_key) && - !stored_attributes.at(attribute_key).is_null()) { - metadata_block[attribute_key] = stored_attributes.at(attribute_key); - } - } - entry["metadata"] = metadata_block; - - if (!coordinates.empty()) { - entry["coordinates"] = coordinates; - } - return entry; -} } // namespace internal using coordinate_map = diff --git a/mdio/dataset_factory.h b/mdio/dataset_factory.h index 3d67050..1dcc9ab 100644 --- a/mdio/dataset_factory.h +++ b/mdio/dataset_factory.h @@ -30,6 +30,7 @@ #include "absl/strings/str_split.h" #include "mdio/dataset_validator.h" #include "mdio/impl.h" +#include "mdio/variable.h" #include "mdio/zarr/zarr.h" /** @@ -737,4 +738,386 @@ Construct(nlohmann::json& spec /*NOLINT*/, const std::string& path, return std::make_tuple(spec["metadata"], datasetSpec); } +namespace mdio { +namespace internal { + +// ========================================================================== +// Creation-JSON serialization helpers (Dataset::to_json) +// ========================================================================== + +/** + * @brief Reverse of zarr::v2::ToZarrDtype: maps a numpy-style dtype string to + * the MDIO scalar name. + * + * One-byte dtypes accept both the "<" and "|" byte-order prefixes (numpy + * emits "|" for types that carry no endianness). + * @param dtype The numpy-style dtype string (e.g. " FromZarrV2Dtype(const std::string& dtype) { + if (dtype == "/.zattrs" and V3 stores in + * "/zarr.json" under "attributes". The stored form is authoritative + * for "statsV1"/"unitsV1"/"attributes" because the in-memory UserAttributes + * flatten unitsV1 objects (e.g. {"speed": "m/s"} becomes "m/s"), a form the + * creation schema rejects. + * @param var The Variable to read from. + * @param version The Zarr version of the underlying store. + * @return The stored attributes object; a missing attribute file yields an + * empty object. Read and parse errors are propagated. + */ +inline Result ReadStoredAttributes( + const Variable<>& var, const zarr::ZarrVersion version) { + const tensorstore::KvStore kvstore = var.get_store().kvstore(); + // The store's kvstore path keeps its trailing slash on local file systems; + // joining it with a leading-slash key would form an invalid "//" path. + const char* attribute_file = + version == zarr::ZarrVersion::kV3 ? "zarr.json" : ".zattrs"; + const std::string separator = + kvstore.path.empty() || kvstore.path.back() == '/' ? "" : "/"; + auto read_result = + tensorstore::kvstore::Read(kvstore, separator + attribute_file).result(); + if (!read_result.ok()) { + return read_result.status(); + } + if (!read_result->has_value()) { + return nlohmann::json::object(); + } + const auto parsed = + nlohmann::json::parse(std::string(read_result->value), nullptr, false); + if (parsed.is_discarded()) { + return absl::InvalidArgumentError( + "Failed to parse stored attributes of variable '" + + var.get_variable_name() + "'."); + } + if (version == zarr::ZarrVersion::kV3) { + if (!parsed.contains("attributes")) { + return nlohmann::json::object(); + } + return parsed.at("attributes"); + } + return parsed; +} + +/** + * @brief Builds the creation-schema struct fields from a V2 numpy-style dtype + * array (e.g. [["cdp-x", " StructFieldsFromV2Dtype( + const nlohmann::json& dtype, const std::string& variable_name) { + nlohmann::json fields = nlohmann::json::array(); + for (const nlohmann::json& pair : dtype) { + if (!pair.is_array() || pair.size() != 2 || !pair.at(0).is_string() || + !pair.at(1).is_string()) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a malformed V2 struct dtype entry."); + } + MDIO_ASSIGN_OR_RETURN(const std::string format, + FromZarrV2Dtype(pair.at(1).get())); + fields.push_back({{"name", pair.at(0)}, {"format", format}}); + } + return nlohmann::json{{"fields", fields}}; +} + +/** + * @brief Builds the creation-schema struct fields from a V3 data_type + * descriptor ({"name": "struct", "configuration": {"fields": [...]}}). + */ +inline Result StructFieldsFromV3DataType( + const nlohmann::json& data_type, const std::string& variable_name) { + if (!data_type.is_object() || !data_type.contains("configuration") || + !data_type.at("configuration").contains("fields")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has an unsupported V3 data_type layout."); + } + nlohmann::json fields = nlohmann::json::array(); + for (const nlohmann::json& field : + data_type.at("configuration").at("fields")) { + if (!field.contains("name") || !field.contains("data_type")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a struct field without a name or data_type."); + } + fields.push_back( + {{"name", field.at("name")}, {"format", field.at("data_type")}}); + } + return nlohmann::json{{"fields", fields}}; +} + +/** + * @brief Builds the creation-schema "dataType" from a Variable's zarr spec. + * + * Scalars map through the zarr dtype metadata (V2 numpy-style strings, V3 + * MDIO names). Structured arrays translate their per-field layout into the + * schema's {"fields": [{"name", "format"}]} form. The top-level spec "dtype" + * is deliberately not used: for structured arrays it is the byte view + * ("byte"), which the creation schema rejects. + * @param spec The Variable's zarr spec (from Variable::spec()). + * @param version The Zarr version of the underlying store. + * @param variable_name The Variable name, for error context. + * @return The creation-schema dataType (a scalar name or a fields object), or + * InvalidArgumentError for an unsupported layout. + */ +inline Result DataTypeFromSpec( + const nlohmann::json& spec, const zarr::ZarrVersion version, + const std::string& variable_name) { + const nlohmann::json& metadata = spec.at("metadata"); + if (version == zarr::ZarrVersion::kV3) { + const nlohmann::json& data_type = metadata.at("data_type"); + if (data_type.is_string()) { + return data_type; + } + return StructFieldsFromV3DataType(data_type, variable_name); + } + const nlohmann::json& dtype = metadata.at("dtype"); + if (dtype.is_string()) { + return FromZarrV2Dtype(dtype.get()); + } + if (dtype.is_array()) { + return StructFieldsFromV2Dtype(dtype, variable_name); + } + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has an unsupported V2 dtype."); +} + +/** + * @brief Builds the creation-schema Blosc compressor from a zarr Blosc + * configuration. + * + * The V2 (numcodecs) integer shuffle form is translated to the schema's + * string enum; the V3 form already uses the string enum and passes through. + * @param config The Blosc configuration (cname, clevel, shuffle, blocksize, + * optional typesize). + * @param variable_name The Variable name, for error context. + * @return The creation-schema compressor object, or InvalidArgumentError when + * a required key is missing. + */ +inline Result BloscCompressorFromConfig( + const nlohmann::json& config, const std::string& variable_name) { + for (const auto* key : {"cname", "clevel", "shuffle", "blocksize"}) { + if (!config.contains(key)) { + return absl::InvalidArgumentError("Blosc configuration of variable '" + + variable_name + "' is missing key '" + + key + "'."); + } + } + nlohmann::json compressor = { + {"name", "blosc"}, + {"cname", config.at("cname")}, + {"clevel", config.at("clevel")}, + {"shuffle", blosc_shuffle_to_string(config.at("shuffle"))}, + {"blocksize", config.at("blocksize")}}; + if (config.contains("typesize")) { + compressor["typesize"] = config.at("typesize"); + } + return compressor; +} + +/** + * @brief Builds the creation-schema "compressor" from a Variable's zarr spec. + * + * V2 stores express compression as metadata.compressor (numcodecs form, null + * when uncompressed); V3 stores as a metadata.codecs pipeline ([bytes] alone + * means uncompressed, [bytes, blosc] means Blosc). Only Blosc is supported by + * the creation path (transform_compressor), so any other codec is reported + * instead of being silently dropped. + * @param spec The Variable's zarr spec (from Variable::spec()). + * @param version The Zarr version of the underlying store. + * @param variable_name The Variable name, for error context. + * @return The creation-schema compressor object, null when the Variable is + * uncompressed, or InvalidArgumentError for an unsupported codec. + */ +inline Result CompressorFromSpec( + const nlohmann::json& spec, const zarr::ZarrVersion version, + const std::string& variable_name) { + const nlohmann::json& metadata = spec.at("metadata"); + if (version == zarr::ZarrVersion::kV3) { + if (!metadata.contains("codecs")) { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has no codecs pipeline."); + } + for (const nlohmann::json& codec : metadata.at("codecs")) { + if (!codec.is_object() || !codec.contains("name")) { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' has a malformed codec."); + } + if (codec.at("name") == "bytes") { + continue; // Endianness codec, always present. + } + if (codec.at("name") != "blosc") { + return absl::InvalidArgumentError( + "Variable '" + variable_name + "' uses codec '" + + codec.at("name").get() + + "', which the creation schema does not support."); + } + if (!codec.contains("configuration")) { + return absl::InvalidArgumentError( + "Variable '" + variable_name + + "' has a blosc codec without configuration."); + } + return BloscCompressorFromConfig(codec.at("configuration"), + variable_name); + } + return nlohmann::json(); // [bytes] only: uncompressed. + } + if (!metadata.contains("compressor") || metadata.at("compressor").is_null()) { + return nlohmann::json(); // Uncompressed. + } + const nlohmann::json& compressor = metadata.at("compressor"); + if (!compressor.contains("id") || compressor.at("id") != "blosc") { + return absl::InvalidArgumentError("Variable '" + variable_name + + "' uses a compressor the creation " + "schema does not support."); + } + return BloscCompressorFromConfig(compressor, variable_name); +} + +/** + * @brief Reads the chunk grid name from a Variable's zarr spec. + * + * V3 specs carry it in metadata.chunk_grid.name; V2 "chunks" arrays are + * regular grids by definition, so "regular" is returned for them. + */ +inline std::string ChunkGridNameFromSpec(const nlohmann::json& spec, + const zarr::ZarrVersion version) { + if (version == zarr::ZarrVersion::kV3) { + const nlohmann::json& metadata = spec.at("metadata"); + if (metadata.contains("chunk_grid") && + metadata.at("chunk_grid").contains("name")) { + return metadata.at("chunk_grid").at("name").get(); + } + } + return "regular"; +} + +/** + * @brief Builds the creation-schema "dimensions" for a Variable. + * + * Dimension coordinates (rank 1, name matching the dimension label) use the + * object form [{"name", "size"}] so the create path registers the dimension; + * every other variable uses the string form. The trailing unlabeled byte axis + * of structured arrays is not part of the logical schema and is dropped. + * @param var The Variable to serialize. + * @return The creation-schema dimensions entry, or InvalidArgumentError when + * a dimension is unlabeled in a way the schema cannot express. + */ +inline Result DimensionsFromVariable(const Variable<>& var) { + const std::string& name = var.get_variable_name(); + auto dimensions_view = var.dimensions(); + std::vector labels(dimensions_view.labels().begin(), + dimensions_view.labels().end()); + std::vector shape(dimensions_view.shape().begin(), + dimensions_view.shape().end()); + // Drop the trailing byte axis of structured arrays (unlabeled). + while (!labels.empty() && labels.back().empty()) { + labels.pop_back(); + shape.pop_back(); + } + for (const std::string_view& label : labels) { + if (label.empty()) { + return absl::InvalidArgumentError( + "Variable '" + name + + "' has an unlabeled dimension that is not " + "the trailing byte axis; the creation schema cannot express it."); + } + } + if (labels.size() == 1 && labels.front() == name) { + // Dimension coordinate: the object form registers name and size. + nlohmann::json dimension = {{"name", name}, {"size", shape.front()}}; + return nlohmann::json::array({dimension}); + } + return nlohmann::json(std::vector(labels.begin(), labels.end())); +} + +/** + * @brief Serializes one Variable into a creation-schema variable entry. + * + * The zarr spec supplies the dataType, dimensions and chunk grid; the stored + * attributes ("statsV1"/"unitsV1"/"attributes") are read from the kvstore + * because the in-memory forms flatten unitsV1 objects (see + * ReadStoredAttributes). + * @param var The Variable to serialize. + * @param coordinates The non-dimension coordinate names of the Variable. + * @param defaults Whether to include default values in the spec + * serialization. + * @return The creation-schema variable entry, or an error. + */ +inline Result VariableToCreationJson( + const Variable<>& var, const std::vector& coordinates, + const IncludeDefaults defaults) { + const std::string& name = var.get_variable_name(); + + auto spec_result = var.spec(); + if (!spec_result.ok()) { + return spec_result.status(); + } + auto json_result = spec_result.value().ToJson(defaults); + if (!json_result.ok()) { + return json_result.status(); + } + const nlohmann::json spec = json_result.value(); + const zarr::ZarrVersion version = zarr::GetVersionFromSpec(spec); + + nlohmann::json entry; + entry["name"] = name; + MDIO_ASSIGN_OR_RETURN(entry["dataType"], + DataTypeFromSpec(spec, version, name)); + MDIO_ASSIGN_OR_RETURN(entry["dimensions"], DimensionsFromVariable(var)); + + if (!var.get_long_name().empty()) { + entry["longName"] = var.get_long_name(); + } + + MDIO_ASSIGN_OR_RETURN(const nlohmann::json compressor, + CompressorFromSpec(spec, version, name)); + if (!compressor.is_null()) { + entry["compressor"] = compressor; + } + + nlohmann::json metadata_block; + MDIO_ASSIGN_OR_RETURN(const auto chunk_shape, var.get_chunk_shape()); + metadata_block["chunkGrid"] = { + {"name", ChunkGridNameFromSpec(spec, version)}, + {"configuration", {{"chunkShape", chunk_shape}}}}; + MDIO_ASSIGN_OR_RETURN(const nlohmann::json stored_attributes, + ReadStoredAttributes(var, version)); + for (const auto* attribute_key : {"statsV1", "unitsV1", "attributes"}) { + if (stored_attributes.contains(attribute_key) && + !stored_attributes.at(attribute_key).is_null()) { + metadata_block[attribute_key] = stored_attributes.at(attribute_key); + } + } + entry["metadata"] = metadata_block; + + if (!coordinates.empty()) { + entry["coordinates"] = coordinates; + } + return entry; +} + +} // namespace internal +} // namespace mdio + #endif // MDIO_DATASET_FACTORY_H_