diff --git a/.github/workflows/compatibility-test.yml b/.github/workflows/compatibility-test.yml index b144ca998..dcd154fbb 100644 --- a/.github/workflows/compatibility-test.yml +++ b/.github/workflows/compatibility-test.yml @@ -63,13 +63,14 @@ env: jobs: generate-fixtures: - name: Generate ${{ matrix.language }} fixtures + name: Generate ${{ matrix.language }} ${{ matrix.compression }} fixtures runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: language: [ java, cpp ] + compression: [ UNCOMPRESSED, SNAPPY, GZIP, LZ4, ZSTD, LZMA2 ] steps: - name: Checkout repository uses: actions/checkout@v7 @@ -102,8 +103,9 @@ jobs: ./mvnw -P with-java \ -Dtest=TableModelEncodingCompressionCompatibilityTest#generateFixtures \ -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtsfile.compat.compression="${{ matrix.compression }}" \ -Dtsfile.compat.generate.dir="$PWD/compat-fixtures/java" \ - test + test-compile surefire:test - name: Build C++ tests if: matrix.language == 'cpp' @@ -115,10 +117,10 @@ jobs: -Dspotless.skip=true \ -Dtsfile.dependency.source=BUNDLED \ -Denable.antlr4=OFF \ - -Denable.snappy=OFF \ + -Denable.snappy=ON \ -Denable.lz4=ON \ -Denable.lzokay=OFF \ - -Denable.zlib=OFF \ + -Denable.zlib=ON \ -Denable.zstd=ON \ -Denable.lzma2=ON \ -Denable.simd=OFF \ @@ -129,6 +131,7 @@ jobs: shell: bash run: | mkdir -p compat-fixtures/cpp + TSFILE_COMPAT_COMPRESSION="${{ matrix.compression }}" \ TSFILE_COMPAT_GENERATE_DIR="$PWD/compat-fixtures/cpp" \ ./cpp/target/build/test/lib/TsFile_Test \ --gtest_filter=TableModelEncodingCompressionCompatibilityTest.GenerateFixtures @@ -136,20 +139,21 @@ jobs: - name: Upload ${{ matrix.language }} fixtures uses: actions/upload-artifact@v7 with: - name: tsfile-${{ matrix.language }}-compat-fixtures + name: tsfile-${{ matrix.language }}-${{ matrix.compression }}-compat-fixtures path: compat-fixtures/${{ matrix.language }}/ if-no-files-found: error retention-days: 1 validate-fixtures: - name: Validate ${{ matrix.writer }} fixtures with ${{ matrix.reader }} + name: Validate ${{ matrix.pair.writer }} ${{ matrix.compression }} with ${{ matrix.pair.reader }} needs: generate-fixtures runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: - include: + compression: [ UNCOMPRESSED, SNAPPY, GZIP, LZ4, ZSTD, LZMA2 ] + pair: - writer: java reader: cpp - writer: cpp @@ -172,30 +176,31 @@ jobs: restore-keys: ${{ runner.os }}-m2- - name: Cache verified C++ dependency archives - if: matrix.reader == 'cpp' + if: matrix.pair.reader == 'cpp' uses: actions/cache@v6 with: path: cpp/target/dependency-cache key: ${{ runner.os }}-tsfile-cpp-deps-${{ hashFiles('cpp/CMakeLists.txt', 'cpp/cmake/*Source.cmake', 'cpp/third_party/CMakeLists.txt') }} - - name: Download ${{ matrix.writer }} fixtures + - name: Download ${{ matrix.pair.writer }} fixtures uses: actions/download-artifact@v8 with: - name: tsfile-${{ matrix.writer }}-compat-fixtures - path: compat-fixtures/${{ matrix.writer }} + name: tsfile-${{ matrix.pair.writer }}-${{ matrix.compression }}-compat-fixtures + path: compat-fixtures/${{ matrix.pair.writer }} - - name: Validate ${{ matrix.writer }} fixtures with Java - if: matrix.reader == 'java' + - name: Validate ${{ matrix.pair.writer }} fixtures with Java + if: matrix.pair.reader == 'java' shell: bash run: | ./mvnw -P with-java \ -Dtest=TableModelEncodingCompressionCompatibilityTest#validateFixtures \ -Dsurefire.failIfNoSpecifiedTests=false \ - -Dtsfile.compat.validate.dir="$PWD/compat-fixtures/${{ matrix.writer }}" \ - test + -Dtsfile.compat.compression="${{ matrix.compression }}" \ + -Dtsfile.compat.validate.dir="$PWD/compat-fixtures/${{ matrix.pair.writer }}" \ + test-compile surefire:test - name: Build C++ tests - if: matrix.reader == 'cpp' + if: matrix.pair.reader == 'cpp' shell: bash run: | ./mvnw -P with-cpp \ @@ -204,19 +209,20 @@ jobs: -Dspotless.skip=true \ -Dtsfile.dependency.source=BUNDLED \ -Denable.antlr4=OFF \ - -Denable.snappy=OFF \ + -Denable.snappy=ON \ -Denable.lz4=ON \ -Denable.lzokay=OFF \ - -Denable.zlib=OFF \ + -Denable.zlib=ON \ -Denable.zstd=ON \ -Denable.lzma2=ON \ -Denable.simd=OFF \ package - - name: Validate ${{ matrix.writer }} fixtures with C++ - if: matrix.reader == 'cpp' + - name: Validate ${{ matrix.pair.writer }} fixtures with C++ + if: matrix.pair.reader == 'cpp' shell: bash run: | - TSFILE_COMPAT_VALIDATE_DIR="$PWD/compat-fixtures/${{ matrix.writer }}" \ + TSFILE_COMPAT_COMPRESSION="${{ matrix.compression }}" \ + TSFILE_COMPAT_VALIDATE_DIR="$PWD/compat-fixtures/${{ matrix.pair.writer }}" \ ./cpp/target/build/test/lib/TsFile_Test \ --gtest_filter=TableModelEncodingCompressionCompatibilityTest.ValidateFixtures diff --git a/cpp/src/encoding/decoder_factory.h b/cpp/src/encoding/decoder_factory.h index 16111a690..3247b93ba 100644 --- a/cpp/src/encoding/decoder_factory.h +++ b/cpp/src/encoding/decoder_factory.h @@ -82,6 +82,7 @@ class DecoderFactory { case RLE: switch (data_type) { + case BOOLEAN: case INT32: case DATE: ALLOC_AND_RETURN_DECODER(Int32RleDecoder); diff --git a/cpp/test/encoding/encoding_coverage_test.cc b/cpp/test/encoding/encoding_coverage_test.cc index 5d6ef813e..4c715d8b8 100644 --- a/cpp/test/encoding/encoding_coverage_test.cc +++ b/cpp/test/encoding/encoding_coverage_test.cc @@ -27,6 +27,7 @@ #include #include "common/allocator/byte_stream.h" +#include "encoding/decoder_factory.h" #include "encoding/dictionary_decoder.h" #include "encoding/dictionary_encoder.h" #include "encoding/gorilla_decoder.h" @@ -45,6 +46,13 @@ namespace storage { +TEST(EncodingCoverage, BooleanRleDecoderFactory) { + Decoder* decoder = + DecoderFactory::alloc_value_decoder(common::RLE, common::BOOLEAN); + ASSERT_NE(nullptr, decoder); + DecoderFactory::free(decoder); +} + // ── Type-mismatch returns ──────────────────────────────────────────────── // // Every codec exposes read_boolean / read_int32 / read_int64 / read_float / diff --git a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc b/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc index 8dedc34eb..c6e4edeb3 100644 --- a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc +++ b/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -65,16 +67,16 @@ const char* const kValidateDirEnv = "TSFILE_COMPAT_VALIDATE_DIR"; const char* const kManifestFile = "manifest.csv"; const char* const kManifestHeader = "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount," - "nullPattern"; + "nullPattern,booleanEncoding"; const char* const kTableName = "compat_table"; const char* const kTagColumn = "device"; const char* const kValueColumn = "value"; const char* const kTagValue = "compat_device"; +const char* const kBooleanColumn = "boolean_value"; +constexpr int kRowCounts[] = {32, 4096, 1000000}; +constexpr uint32_t kNullPercent = 10; const char* const kAnchorColumn = "anchor"; const int kNullPageRows = 64; -// Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so -// page-wide bitmaps must survive block transitions. Applied to every case. -constexpr int kRowCount = 300; const int32_t kIntValues[] = { 0, @@ -187,6 +189,7 @@ struct FixtureCase { CompressionType compression; int row_count; std::string null_pattern = "NONE"; + TSEncoding boolean_encoding = PLAIN; FixtureCase() : table_name(kTableName), @@ -228,7 +231,7 @@ struct FixtureCase { << value_column << "," << get_data_type_name(data_type) << "," << get_encoding_name(encoding) << "," << get_compression_name(compression) << "," << row_count << "," - << null_pattern; + << null_pattern << "," << get_encoding_name(boolean_encoding); return out.str(); } }; @@ -271,7 +274,7 @@ bool ParseCompressionName(const std::string& value, CompressionType& out) { FixtureCase ParseManifestLine(const std::string& line) { FixtureCase fixture_case; std::vector parts = Split(line, ','); - if (parts.size() != 9) { + if (parts.size() != 10) { ADD_FAILURE() << "Bad manifest line: " << line; return fixture_case; } @@ -290,7 +293,9 @@ FixtureCase ParseManifestLine(const std::string& line) { } fixture_case.row_count = std::atoi(parts[7].c_str()); fixture_case.null_pattern = parts[8]; + EXPECT_TRUE(ParseEncodingName(parts[9], fixture_case.boolean_encoding)); EXPECT_TRUE(fixture_case.null_pattern == "NONE" || + fixture_case.null_pattern == "RANDOM" || fixture_case.null_pattern == "MIXED" || fixture_case.null_pattern == "ALL"); return fixture_case; @@ -481,22 +486,51 @@ int32_t DateValue(int row) { return kDateValues[row % (sizeof(kDateValues) / sizeof(kDateValues[0]))]; } +// Implemented intersection with Java (keep in sync with cppSupports there). +// Fixture generation and validation exercise the real factories for every case. +bool SupportsCommonEncoding(TSDataType type, TSEncoding encoding) { + bool integer = type == INT32 || type == INT64; + bool integral = integer || type == DATE || type == TIMESTAMP; + bool floating = type == FLOAT || type == DOUBLE; + switch (encoding) { + case PLAIN: + return true; + case DICTIONARY: + return type == TEXT || type == STRING; + case RLE: + return integral; + case TS_2DIFF: + case GORILLA: + case CHIMP: + case RLBE: + return integral || floating; + case ZIGZAG: + return integer; + case SPRINTZ: + return integer || floating; + case CAMEL: + return type == DOUBLE; + default: + return false; + } +} + std::vector BuildMatrix() { std::vector cases; - CompressionType compressions[] = {UNCOMPRESSED, ZSTD, LZMA2}; + CompressionType compressions[] = {UNCOMPRESSED, SNAPPY, GZIP, + LZ4, ZSTD, LZMA2}; TSDataType data_types[] = {INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE}; + TSDataType all_types[] = {BOOLEAN, INT32, DATE, INT64, TIMESTAMP, + FLOAT, DOUBLE, TEXT, STRING, BLOB}; for (CompressionType compression : compressions) { - for (TSDataType data_type : data_types) { - cases.emplace_back(data_type, CHIMP, compression, kRowCount); - cases.emplace_back(data_type, RLBE, compression, kRowCount); - if (data_type == FLOAT || data_type == DOUBLE) { - // The value set exercises scaled, scale-overflow, and - // raw-bit forms, including the page-wide bitmaps across - // the 129-value TS_2DIFF block boundary. - cases.emplace_back(data_type, TS_2DIFF, compression, kRowCount); + for (TSDataType data_type : all_types) { + for (int value = PLAIN; value <= CAMEL; ++value) { + TSEncoding encoding = static_cast(value); + if (SupportsCommonEncoding(data_type, encoding)) + cases.emplace_back(data_type, encoding, compression, + kRowCounts[0]); } } - cases.emplace_back(DOUBLE, CAMEL, compression, kRowCount); } for (CompressionType compression : {UNCOMPRESSED, LZ4, ZSTD, LZMA2}) { for (TSDataType data_type : data_types) { @@ -513,12 +547,78 @@ std::vector BuildMatrix() { cases.push_back(fixture); } } - return cases; + std::vector scaled; + const char* compression_filter = std::getenv("TSFILE_COMPAT_COMPRESSION"); + const char* rows_filter = std::getenv("TSFILE_COMPAT_ROWS"); + for (const FixtureCase& fixture : cases) { + if (fixture.null_pattern != "NONE") { + scaled.push_back(fixture); + } else { + for (int rows : kRowCounts) { + FixtureCase copy = fixture; + copy.row_count = rows; + copy.null_pattern = "RANDOM"; + copy.file_name += "." + std::to_string(rows); + scaled.push_back(copy); + } + } + } + scaled.erase( + std::remove_if(scaled.begin(), scaled.end(), + [&](const FixtureCase& f) { + return (compression_filter != nullptr && + std::string(get_compression_name( + f.compression)) != compression_filter) || + (rows_filter != nullptr && + f.row_count != std::atoi(rows_filter)); + }), + scaled.end()); + return scaled; +} + +std::set CoverageKeys(const std::vector& cases) { + std::set keys; + const char* compression_filter = std::getenv("TSFILE_COMPAT_COMPRESSION"); + const char* rows_filter = std::getenv("TSFILE_COMPAT_ROWS"); + for (const FixtureCase& f : cases) { + if ((compression_filter != nullptr && + std::string(get_compression_name(f.compression)) != + compression_filter) || + (rows_filter != nullptr && f.row_count != std::atoi(rows_filter))) + continue; + keys.insert(std::string(get_data_type_name(f.data_type)) + "/" + + get_encoding_name(f.encoding) + "/" + + get_compression_name(f.compression) + "/" + + std::to_string(f.row_count) + "/" + f.null_pattern); + } + return keys; +} + +// Same fixed-seed unsigned 32-bit mixing and per-column salts as Java. +uint32_t RowHash(int row, uint32_t salt) { + uint32_t value = + (static_cast(row) + 1) * UINT32_C(0x9e3779b9) ^ salt; + value ^= value >> 16; + value *= UINT32_C(0x85ebca6b); + value ^= value >> 13; + return value; +} + +bool RandomNull(int row, uint32_t salt) { + return RowHash(row, salt) % 100 < kNullPercent; +} + +bool BooleanNull(int row) { return RandomNull(row, UINT32_C(0x87654321)); } + +bool BooleanValue(int row) { + return row % 192 < 64 ? row % 2 == 0 : row % 192 < 128; } // Shared contract with the Java test: empty first/middle/consecutive/trailing // pages, non-empty pages 1/5, and alternating null values inside page 4. bool IsNull(const FixtureCase& fixture_case, int row) { + if (fixture_case.null_pattern == "RANDOM") + return RandomNull(row, UINT32_C(0x12345678)); if (fixture_case.null_pattern == "ALL") return true; if (fixture_case.null_pattern == "NONE") return false; int page = row / kNullPageRows; @@ -533,7 +633,8 @@ class ScopedNullPageSize { : old_rows_(g_config_value_.page_writer_max_point_num_), old_bytes_(g_config_value_.page_writer_max_memory_bytes_) { if (fixture_case.null_pattern != "NONE") { - g_config_value_.page_writer_max_point_num_ = kNullPageRows; + g_config_value_.page_writer_max_point_num_ = + fixture_case.null_pattern == "RANDOM" ? 4096 : kNullPageRows; g_config_value_.page_writer_max_memory_bytes_ = 1024 * 1024; } } @@ -557,6 +658,10 @@ TableSchema* CreateTableSchema(const FixtureCase& fixture_case) { new MeasurementSchema(fixture_case.value_column, fixture_case.data_type, fixture_case.encoding, fixture_case.compression)); column_categories.emplace_back(ColumnCategory::FIELD); + measurement_schemas.emplace_back(new MeasurementSchema( + kBooleanColumn, BOOLEAN, fixture_case.boolean_encoding, + fixture_case.compression)); + column_categories.emplace_back(ColumnCategory::FIELD); if (fixture_case.null_pattern != "NONE") { measurement_schemas.emplace_back(new MeasurementSchema( kAnchorColumn, INT64, PLAIN, fixture_case.compression)); @@ -566,8 +671,27 @@ TableSchema* CreateTableSchema(const FixtureCase& fixture_case) { column_categories); } +std::string BinaryValue(int row) { + static const std::string values[] = { + "", "alpha", std::string("\x00\x01\xff\x7f", 4), "alpha", "中文"}; + return values[row % 5]; +} + void AddValue(Tablet& tablet, const FixtureCase& fixture_case, int row) { switch (fixture_case.data_type) { + case BOOLEAN: + ASSERT_EQ(E_OK, + tablet.add_value(row, kValueColumn, BooleanValue(row))); + break; + case TEXT: + case STRING: + case BLOB: { + std::string bytes = BinaryValue(row); + common::String value(bytes.data(), + static_cast(bytes.size())); + ASSERT_EQ(E_OK, tablet.add_value(row, kValueColumn, value)); + break; + } case INT32: ASSERT_EQ(E_OK, tablet.add_value(row, kValueColumn, IntValue(row))); break; @@ -609,6 +733,10 @@ Tablet CreateTablet(TableSchema* table_schema, if (!IsNull(fixture_case, row)) { AddValue(tablet, fixture_case, row); } + if (!BooleanNull(row)) { + EXPECT_EQ(E_OK, + tablet.add_value(row, kBooleanColumn, BooleanValue(row))); + } if (fixture_case.null_pattern != "NONE") { EXPECT_EQ(E_OK, tablet.add_value(row, kAnchorColumn, static_cast(row))); @@ -672,6 +800,17 @@ std::vector ReadManifest(const std::string& directory) { void AssertValue(const FixtureCase& fixture_case, int row, TableResultSet* result_set) { switch (fixture_case.data_type) { + case TEXT: + case STRING: + case BLOB: + ASSERT_EQ( + BinaryValue(row), + result_set->get_value(3)->to_std_string()); + break; + case BOOLEAN: + ASSERT_EQ(row % 192 < 64 ? row % 2 == 0 : row % 192 < 128, + result_set->get_value(3)); + break; case INT32: ASSERT_EQ(IntValue(row), result_set->get_value(3)); break; @@ -696,8 +835,9 @@ void AssertValue(const FixtureCase& fixture_case, int row, } } -void AssertOnWireCodec(const std::string& directory, - const FixtureCase& fixture_case) { +void AssertColumnCodec(const std::string& directory, + const FixtureCase& fixture_case, + const std::string& measurement, TSEncoding encoding) { TsFileIOReader io_reader; ASSERT_EQ(E_OK, io_reader.init(JoinPath(directory, fixture_case.file_name))); @@ -714,7 +854,7 @@ void AssertOnWireCodec(const std::string& directory, common::PageArena arena; arena.init(512, common::MOD_TSFILE_READER); std::vector indexes(1, nullptr); - std::unordered_set measurements = {fixture_case.value_column}; + std::unordered_set measurements = {measurement}; ASSERT_EQ(E_OK, io_reader.get_timeseries_indexes(device_id, measurements, indexes, arena)); ASSERT_NE(nullptr, indexes.front()); @@ -743,11 +883,21 @@ void AssertOnWireCodec(const std::string& directory, stream.wrap_from(header_buf, read_len); ChunkHeader chunk_header; ASSERT_EQ(E_OK, chunk_header.deserialize_from(stream)); - ASSERT_EQ(fixture_case.encoding, chunk_header.encoding_type_); + ASSERT_EQ(encoding, chunk_header.encoding_type_); ASSERT_EQ(fixture_case.compression, chunk_header.compression_type_); } } +void AssertOnWireCodec(const std::string& directory, + const FixtureCase& fixture_case) { + ASSERT_NO_FATAL_FAILURE(AssertColumnCodec(directory, fixture_case, + fixture_case.value_column, + fixture_case.encoding)); + ASSERT_NO_FATAL_FAILURE(AssertColumnCodec(directory, fixture_case, + kBooleanColumn, + fixture_case.boolean_encoding)); +} + void ValidateQuery(const std::string& directory, const FixtureCase& fixture_case, bool with_anchor, int start, int end) { @@ -756,8 +906,8 @@ void ValidateQuery(const std::string& directory, << " range=" << start << ":" << end); TsFileReader reader; ASSERT_EQ(E_OK, reader.open(JoinPath(directory, fixture_case.file_name))); - std::vector columns = {fixture_case.tag_column, - fixture_case.value_column}; + std::vector columns = { + fixture_case.tag_column, fixture_case.value_column, kBooleanColumn}; if (with_anchor) columns.push_back(kAnchorColumn); ResultSet* raw_result_set = nullptr; ASSERT_EQ(E_OK, reader.query(fixture_case.table_name, columns, start, end, @@ -771,7 +921,7 @@ void ValidateQuery(const std::string& directory, std::unique_ptr owned_result(result_set, cleanup); std::shared_ptr metadata = result_set->get_metadata(); - ASSERT_EQ(with_anchor ? 4U : 3U, metadata->get_column_count()); + ASSERT_EQ(with_anchor ? 5U : 4U, metadata->get_column_count()); ASSERT_EQ("time", metadata->get_column_name(1)); ASSERT_EQ(INT64, metadata->get_column_type(1)); ASSERT_EQ(fixture_case.tag_column, metadata->get_column_name(2)); @@ -779,6 +929,8 @@ void ValidateQuery(const std::string& directory, ASSERT_EQ(fixture_case.value_column, metadata->get_column_name(3)); ASSERT_EQ(fixture_case.data_type, metadata->get_column_type(3)); + ASSERT_EQ(kBooleanColumn, metadata->get_column_name(4)); + ASSERT_EQ(BOOLEAN, metadata->get_column_type(4)); bool has_next = false; int row = start; while (true) { @@ -794,9 +946,13 @@ void ValidateQuery(const std::string& directory, if (!IsNull(fixture_case, row)) { ASSERT_NO_FATAL_FAILURE(AssertValue(fixture_case, row, result_set)); } + ASSERT_EQ(BooleanNull(row), result_set->is_null(4)); + if (!BooleanNull(row)) { + ASSERT_EQ(BooleanValue(row), result_set->get_value(4)); + } if (with_anchor) { - ASSERT_FALSE(result_set->is_null(4)); - ASSERT_EQ(row, result_set->get_value(4)); + ASSERT_FALSE(result_set->is_null(5)); + ASSERT_EQ(row, result_set->get_value(5)); } ++row; } @@ -809,12 +965,21 @@ void ValidateFixture(const std::string& directory, const FixtureCase& fixture_case) { SCOPED_TRACE(fixture_case.ToManifestLine()); ASSERT_NO_FATAL_FAILURE(AssertOnWireCodec(directory, fixture_case)); + if (fixture_case.null_pattern == "RANDOM") { + ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, true, 0, + fixture_case.row_count - 1)); + ASSERT_NO_FATAL_FAILURE( + ValidateQuery(directory, fixture_case, false, + std::max(0, fixture_case.row_count - 3), + fixture_case.row_count - 1)); + return; + } ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, false, 0, fixture_case.row_count - 1)); if (fixture_case.null_pattern != "NONE") { ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, true, 0, fixture_case.row_count - 1)); - if (fixture_case.row_count > kNullPageRows) { + if (fixture_case.row_count > 383) { ASSERT_NO_FATAL_FAILURE( ValidateQuery(directory, fixture_case, false, 63, 321)); ASSERT_NO_FATAL_FAILURE( @@ -840,6 +1005,7 @@ TEST_F(TableModelEncodingCompressionCompatibilityTest, GenerateFixtures) { } EnsureDirectory(output_dir); std::vector cases = BuildMatrix(); + ASSERT_FALSE(cases.empty()); for (const FixtureCase& fixture_case : cases) { WriteFixture(output_dir, fixture_case); AssertOnWireCodec(output_dir, fixture_case); @@ -853,7 +1019,19 @@ TEST_F(TableModelEncodingCompressionCompatibilityTest, ValidateFixtures) { GTEST_SKIP() << kValidateDirEnv << " is not set"; } std::vector cases = ReadManifest(input_dir); + auto expected = CoverageKeys(BuildMatrix()); + ASSERT_FALSE(expected.empty()); + ASSERT_EQ(expected, CoverageKeys(cases)) << "writer/reader matrix coverage"; for (const FixtureCase& fixture_case : cases) { - ASSERT_NO_FATAL_FAILURE(ValidateFixture(input_dir, fixture_case)); + const char* compression_filter = + std::getenv("TSFILE_COMPAT_COMPRESSION"); + const char* rows_filter = std::getenv("TSFILE_COMPAT_ROWS"); + if ((compression_filter != nullptr && + std::string(get_compression_name(fixture_case.compression)) != + compression_filter) || + (rows_filter != nullptr && + fixture_case.row_count != std::atoi(rows_filter))) + continue; + EXPECT_NO_FATAL_FAILURE(ValidateFixture(input_dir, fixture_case)); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java index 033aea3a9..37f56fd4b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java @@ -42,6 +42,7 @@ import org.junit.Assume; import org.junit.Test; +import org.junit.runners.model.MultipleFailureException; import java.io.BufferedWriter; import java.io.File; @@ -55,10 +56,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -69,15 +73,16 @@ public class TableModelEncodingCompressionCompatibilityTest { private static final String VALIDATE_DIR_PROPERTY = "tsfile.compat.validate.dir"; private static final String MANIFEST_FILE = "manifest.csv"; private static final String MANIFEST_HEADER = - "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount,nullPattern"; + "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount,nullPattern,booleanEncoding"; private static final String TABLE_NAME = "compat_table"; private static final String TAG_COLUMN = "device"; private static final String VALUE_COLUMN = "value"; private static final String TAG_VALUE = "compat_device"; - // Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so - // page-wide bitmaps must survive block transitions. Applied to every case. - private static final int ROW_COUNT = 300; + private static final int NULL_PAGE_ROWS = 64; + private static final String BOOLEAN_COLUMN = "boolean_value"; + private static final int[] ROW_COUNTS = {32, 4096, 1_000_000}; + private static final int NULL_PERCENT = 10; private static final String ANCHOR_COLUMN = "anchor"; // Keep this contract in sync with the C++ compatibility test. MIXED has @@ -85,11 +90,15 @@ public class TableModelEncodingCompressionCompatibilityTest { // page 4 mixes null/non-null rows and pages 1/5 are entirely non-null. private enum NullPattern { NONE, + RANDOM, MIXED, ALL } private static boolean isNull(FixtureCase fixtureCase, int row) { + if (fixtureCase.nullPattern == NullPattern.RANDOM) { + return randomNull(row, 0x12345678); + } if (fixtureCase.nullPattern == NullPattern.ALL) { return true; } @@ -100,6 +109,24 @@ private static boolean isNull(FixtureCase fixtureCase, int row) { return page == 0 || page == 2 || page == 3 || page >= 6 || (page == 4 && row % 2 == 0); } + // Fixed-seed 32-bit mixing, shared with C++. Separate salts make each + // column's 10% NULL mask independent and reproducible in either language. + private static int rowHash(int row, int salt) { + int value = (row + 1) * 0x9e3779b9 ^ salt; + value ^= value >>> 16; + value *= 0x85ebca6b; + value ^= value >>> 13; + return value; + } + + private static boolean randomNull(int row, int salt) { + return Integer.remainderUnsigned(rowHash(row, salt), 100) < NULL_PERCENT; + } + + private static boolean booleanNull(int row) { + return randomNull(row, 0x87654321); + } + private static final int[] INT_VALUES = { 0, 1, @@ -235,6 +262,7 @@ public void generateFixtures() throws Exception { Files.createDirectories(directory); List cases = buildMatrix(); + assertFalse("empty compatibility matrix", cases.isEmpty()); for (FixtureCase fixtureCase : cases) { writeFixture(directory, fixtureCase); } @@ -248,34 +276,75 @@ public void validateFixtures() throws Exception { VALIDATE_DIR_PROPERTY + " is not set", inputDir != null && !inputDir.isEmpty()); Path directory = Paths.get(inputDir); - for (FixtureCase fixtureCase : readManifest(directory)) { - validateFixture(directory, fixtureCase); + List fixtures = readManifest(directory); + Set expectedCoverage = new HashSet<>(); + for (FixtureCase fixtureCase : buildMatrix()) { + expectedCoverage.add(coverageKey(fixtureCase)); + } + Set actualCoverage = new HashSet<>(); + List failures = new ArrayList<>(); + for (FixtureCase fixtureCase : fixtures) { + String compressionFilter = System.getProperty("tsfile.compat.compression", ""); + String rowsFilter = System.getProperty("tsfile.compat.rows", ""); + if ((!compressionFilter.isEmpty() + && !fixtureCase.compression.name().equals(compressionFilter)) + || (!rowsFilter.isEmpty() && fixtureCase.rowCount != Integer.parseInt(rowsFilter))) { + continue; + } + actualCoverage.add(coverageKey(fixtureCase)); + try { + validateFixture(directory, fixtureCase); + } catch (Exception | AssertionError failure) { + failures.add(new AssertionError(fixtureCase.fileName, failure)); + } } + assertFalse("empty compatibility coverage", expectedCoverage.isEmpty()); + assertEquals("writer/reader matrix coverage", expectedCoverage, actualCoverage); + MultipleFailureException.assertEmpty(failures); + } + + private static String coverageKey(FixtureCase fixtureCase) { + return fixtureCase.dataType + + "/" + + fixtureCase.encoding + + "/" + + fixtureCase.compression + + "/" + + fixtureCase.rowCount + + "/" + + fixtureCase.nullPattern; } private static List buildMatrix() { List cases = new ArrayList<>(); List compressions = - Arrays.asList(CompressionType.UNCOMPRESSED, CompressionType.ZSTD, CompressionType.LZMA2); + Arrays.asList( + CompressionType.UNCOMPRESSED, + CompressionType.SNAPPY, + CompressionType.GZIP, + CompressionType.LZ4, + CompressionType.ZSTD, + CompressionType.LZMA2); for (CompressionType compression : compressions) { for (TSDataType dataType : Arrays.asList( + TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.DATE, TSDataType.INT64, TSDataType.TIMESTAMP, TSDataType.FLOAT, - TSDataType.DOUBLE)) { - cases.add(new FixtureCase(dataType, TSEncoding.CHIMP, compression, ROW_COUNT)); - cases.add(new FixtureCase(dataType, TSEncoding.RLBE, compression, ROW_COUNT)); + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB)) { + for (TSEncoding encoding : TSEncoding.values()) { + if (cppSupports(dataType, encoding)) { + assertTrue(dataType + "/" + encoding, TSEncoding.isSupported(dataType, encoding)); + cases.add(new FixtureCase(dataType, encoding, compression, ROW_COUNTS[0])); + } + } if (dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE) { - // The value set exercises scaled, scale-overflow, and raw-bit - // forms, including the page-wide bitmaps across the 129-value - // TS_2DIFF block boundary. - cases.add(new FixtureCase(dataType, TSEncoding.TS_2DIFF, compression, ROW_COUNT)); - // Explicit max_point_number=0 fixture: a canonical page starting - // with the 0x00 maxPointNumber byte must enter the Form 1 - // parsing path. cases.add( new FixtureCase( FixtureCase.fileName(dataType, TSEncoding.TS_2DIFF, compression) + ".mpn0", @@ -285,10 +354,9 @@ private static List buildMatrix() { dataType, TSEncoding.TS_2DIFF, compression, - ROW_COUNT)); + ROW_COUNTS[0])); } } - cases.add(new FixtureCase(TSDataType.DOUBLE, TSEncoding.CAMEL, compression, ROW_COUNT)); } for (CompressionType compression : Arrays.asList( @@ -310,7 +378,63 @@ private static List buildMatrix() { cases.add(new FixtureCase(dataType, compression, NULL_PAGE_ROWS, NullPattern.ALL)); } } - return cases; + List scaled = new ArrayList<>(); + for (FixtureCase fixtureCase : cases) { + if (fixtureCase.nullPattern != NullPattern.NONE) { + scaled.add(fixtureCase); + continue; + } + for (int rows : ROW_COUNTS) { + String name = fixtureCase.fileName.replace(".tsfile", "." + rows + ".tsfile"); + scaled.add( + new FixtureCase( + name, + TABLE_NAME, + TAG_COLUMN, + VALUE_COLUMN, + fixtureCase.dataType, + fixtureCase.encoding, + fixtureCase.compression, + rows, + NullPattern.RANDOM)); + } + } + String compressionFilter = System.getProperty("tsfile.compat.compression", ""); + String rowsFilter = System.getProperty("tsfile.compat.rows", ""); + scaled.removeIf( + f -> + (!compressionFilter.isEmpty() && !f.compression.name().equals(compressionFilter)) + || (!rowsFilter.isEmpty() && f.rowCount != Integer.parseInt(rowsFilter))); + return scaled; + } + + // Intersection with the implemented C++ encoder/decoder factories. Excludes + // enum placeholders and Java-only FLOAT RLE/GORILLA_V1/REGULAR combinations. + private static boolean cppSupports(TSDataType type, TSEncoding encoding) { + boolean integer = type == TSDataType.INT32 || type == TSDataType.INT64; + boolean integral = integer || type == TSDataType.DATE || type == TSDataType.TIMESTAMP; + boolean floating = type == TSDataType.FLOAT || type == TSDataType.DOUBLE; + switch (encoding) { + case PLAIN: + return true; + case DICTIONARY: + return type == TSDataType.TEXT || type == TSDataType.STRING; + case RLE: + return integral; + case TS_2DIFF: + case GORILLA: + case CHIMP: + case RLBE: + return integral || floating; + case ZIGZAG: + return integer; + case SPRINTZ: + return integer || floating; + case CAMEL: + return type == TSDataType.DOUBLE; + default: + return false; + } } private static void writeFixture(Path directory, FixtureCase fixtureCase) throws Exception { @@ -321,7 +445,8 @@ private static void writeFixture(Path directory, FixtureCase fixtureCase) throws int oldPageBytes = config.getPageSizeInByte(); try { if (fixtureCase.nullPattern != NullPattern.NONE) { - config.setMaxNumberOfPointsInPage(NULL_PAGE_ROWS); + config.setMaxNumberOfPointsInPage( + fixtureCase.nullPattern == NullPattern.RANDOM ? 4096 : NULL_PAGE_ROWS); config.setPageSizeInByte(1024 * 1024); } try (TsFileWriter writer = new TsFileWriter(file)) { @@ -365,10 +490,21 @@ private static void assertEmptyPages(File file, FixtureCase fixtureCase) throws } private static void validateFixture(Path directory, FixtureCase fixtureCase) throws Exception { + assertOnWireCodecs(directory, fixtureCase); + if (fixtureCase.nullPattern == NullPattern.RANDOM) { + validateQuery(directory, fixtureCase, true, 0, fixtureCase.rowCount - 1); + validateQuery( + directory, + fixtureCase, + false, + Math.max(0, fixtureCase.rowCount - 3), + fixtureCase.rowCount - 1); + return; + } validateQuery(directory, fixtureCase, false, 0, fixtureCase.rowCount - 1); if (fixtureCase.nullPattern != NullPattern.NONE) { validateQuery(directory, fixtureCase, true, 0, fixtureCase.rowCount - 1); - if (fixtureCase.rowCount > NULL_PAGE_ROWS) { + if (fixtureCase.rowCount > 383) { // Start inside/at page boundaries and query an entirely null interval. validateQuery(directory, fixtureCase, false, 63, 321); validateQuery(directory, fixtureCase, true, 128, 255); @@ -377,12 +513,31 @@ private static void validateFixture(Path directory, FixtureCase fixtureCase) thr } } + private static void assertOnWireCodecs(Path directory, FixtureCase fixtureCase) throws Exception { + try (TsFileSequenceReader reader = + new TsFileSequenceReader(directory.resolve(fixtureCase.fileName).toString())) { + String[] columns = {fixtureCase.valueColumn, BOOLEAN_COLUMN}; + TSEncoding[] encodings = {fixtureCase.encoding, fixtureCase.booleanEncoding}; + for (int index = 0; index < columns.length; index++) { + List chunks = + reader.getChunkMetadataList(reader.getAllDevices().get(0), columns[index], false); + assertFalse(columns[index] + " has no chunks", chunks.isEmpty()); + for (ChunkMetadata chunk : chunks) { + Chunk data = reader.readMemChunk(chunk); + assertEquals(encodings[index], data.getHeader().getEncodingType()); + assertEquals(fixtureCase.compression, data.getHeader().getCompressionType()); + } + } + } + } + private static void validateQuery( Path directory, FixtureCase fixtureCase, boolean withAnchor, int start, int end) throws Exception { File file = directory.resolve(fixtureCase.fileName).toFile(); List columns = - new ArrayList<>(Arrays.asList(fixtureCase.tagColumn, fixtureCase.valueColumn)); + new ArrayList<>( + Arrays.asList(fixtureCase.tagColumn, fixtureCase.valueColumn, BOOLEAN_COLUMN)); if (withAnchor) { columns.add(ANCHOR_COLUMN); } @@ -395,6 +550,8 @@ private static void validateQuery( assertEquals(TSDataType.STRING, metadata.getColumnType(2)); assertEquals(fixtureCase.valueColumn, metadata.getColumnName(3)); assertEquals(fixtureCase.dataType, metadata.getColumnType(3)); + assertEquals(BOOLEAN_COLUMN, metadata.getColumnName(4)); + assertEquals(TSDataType.BOOLEAN, metadata.getColumnType(4)); int row = start; while (resultSet.next()) { assertTrue("unexpected row in " + fixtureCase.fileName, row <= end); @@ -408,9 +565,13 @@ private static void validateQuery( if (!isNull(fixtureCase, row)) { assertValue(fixtureCase, row, resultSet); } + assertEquals("BOOLEAN NULL at row " + row, booleanNull(row), resultSet.isNull(4)); + if (!booleanNull(row)) { + assertEquals("BOOLEAN at row " + row, booleanValue(row), resultSet.getBoolean(4)); + } if (withAnchor) { - assertFalse(resultSet.isNull(4)); - assertEquals(row, resultSet.getLong(4)); + assertFalse(resultSet.isNull(5)); + assertEquals(row, resultSet.getLong(5)); } row++; } @@ -423,6 +584,16 @@ private static TableSchema tableSchema(FixtureCase fixtureCase) { if (isTs2DiffFloatCase(fixtureCase) && fixtureCase.fileName.endsWith(".mpn0")) { props = Collections.singletonMap(Encoder.MAX_POINT_NUMBER, "0"); } + MeasurementSchema valueSchema = + new MeasurementSchema( + fixtureCase.valueColumn, + fixtureCase.dataType, + fixtureCase.encoding, + fixtureCase.compression, + props); + MeasurementSchema booleanSchema = new MeasurementSchema(BOOLEAN_COLUMN, TSDataType.BOOLEAN); + assertEquals("Java default BOOLEAN encoding", TSEncoding.RLE, booleanSchema.getEncodingType()); + booleanSchema.setCompressionType(fixtureCase.compression); List schemas = Arrays.asList( new MeasurementSchema( @@ -430,14 +601,11 @@ private static TableSchema tableSchema(FixtureCase fixtureCase) { TSDataType.STRING, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED), - new MeasurementSchema( - fixtureCase.valueColumn, - fixtureCase.dataType, - fixtureCase.encoding, - fixtureCase.compression, - props)); + valueSchema, + booleanSchema); List categories = - new ArrayList<>(Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + new ArrayList<>( + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); if (fixtureCase.nullPattern != NullPattern.NONE) { schemas = new ArrayList<>(schemas); schemas.add( @@ -464,6 +632,7 @@ private static Tablet tablet(TableSchema tableSchema, FixtureCase fixtureCase) { } else { addValue(tablet, fixtureCase, row); } + tablet.addValue(BOOLEAN_COLUMN, row, booleanNull(row) ? null : booleanValue(row)); if (fixtureCase.nullPattern != NullPattern.NONE) { tablet.addValue(ANCHOR_COLUMN, row, (long) row); } @@ -474,6 +643,14 @@ private static Tablet tablet(TableSchema tableSchema, FixtureCase fixtureCase) { private static void addValue(Tablet tablet, FixtureCase fixtureCase, int row) { TSDataType dataType = fixtureCase.dataType; switch (dataType) { + case TEXT: + case STRING: + case BLOB: + tablet.addValue(row, VALUE_COLUMN, binaryValue(row)); + break; + case BOOLEAN: + tablet.addValue(row, VALUE_COLUMN, booleanValue(row)); + break; case INT32: tablet.addValue(row, VALUE_COLUMN, intValue(row)); break; @@ -497,6 +674,14 @@ private static void addValue(Tablet tablet, FixtureCase fixtureCase, int row) { private static void assertValue(FixtureCase fixtureCase, int row, ResultSet resultSet) { switch (fixtureCase.dataType) { + case TEXT: + case STRING: + case BLOB: + assertArrayEquals("binary at row " + row, binaryValue(row), resultSet.getBinary(3)); + break; + case BOOLEAN: + assertEquals("BOOLEAN at row " + row, booleanValue(row), resultSet.getBoolean(3)); + break; case INT32: assertEquals("INT32 at row " + row, intValue(row), resultSet.getInt(3)); break; @@ -524,6 +709,23 @@ private static void assertValue(FixtureCase fixtureCase, int row, ResultSet resu } } + private static byte[] binaryValue(int row) { + // Empty values, repeated dictionary entries, embedded NUL and non-ASCII bytes. + return new byte[][] { + new byte[0], + {97, 108, 112, 104, 97}, + {0, 1, -1, 127}, + {97, 108, 112, 104, 97}, + {-28, -72, -83, -26, -106, -121} + } + [row % 5]; + } + + // Alternate bits and long true/false runs to cover both RLE packing modes. + private static boolean booleanValue(int row) { + return row % 192 < 64 ? row % 2 == 0 : row % 192 < 128; + } + private static int intValue(int row) { return INT_VALUES[row % INT_VALUES.length]; } @@ -639,6 +841,7 @@ private static class FixtureCase { private final CompressionType compression; private final int rowCount; private final NullPattern nullPattern; + private TSEncoding booleanEncoding = TSEncoding.RLE; private FixtureCase( TSDataType dataType, TSEncoding encoding, CompressionType compression, int rowCount) { @@ -715,19 +918,22 @@ private FixtureCase( private static FixtureCase fromManifestLine(String line) { String[] parts = line.split(",", -1); - if (parts.length != 9) { + if (parts.length != 10) { throw new IllegalArgumentException("Bad manifest line: " + line); } - return new FixtureCase( - parts[0], - parts[1], - parts[2], - parts[3], - TSDataType.valueOf(parts[4]), - TSEncoding.valueOf(parts[5]), - CompressionType.valueOf(parts[6]), - Integer.parseInt(parts[7]), - NullPattern.valueOf(parts[8])); + FixtureCase fixtureCase = + new FixtureCase( + parts[0], + parts[1], + parts[2], + parts[3], + TSDataType.valueOf(parts[4]), + TSEncoding.valueOf(parts[5]), + CompressionType.valueOf(parts[6]), + Integer.parseInt(parts[7]), + NullPattern.valueOf(parts[8])); + fixtureCase.booleanEncoding = TSEncoding.valueOf(parts[9]); + return fixtureCase; } private String toManifestLine() { @@ -741,7 +947,8 @@ private String toManifestLine() { encoding.name(), compression.name(), String.valueOf(rowCount), - nullPattern.name()); + nullPattern.name(), + booleanEncoding.name()); } private static String fileName(