From f57a920febe9ab2f9df3824cd407218f787e2d05 Mon Sep 17 00:00:00 2001 From: tamirms Date: Sun, 25 Oct 2026 16:47:04 +0000 Subject: [PATCH 1/2] Add a sorted-file format for index build inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stellar-rpc keeps its txhash build inputs in sorted .bin files of its own and merges them with code ported from cmd/bench. This moves the format and the merge into streamhash, so that a later change can build an index straight from many sorted files. A sorted file is a header padded to 4096 bytes, then fixed-width records of a key's first 16 bytes and a payload of 0 to 8 bytes, in key order (spec §3.9). SortedFileWriter writes one and refuses keys out of order, ReadSortedFileHeader and OpenSortedFile check and read one, and MergeSortedFiles merges several, checking each input's order as it reads. ErrTruncatedFile and ErrCorruptedIndex no longer say "index" in their messages, since sorted files return them too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DjQed1VX9uGU9PWpAjBiz7 --- doc.go | 3 +- internal/sherr/errors.go | 6 +- odirect.go | 16 ++ odirect_linux.go | 21 ++ odirect_other.go | 11 + sentinels.go | 2 +- sorted_file.go | 403 ++++++++++++++++++++++++++++++++ sorted_file_test.go | 487 +++++++++++++++++++++++++++++++++++++++ sorted_merge.go | 242 +++++++++++++++++++ sorted_merge_test.go | 285 +++++++++++++++++++++++ streamhash-spec.md | 47 ++++ 11 files changed, 1518 insertions(+), 5 deletions(-) create mode 100644 odirect.go create mode 100644 odirect_linux.go create mode 100644 odirect_other.go create mode 100644 sorted_file.go create mode 100644 sorted_file_test.go create mode 100644 sorted_merge.go create mode 100644 sorted_merge_test.go diff --git a/doc.go b/doc.go index 29c0971..d65fbf1 100644 --- a/doc.go +++ b/doc.go @@ -59,8 +59,9 @@ // - Public API: builder.go (NewSortedBuilder), builder_unsorted.go (NewUnsortedBuilder), index.go (Open, QueryRank, QueryBatch, PayloadIndex) // - Configuration: builder_options.go (BuildOption, With* functions) // - Serialization: header.go (header, footer, ramIndexEntry), index_writer.go +// - Sorted files: sorted_file.go (SortedFileWriter, ReadSortedFileHeader, OpenSortedFile), sorted_merge.go (MergeSortedFiles) // - Key routing & hashing: key.go (fingerprint extraction, payload packing), prehash.go (PreHash); block routing is FastRange32 over the big-endian prefix, inline in builder.go/index.go // - Algorithm dispatch: algorithm.go (blockBuilder/blockDecoder interfaces, factory functions) // - Block algorithms: internal/bijection/ (EF/GR), internal/ptrhash/ (Cuckoo) -// - Platform: platform_*.go (OS-specific: fallocate), madvise_*.go (mmap read-around advice) +// - Platform: platform_*.go (OS-specific: fallocate), madvise_*.go (mmap read-around advice), odirect*.go (direct reads) package streamhash diff --git a/internal/sherr/errors.go b/internal/sherr/errors.go index 0d72baf..906bf60 100644 --- a/internal/sherr/errors.go +++ b/internal/sherr/errors.go @@ -24,13 +24,13 @@ var ( ErrIndistinguishableHashes = errors.New("streamhash: indistinguishable hashes in bucket - retry with different globalSeed") ) -// Index errors +// Index and sorted-file errors var ( ErrInvalidMagic = errors.New("streamhash: invalid magic number") ErrInvalidVersion = errors.New("streamhash: unsupported version") ErrChecksumFailed = errors.New("streamhash: file checksum verification failed") - ErrTruncatedFile = errors.New("streamhash: index file is truncated") - ErrCorruptedIndex = errors.New("streamhash: index data is corrupted") + ErrTruncatedFile = errors.New("streamhash: file is truncated") + ErrCorruptedIndex = errors.New("streamhash: data is corrupted") ) // Query errors diff --git a/odirect.go b/odirect.go new file mode 100644 index 0000000..8780d07 --- /dev/null +++ b/odirect.go @@ -0,0 +1,16 @@ +package streamhash + +import "unsafe" + +// directAlign is the alignment direct reads need of the buffer's address, +// the file offset and the length. +const directAlign = 4096 + +// alignedBuffer returns a size-byte slice aligned to directAlign. Go's GC does +// not move it, so the alignment holds. +func alignedBuffer(size int) []byte { + backing := make([]byte, size+directAlign) + misalign := int(uintptr(unsafe.Pointer(&backing[0])) % directAlign) + off := (directAlign - misalign) % directAlign + return backing[off : off+size : off+size] +} diff --git a/odirect_linux.go b/odirect_linux.go new file mode 100644 index 0000000..7d0943f --- /dev/null +++ b/odirect_linux.go @@ -0,0 +1,21 @@ +//go:build linux + +package streamhash + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +// openDirect opens path for reading with O_DIRECT, so reading a file once +// does not fill the page cache. A filesystem that refuses O_DIRECT gets a +// plain open; reads into aligned buffers at aligned offsets work either way. +func openDirect(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDONLY|unix.O_DIRECT, 0) + if errors.Is(err, unix.EINVAL) { + return os.Open(path) + } + return f, err +} diff --git a/odirect_other.go b/odirect_other.go new file mode 100644 index 0000000..dd3b582 --- /dev/null +++ b/odirect_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package streamhash + +import "os" + +// openDirect opens path for reading. Off Linux there is no O_DIRECT, so the +// reads go through the page cache. +func openDirect(path string) (*os.File, error) { + return os.Open(path) +} diff --git a/sentinels.go b/sentinels.go index 6a1733d..409cb18 100644 --- a/sentinels.go +++ b/sentinels.go @@ -29,7 +29,7 @@ var ( ErrBlockOverflow = sherr.ErrBlockOverflow ) -// Index errors. +// Index and sorted-file errors. var ( ErrInvalidMagic = sherr.ErrInvalidMagic ErrInvalidVersion = sherr.ErrInvalidVersion diff --git a/sorted_file.go b/sorted_file.go new file mode 100644 index 0000000..7e82180 --- /dev/null +++ b/sorted_file.go @@ -0,0 +1,403 @@ +package streamhash + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "os" +) + +// Sorted file header (spec §3.9): +// +// Offset Size Field Type +// 0 4 Magic 0x53545346 ("STSF") +// 4 2 Version 0x0001 +// 6 1 PayloadSize uint8 (0-8) +// 7 1 Reserved zero +// 8 8 Count uint64_le +// 16 4 MetadataLen uint32_le +// 20 ML Metadata +// +// Zero bytes pad the header to a multiple of 4096, and the records follow. +const ( + sortedFileMagic = uint32(0x53545346) + sortedFileVersion = uint16(0x0001) + + sortedFileFixedSize = 20 + sortedFileCountOffset = 8 + + // sortedFileAlign aligns the records, so direct reads can start at them. + // It is a multiple of directAlign. + sortedFileAlign = 4096 + + // maxSortedFileMetadata bounds what a corrupt header can make a reader allocate. + maxSortedFileMetadata = 64 << 10 +) + +// SortedFileConfig configures a sorted file. +type SortedFileConfig struct { + PayloadSize int // bytes per payload, 0 to 8 + Metadata []byte // stored in the header for ReadSortedFileHeader, at most 64 KiB + BytesPerSync int // start writeback every this many bytes, as WithBytesPerSync; 0 leaves it to the kernel +} + +// SortedFileHeader is a sorted file's header. +type SortedFileHeader struct { + PayloadSize int // bytes per payload, 0 to 8 + Count uint64 // records in the file + Metadata []byte // the SortedFileConfig.Metadata the file was written with +} + +// recordsOffset is where the file's records start. +func (h SortedFileHeader) recordsOffset() int64 { + return sortedFileRecordsOffset(len(h.Metadata)) +} + +// sortedFileRecordsOffset is the header's size: its fixed fields and the +// metadata, padded to sortedFileAlign. +func sortedFileRecordsOffset(metadataLen int) int64 { + return (sortedFileFixedSize + int64(metadataLen) + sortedFileAlign - 1) &^ (sortedFileAlign - 1) +} + +// sortKey is a key's first 16 bytes as two big-endian uint64s, which compare +// as the bytes do. +type sortKey struct{ hi, lo uint64 } + +func sortKeyOf(key []byte) sortKey { + return sortKey{binary.BigEndian.Uint64(key[0:8]), binary.BigEndian.Uint64(key[8:16])} +} + +func (k sortKey) less(o sortKey) bool { + if k.hi != o.hi { + return k.hi < o.hi + } + return k.lo < o.lo +} + +// SortedFileWriter writes a sorted file: keys with their payloads in key +// order, in fixed-width records of the key's first 16 bytes, then the payload +// as PayloadSize little-endian bytes. MergeSortedFiles merges several into one. +type SortedFileWriter struct { + f *os.File + bw *bufio.Writer + recordSize int + maxPayload uint64 + last sortKey + count uint64 + // record lives here rather than in Add: bufio may hand the slice to the + // file, which would move a local array to the heap per record. + record [MinKeySize + maxPayloadSize]byte + closed bool +} + +// NewSortedFileWriter creates the sorted file at path. Keys must be added in +// non-decreasing order of their first 16 bytes; a key out of order fails Add. +// Finish writes the record count and syncs. Close without Finish removes the file. +func NewSortedFileWriter(path string, cfg SortedFileConfig) (*SortedFileWriter, error) { + if cfg.PayloadSize < 0 || cfg.PayloadSize > maxPayloadSize { + return nil, ErrPayloadTooLarge + } + if len(cfg.Metadata) > maxSortedFileMetadata { + return nil, fmt.Errorf("sorted file metadata is %d bytes, over the maximum of %d", len(cfg.Metadata), maxSortedFileMetadata) + } + f, err := os.Create(path) + if err != nil { + return nil, err + } + w := &SortedFileWriter{ + f: f, + bw: bufio.NewWriterSize(&writebackWriter{f: f, bytesPerSync: cfg.BytesPerSync}, 1<<20), + recordSize: MinKeySize + cfg.PayloadSize, + maxPayload: ^uint64(0) >> (64 - 8*cfg.PayloadSize), + } + // The header fits in the empty buffer, so this write cannot fail. + _, _ = w.bw.Write(encodeSortedFileHeader(cfg)) + return w, nil +} + +// Add adds a record of key's first 16 bytes and payload, which must fit in +// PayloadSize bytes. +func (w *SortedFileWriter) Add(key []byte, payload uint64) error { + if w.closed { + return os.ErrClosed + } + if len(key) < MinKeySize { + return ErrKeyTooShort + } + if payload > w.maxPayload { + return ErrPayloadOverflow + } + k := sortKeyOf(key) + if k.less(w.last) { + return ErrUnsortedInput + } + w.last = k + copy(w.record[:MinKeySize], key) + binary.LittleEndian.PutUint64(w.record[MinKeySize:], payload) + if _, err := w.bw.Write(w.record[:w.recordSize]); err != nil { + return err + } + w.count++ + return nil +} + +// appendEncoded appends records already in the file's encoding, whose order +// the caller has checked. +func (w *SortedFileWriter) appendEncoded(records []byte) error { + if _, err := w.bw.Write(records); err != nil { + return err + } + w.count += uint64(len(records) / w.recordSize) + return nil +} + +// Finish writes the record count, syncs and closes the file. If it fails, it +// removes the file. +func (w *SortedFileWriter) Finish() error { + if w.closed { + return os.ErrClosed + } + w.closed = true + if err := w.commit(); err != nil { + return errors.Join(err, os.Remove(w.f.Name())) + } + return nil +} + +// commit flushes, patches the count, syncs and closes, in that order: a short +// file's header is still in the buffer until the flush, and ENOSPC and EIO +// can surface only at close. +func (w *SortedFileWriter) commit() error { + err := w.bw.Flush() + if err == nil { + var count [8]byte + binary.LittleEndian.PutUint64(count[:], w.count) + _, err = w.f.WriteAt(count[:], sortedFileCountOffset) + } + if err == nil { + err = w.f.Sync() + } + return errors.Join(err, w.f.Close()) +} + +// Close removes the file unless Finish has run. It is safe to call after Finish. +func (w *SortedFileWriter) Close() error { + if w.closed { + return nil + } + w.closed = true + return errors.Join(w.f.Close(), os.Remove(w.f.Name())) +} + +func encodeSortedFileHeader(cfg SortedFileConfig) []byte { + buf := make([]byte, sortedFileRecordsOffset(len(cfg.Metadata))) + binary.LittleEndian.PutUint32(buf[0:4], sortedFileMagic) + binary.LittleEndian.PutUint16(buf[4:6], sortedFileVersion) + buf[6] = byte(cfg.PayloadSize) + binary.LittleEndian.PutUint32(buf[16:20], uint32(len(cfg.Metadata))) + copy(buf[sortedFileFixedSize:], cfg.Metadata) + return buf +} + +// writebackWriter writes to f, starting writeback every bytesPerSync bytes. +type writebackWriter struct { + f *os.File + bytesPerSync int + unsynced int +} + +func (w *writebackWriter) Write(p []byte) (int, error) { + n, err := w.f.Write(p) + if w.bytesPerSync > 0 { + w.unsynced += n + if w.unsynced >= w.bytesPerSync { + w.unsynced = 0 + startWriteback(w.f) + } + } + return n, err +} + +// ReadSortedFileHeader reads and checks the header at path: magic, version, +// payload size, and that the record count matches the file's length. +func ReadSortedFileHeader(path string) (SortedFileHeader, error) { + f, err := os.Open(path) + if err != nil { + return SortedFileHeader{}, err + } + defer f.Close() + return readSortedFileHeader(f) +} + +// readSortedFileHeader reads the header of f, which must not be open for +// direct reads: these reads are not aligned. +func readSortedFileHeader(f *os.File) (SortedFileHeader, error) { + info, err := f.Stat() + if err != nil { + return SortedFileHeader{}, err + } + size := info.Size() + if size < sortedFileFixedSize { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w", f.Name(), ErrTruncatedFile) + } + var fixed [sortedFileFixedSize]byte + if _, err := f.ReadAt(fixed[:], 0); err != nil { + return SortedFileHeader{}, err + } + if binary.LittleEndian.Uint32(fixed[0:4]) != sortedFileMagic { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w", f.Name(), ErrInvalidMagic) + } + if binary.LittleEndian.Uint16(fixed[4:6]) != sortedFileVersion { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w", f.Name(), ErrInvalidVersion) + } + h := SortedFileHeader{PayloadSize: int(fixed[6]), Count: binary.LittleEndian.Uint64(fixed[8:16])} + if h.PayloadSize > maxPayloadSize { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w: payload size %d", f.Name(), ErrCorruptedIndex, h.PayloadSize) + } + metadataLen := binary.LittleEndian.Uint32(fixed[16:20]) + if metadataLen > maxSortedFileMetadata { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w: metadata length %d is over the maximum of %d", f.Name(), ErrCorruptedIndex, metadataLen, maxSortedFileMetadata) + } + recordsOffset := sortedFileRecordsOffset(int(metadataLen)) + if size < recordsOffset { + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w", f.Name(), ErrTruncatedFile) + } + if metadataLen > 0 { + h.Metadata = make([]byte, metadataLen) + if _, err := f.ReadAt(h.Metadata, sortedFileFixedSize); err != nil { + return SortedFileHeader{}, err + } + } + // Divide the trusted size rather than multiply the untrusted count, so a + // corrupt count cannot overflow. + body, recordSize := size-recordsOffset, int64(MinKeySize+h.PayloadSize) + if holds := uint64(body / recordSize); holds != h.Count || body%recordSize != 0 { + sentinel := ErrKeyCountMismatch + if holds < h.Count { + sentinel = ErrTruncatedFile + } + return SortedFileHeader{}, fmt.Errorf("sorted file %s: %w: the header counts %d records of %d bytes, the file holds %d bytes of records", + f.Name(), sentinel, h.Count, recordSize, body) + } + return h, nil +} + +// sortedRun is a sorted file's records: the file offsets where they start +// and end. +type sortedRun struct { + path string + start, end int64 +} + +// readAt reads the run's file f at off into buf, and returns how many bytes +// of buf are the run's records. A file shorter than its records fails with +// ErrTruncatedFile. +func (r sortedRun) readAt(f *os.File, buf []byte, off int64) (int, error) { + n, err := f.ReadAt(buf, off) + if err != nil && !errors.Is(err, io.EOF) { + return 0, err + } + want := int(min(int64(len(buf)), r.end-off)) + if n < want { + return 0, fmt.Errorf("sorted file %s: %w: it ends before its last record", f.Name(), ErrTruncatedFile) + } + return want, nil +} + +// readSortedRuns checks the headers of the sorted files at paths, which must +// share a payload size, and returns the runs of those with records, the +// payload size, and the total record count. +func readSortedRuns(paths []string) ([]sortedRun, int, uint64, error) { + var runs []sortedRun + var payloadSize int + var total uint64 + for i, path := range paths { + h, err := ReadSortedFileHeader(path) + if err != nil { + return nil, 0, 0, err + } + if i == 0 { + payloadSize = h.PayloadSize + } else if h.PayloadSize != payloadSize { + return nil, 0, 0, fmt.Errorf("sorted file %s has %d-byte payloads, %s has %d-byte payloads", path, h.PayloadSize, paths[0], payloadSize) + } + if h.Count > 0 { + start := h.recordsOffset() + runs = append(runs, sortedRun{path: path, start: start, end: start + int64(h.Count)*int64(MinKeySize+h.PayloadSize)}) + } + total += h.Count + } + return runs, payloadSize, total, nil +} + +// checkOutputNotInput fails if output is one of the inputs, which creating +// the output would truncate. +func checkOutputNotInput(output string, inputs []string) error { + out, err := os.Stat(output) + if err != nil { + return nil + } + for _, in := range inputs { + if info, err := os.Stat(in); err == nil && os.SameFile(out, info) { + return fmt.Errorf("the output %s is one of the inputs", output) + } + } + return nil +} + +// SortedFileReader reads a sorted file's records in order. +type SortedFileReader struct { + f *os.File + br *bufio.Reader + payloadSize int + left uint64 + last sortKey + record [MinKeySize + maxPayloadSize]byte +} + +// OpenSortedFile opens the sorted file at path to read its records in order. +// It checks the header as ReadSortedFileHeader does. +func OpenSortedFile(path string) (*SortedFileReader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + h, err := readSortedFileHeader(f) + if err == nil { + _, err = f.Seek(h.recordsOffset(), io.SeekStart) + } + if err != nil { + return nil, errors.Join(err, f.Close()) + } + return &SortedFileReader{f: f, br: bufio.NewReaderSize(f, 1<<20), payloadSize: h.PayloadSize, left: h.Count}, nil +} + +// Next returns the next record's key and payload, and io.EOF after the last +// record. A record out of order fails with ErrUnsortedInput. +func (r *SortedFileReader) Next() (key [MinKeySize]byte, payload uint64, err error) { + if r.left == 0 { + return key, 0, io.EOF + } + rec := r.record[:MinKeySize+r.payloadSize] + if _, err = io.ReadFull(r.br, rec); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + err = fmt.Errorf("sorted file %s: %w: it ends before its last record", r.f.Name(), ErrTruncatedFile) + } + return key, 0, err + } + r.left-- + k := sortKeyOf(rec) + if k.less(r.last) { + return key, 0, fmt.Errorf("sorted file %s: %w", r.f.Name(), ErrUnsortedInput) + } + r.last = k + copy(key[:], rec[:MinKeySize]) + return key, unpackPayloadFromBytes(rec[MinKeySize:], r.payloadSize), nil +} + +// Close closes the file. +func (r *SortedFileReader) Close() error { + return r.f.Close() +} diff --git a/sorted_file_test.go b/sorted_file_test.go new file mode 100644 index 0000000..2d4264b --- /dev/null +++ b/sorted_file_test.go @@ -0,0 +1,487 @@ +package streamhash + +import ( + "bytes" + "cmp" + "encoding/binary" + "errors" + "fmt" + "io" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "testing" +) + +type sortedRecord struct { + key [MinKeySize]byte + payload uint64 +} + +func compareSortedRecords(a, b sortedRecord) int { + if c := bytes.Compare(a.key[:], b.key[:]); c != 0 { + return c + } + return cmp.Compare(a.payload, b.payload) +} + +// randomSortedRecords returns n records in key order, about one in sixteen +// repeating the key before it, with random payloads of payloadSize bytes. +func randomSortedRecords(rng *rand.Rand, n, payloadSize int) []sortedRecord { + recs := make([]sortedRecord, n) + for i := range recs { + if i > 0 && rng.IntN(16) == 0 { + recs[i].key = recs[i-1].key + } else { + fillFromRNG(rng, recs[i].key[:]) + } + recs[i].payload = rng.Uint64() & payloadMaskFor(payloadSize) + } + slices.SortFunc(recs, compareSortedRecords) + return recs +} + +// uniqueSortedRecords is randomSortedRecords without the repeated keys. +func uniqueSortedRecords(rng *rand.Rand, n, payloadSize int) []sortedRecord { + return slices.CompactFunc(randomSortedRecords(rng, n, payloadSize), func(a, b sortedRecord) bool { return a.key == b.key }) +} + +// swapRecords swaps records i and i+1 in the bytes of a sorted file whose +// header is 4096 bytes. +func swapRecords(data []byte, recordSize, i int) { + pair := data[sortedFileAlign+i*recordSize:][:2*recordSize] + copy(pair, slices.Concat(pair[recordSize:], pair[:recordSize])) +} + +func writeSortedFile(t testing.TB, path string, cfg SortedFileConfig, recs []sortedRecord) { + t.Helper() + w, err := NewSortedFileWriter(path, cfg) + if err != nil { + t.Fatal(err) + } + defer w.Close() + for _, r := range recs { + if err := w.Add(r.key[:], r.payload); err != nil { + t.Fatal(err) + } + } + if err := w.Finish(); err != nil { + t.Fatal(err) + } +} + +func readSortedFile(t testing.TB, path string) []sortedRecord { + t.Helper() + r, err := OpenSortedFile(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + var recs []sortedRecord + for { + key, payload, err := r.Next() + if err == io.EOF { + return recs + } + if err != nil { + t.Fatal(err) + } + recs = append(recs, sortedRecord{key, payload}) + } +} + +func TestSortedFileRoundTrip(t *testing.T) { + dir := t.TempDir() + rng := newTestRNG(t) + for ps := 0; ps <= maxPayloadSize; ps++ { + t.Run(fmt.Sprintf("payload=%d", ps), func(t *testing.T) { + recs := randomSortedRecords(rng, 5000, ps) + // The smallest and largest keys, the second with the largest payload. + recs[0] = sortedRecord{} + recs = append(recs, sortedRecord{payload: payloadMaskFor(ps)}) + recs[len(recs)-1].key = [MinKeySize]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + meta := fmt.Appendf(nil, "metadata for %d-byte payloads", ps) + path := filepath.Join(dir, fmt.Sprintf("payload%d.sorted", ps)) + writeSortedFile(t, path, SortedFileConfig{PayloadSize: ps, Metadata: meta}, recs) + + h, err := ReadSortedFileHeader(path) + if err != nil { + t.Fatal(err) + } + if h.PayloadSize != ps || h.Count != uint64(len(recs)) || !bytes.Equal(h.Metadata, meta) { + t.Fatalf("header %+v, want payload size %d, count %d, metadata %q", h, ps, len(recs), meta) + } + if got := readSortedFile(t, path); !slices.Equal(got, recs) { + t.Fatal("records read back differ from the records written") + } + }) + } +} + +// TestSortedFileLayout pins the bytes on disk to streamhash-spec.md §3.9. +func TestSortedFileLayout(t *testing.T) { + path := filepath.Join(t.TempDir(), "layout.sorted") + recs := []sortedRecord{ + {key: [MinKeySize]byte{0x01, 15: 0xaa}, payload: 0x030201}, + {key: [MinKeySize]byte{0x02, 15: 0xbb}, payload: 0x060504}, + } + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 3, Metadata: []byte("meta")}, recs) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want := make([]byte, 4096) + copy(want, []byte{0x46, 0x53, 0x54, 0x53, 0x01, 0x00, 0x03, 0x00}) + binary.LittleEndian.PutUint64(want[8:], 2) + binary.LittleEndian.PutUint32(want[16:], 4) + copy(want[20:], "meta") + want = append(want, recs[0].key[:]...) + want = append(want, 0x01, 0x02, 0x03) + want = append(want, recs[1].key[:]...) + want = append(want, 0x04, 0x05, 0x06) + if !bytes.Equal(data, want) { + t.Fatalf("file bytes differ from the layout\ngot: %x\nwant: %x", data, want) + } +} + +// TestSortedFileMetadataSizes checks that the header pads to the next +// multiple of 4096 when the metadata does not fit in the first one, up to +// the maximum, with and without records. +func TestSortedFileMetadataSizes(t *testing.T) { + dir := t.TempDir() + three := randomSortedRecords(newTestRNG(t), 3, 2) + for _, tc := range []struct{ metadataLen, recordsOffset int }{ + {0, 4096}, + {4096 - sortedFileFixedSize, 4096}, + {4096 - sortedFileFixedSize + 1, 8192}, + {maxSortedFileMetadata, 69632}, + } { + for _, recs := range [][]sortedRecord{nil, three} { + path := filepath.Join(dir, fmt.Sprintf("meta%d-records%d.sorted", tc.metadataLen, len(recs))) + meta := bytes.Repeat([]byte{0x5a}, tc.metadataLen) + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 2, Metadata: meta}, recs) + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if want := int64(tc.recordsOffset + len(recs)*(MinKeySize+2)); info.Size() != want { + t.Fatalf("%d bytes of metadata, %d records: file is %d bytes, want %d", tc.metadataLen, len(recs), info.Size(), want) + } + h, err := ReadSortedFileHeader(path) + if err != nil { + t.Fatal(err) + } + if h.Count != uint64(len(recs)) || h.PayloadSize != 2 || !bytes.Equal(h.Metadata, meta) { + t.Fatalf("%d bytes of metadata, %d records: header %+v", tc.metadataLen, len(recs), h) + } + if got := readSortedFile(t, path); !slices.Equal(got, recs) { + t.Fatalf("%d bytes of metadata, %d records: records read back differ", tc.metadataLen, len(recs)) + } + } + } +} + +func TestNewSortedFileWriterRejectsConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.sorted") + for _, cfg := range []SortedFileConfig{ + {PayloadSize: -1}, + {PayloadSize: maxPayloadSize + 1}, + {Metadata: make([]byte, maxSortedFileMetadata+1)}, + } { + if _, err := NewSortedFileWriter(path, cfg); err == nil { + t.Fatalf("payload size %d and %d bytes of metadata: want an error", cfg.PayloadSize, len(cfg.Metadata)) + } else if cfg.PayloadSize != 0 && !errors.Is(err, ErrPayloadTooLarge) { + t.Fatalf("payload size %d: got %v, want ErrPayloadTooLarge", cfg.PayloadSize, err) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("a rejected config created the file: %v", err) + } + } +} + +func TestSortedFileAddRejects(t *testing.T) { + dir := t.TempDir() + key := func(first, last byte) []byte { + k := make([]byte, MinKeySize) + k[0], k[MinKeySize-1] = first, last + return k + } + path := filepath.Join(dir, "rejects.sorted") + w, err := NewSortedFileWriter(path, SortedFileConfig{PayloadSize: 3}) + if err != nil { + t.Fatal(err) + } + defer w.Close() + steps := []struct { + key []byte + payload uint64 + want error + }{ + {key(5, 5)[:MinKeySize-1], 0, ErrKeyTooShort}, + {key(5, 5), 1 << 24, ErrPayloadOverflow}, + {key(5, 5), 1, nil}, + {key(5, 5), 2, nil}, // a key equal to the one before it + {append(key(5, 5), 0x00), 3, nil}, + {key(4, 9), 0, ErrUnsortedInput}, + {key(5, 4), 0, ErrUnsortedInput}, + {key(6, 0), 1<<24 - 1, nil}, + } + for i, s := range steps { + if err := w.Add(s.key, s.payload); !errors.Is(err, s.want) { + t.Fatalf("step %d: Add returned %v, want %v", i, err, s.want) + } + } + if err := w.Finish(); err != nil { + t.Fatal(err) + } + var want []sortedRecord + for _, s := range steps { + if s.want == nil { + want = append(want, sortedRecord{key: [MinKeySize]byte(s.key), payload: s.payload}) + } + } + if got := readSortedFile(t, path); !slices.Equal(got, want) { + t.Fatalf("read back %v, want only the records Add accepted: %v", got, want) + } + + for _, ps := range []int{0, 1, 7, 8} { + w, err := NewSortedFileWriter(filepath.Join(dir, "payload.sorted"), SortedFileConfig{PayloadSize: ps}) + if err != nil { + t.Fatal(err) + } + if err := w.Add(key(1, 1), payloadMaskFor(ps)); err != nil { + t.Fatalf("payload size %d: Add of the largest payload returned %v", ps, err) + } + if ps < 8 { + if err := w.Add(key(1, 1), payloadMaskFor(ps)+1); !errors.Is(err, ErrPayloadOverflow) { + t.Fatalf("payload size %d: Add of a payload one too large returned %v", ps, err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + } +} + +// TestSortedFileWriterLifecycle checks that Close without Finish removes the +// file, Close after Finish keeps it, and a closed writer refuses more calls. +func TestSortedFileWriterLifecycle(t *testing.T) { + k := make([]byte, MinKeySize) + for _, tc := range []struct { + name string + end func(*SortedFileWriter) error + kept bool + }{ + {"Close", (*SortedFileWriter).Close, false}, + {"Finish", (*SortedFileWriter).Finish, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "w.sorted") + w, err := NewSortedFileWriter(path, SortedFileConfig{}) + if err != nil { + t.Fatal(err) + } + if err := w.Add(k, 0); err != nil { + t.Fatal(err) + } + if err := tc.end(w); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close after %s returned %v", tc.name, err) + } + h, err := ReadSortedFileHeader(path) + if tc.kept && (err != nil || h.Count != 1) { + t.Fatalf("header %+v, %v, want the finished file", h, err) + } + if !tc.kept && !os.IsNotExist(err) { + t.Fatalf("Close without Finish left the file: %v", err) + } + if err := w.Add(k, 0); !errors.Is(err, os.ErrClosed) { + t.Fatalf("Add after %s returned %v", tc.name, err) + } + if err := w.Finish(); !errors.Is(err, os.ErrClosed) { + t.Fatalf("Finish after %s returned %v", tc.name, err) + } + }) + } +} + +// TestSortedFileBytesPerSync checks that writeback changes nothing in the +// file, even when it starts on every flush. +func TestSortedFileBytesPerSync(t *testing.T) { + dir := t.TempDir() + recs := randomSortedRecords(newTestRNG(t), 200000, 3) + var files [][]byte + for _, bytesPerSync := range []int{0, 1, 2 << 20} { + path := filepath.Join(dir, fmt.Sprintf("sync%d.sorted", bytesPerSync)) + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 3, BytesPerSync: bytesPerSync}, recs) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + files = append(files, data) + } + for i := 1; i < len(files); i++ { + if !bytes.Equal(files[0], files[i]) { + t.Fatal("the file written with BytesPerSync differs from the one written without it") + } + } +} + +func TestReadSortedFileHeaderRejects(t *testing.T) { + dir := t.TempDir() + rng := newTestRNG(t) + valid := func(payloadSize int) []byte { + path := filepath.Join(dir, fmt.Sprintf("valid%d.sorted", payloadSize)) + writeSortedFile(t, path, SortedFileConfig{PayloadSize: payloadSize, Metadata: []byte("meta")}, randomSortedRecords(rng, 10, payloadSize)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data + } + data3, data0 := valid(3), valid(0) + indexPath, _ := buildTestIndex(t, 1000, 16) + index, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + mutated := func(data []byte, mutate func([]byte)) []byte { + b := slices.Clone(data) + mutate(b) + return b + } + + for _, tc := range []struct { + name string + data []byte + want error + }{ + {"shorter than the fixed header", data3[:sortedFileFixedSize-1], ErrTruncatedFile}, + {"cut in the padding", data3[:100], ErrTruncatedFile}, + {"metadata longer than the file", mutated(data3, func(b []byte) { binary.LittleEndian.PutUint32(b[16:], 5000) }), ErrTruncatedFile}, + {"cut mid-record", data3[:len(data3)-1], ErrTruncatedFile}, + {"cut a whole record", data3[:len(data3)-19], ErrTruncatedFile}, + {"count one too large", mutated(data3, func(b []byte) { binary.LittleEndian.PutUint64(b[8:], 11) }), ErrTruncatedFile}, + // 16 × (10 + 2^60) wraps to 16 × 10, so only a division catches it. + {"count that overflows", mutated(data0, func(b []byte) { binary.LittleEndian.PutUint64(b[8:], 10+1<<60) }), ErrTruncatedFile}, + {"an extra record", append(slices.Clone(data3), make([]byte, 19)...), ErrKeyCountMismatch}, + {"an unfinished writer's count of 0", mutated(data3, func(b []byte) { binary.LittleEndian.PutUint64(b[8:], 0) }), ErrKeyCountMismatch}, + {"wrong magic", mutated(data3, func(b []byte) { b[0] ^= 1 }), ErrInvalidMagic}, + {"an index file", index, ErrInvalidMagic}, + {"wrong version", mutated(data3, func(b []byte) { b[4] = 2 }), ErrInvalidVersion}, + {"payload size over 8", mutated(data3, func(b []byte) { b[6] = maxPayloadSize + 1 }), ErrCorruptedIndex}, + {"metadata length over the maximum", mutated(data3, func(b []byte) { binary.LittleEndian.PutUint32(b[16:], maxSortedFileMetadata+1) }), ErrCorruptedIndex}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(dir, "bad.sorted") + if err := os.WriteFile(path, tc.data, 0o600); err != nil { + t.Fatal(err) + } + if _, err := ReadSortedFileHeader(path); !errors.Is(err, tc.want) { + t.Fatalf("ReadSortedFileHeader returned %v, want %v", err, tc.want) + } + if r, err := OpenSortedFile(path); !errors.Is(err, tc.want) { + if r != nil { + r.Close() + } + t.Fatalf("OpenSortedFile returned %v, want %v", err, tc.want) + } + }) + } +} + +func TestSortedFileReaderRejectsUnsorted(t *testing.T) { + path := filepath.Join(t.TempDir(), "unsorted.sorted") + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 2}, uniqueSortedRecords(newTestRNG(t), 20, 2)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + swapRecords(data, MinKeySize+2, 5) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + r, err := OpenSortedFile(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + for i := range 6 { + if _, _, err := r.Next(); err != nil { + t.Fatalf("record %d: %v", i, err) + } + } + if _, _, err := r.Next(); !errors.Is(err, ErrUnsortedInput) { + t.Fatalf("the record out of order returned %v, want ErrUnsortedInput", err) + } +} + +// TestSortedFileReaderFileShrinks covers a file cut after its header was checked. +func TestSortedFileReaderFileShrinks(t *testing.T) { + path := filepath.Join(t.TempDir(), "shrinks.sorted") + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 3}, randomSortedRecords(newTestRNG(t), 10, 3)) + r, err := OpenSortedFile(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if err := os.Truncate(path, sortedFileAlign+9*(MinKeySize+3)+1); err != nil { + t.Fatal(err) + } + for i := range 9 { + if _, _, err := r.Next(); err != nil { + t.Fatalf("record %d: %v", i, err) + } + } + if _, _, err := r.Next(); !errors.Is(err, ErrTruncatedFile) { + t.Fatalf("the cut record returned %v, want ErrTruncatedFile", err) + } +} + +// TestSortedFileFeedsSortedBuilder checks that a sorted file's key order is +// one SortedBuilder accepts: bytewise order of the first 16 bytes is +// non-decreasing block order. +func TestSortedFileFeedsSortedBuilder(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys.sorted") + writeSortedFile(t, path, SortedFileConfig{PayloadSize: 3}, uniqueSortedRecords(newTestRNG(t), 20000, 3)) + var keys [][]byte + var payloads []uint64 + for _, r := range readSortedFile(t, path) { + keys = append(keys, r.key[:]) + payloads = append(payloads, r.payload) + } + for _, algo := range []Algorithm{AlgoBijection, AlgoPTRHash} { + idx := buildAndOpen(t, keys, payloads, WithAlgorithm(algo), WithPayload(3), WithFingerprint(1)) + verifyPayloads(t, idx, keys, payloads, 3) + idx.Close() + } +} + +// BenchmarkSortedFileWriter adds 1M records with 3-byte payloads. Finish, +// which syncs, is not timed. +func BenchmarkSortedFileWriter(b *testing.B) { + const n = 1_000_000 + recs := randomSortedRecords(newTestRNG(b), n, 3) + path := filepath.Join(b.TempDir(), "bench.sorted") + for b.Loop() { + w, err := NewSortedFileWriter(path, SortedFileConfig{PayloadSize: 3}) + if err != nil { + b.Fatal(err) + } + for _, r := range recs { + if err := w.Add(r.key[:], r.payload); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + if err := w.Finish(); err != nil { + b.Fatal(err) + } + b.StartTimer() + } + b.ReportMetric(float64(n)*float64(b.N)/b.Elapsed().Seconds(), "records/s") +} diff --git a/sorted_merge.go b/sorted_merge.go new file mode 100644 index 0000000..9d5ab9f --- /dev/null +++ b/sorted_merge.go @@ -0,0 +1,242 @@ +package streamhash + +import ( + "context" + "errors" + "fmt" + "os" +) + +const ( + // mergeBatchRecords is the records per batch handed to the goroutine that writes. + mergeBatchRecords = 4096 + // mergeReadBuffer is each input's read buffer. + mergeReadBuffer = 128 << 10 + // mergePoolDepth and mergeChanDepth size the batch pool and the hand-off channel. + mergePoolDepth = 3 + mergeChanDepth = 2 +) + +// MergeSortedFiles merges the sorted files at inputs, which must all have +// cfg.PayloadSize, into one at output, written with cfg. It is for a writer +// that spilled sorted pieces. It keeps every input open, with a 128 KiB read +// buffer each. An input whose keys are out of order fails the merge, and a +// failed or canceled merge removes the output. +func MergeSortedFiles(ctx context.Context, output string, inputs []string, cfg SortedFileConfig) error { + return mergeSortedFiles(ctx, output, inputs, cfg, mergeReadBuffer) +} + +func mergeSortedFiles(ctx context.Context, output string, inputs []string, cfg SortedFileConfig, bufBytes int) (err error) { + if err := ctx.Err(); err != nil { + return err + } + runs, payloadSize, total, err := readSortedRuns(inputs) + if err != nil { + return err + } + if len(inputs) > 0 && payloadSize != cfg.PayloadSize { + return fmt.Errorf("the sorted files have %d-byte payloads, the merge's are %d bytes", payloadSize, cfg.PayloadSize) + } + if err := checkOutputNotInput(output, inputs); err != nil { + return err + } + w, err := NewSortedFileWriter(output, cfg) + if err != nil { + return err + } + defer func() { + if cerr := w.Close(); cerr != nil { + err = errors.Join(err, cerr) + } + }() + if total > 0 { + if err := mergeRuns(ctx, runs, w, bufBytes); err != nil { + return err + } + } + if w.count != total { + return fmt.Errorf("%w: merged %d records, the inputs count %d", ErrKeyCountMismatch, w.count, total) + } + return w.Finish() +} + +// mergeRuns merges runs into w. One goroutine merges them through a heap and +// hands pooled batches of records to this one, which writes them. The first +// error cancels both. +func mergeRuns(ctx context.Context, runs []sortedRun, w *SortedFileWriter, bufBytes int) error { + ctx, fail := context.WithCancelCause(ctx) + defer fail(nil) + out, pool := make(chan *[]byte, mergeChanDepth), newBatchPool(w.recordSize) + go func() { + defer close(out) + if err := mergeFiles(ctx, runs, w.recordSize, bufBytes, out, pool); err != nil { + fail(err) + } + }() + // Drain out until it closes, so the merge goroutine has closed its files + // by the time this returns. + for batch := range out { + if ctx.Err() == nil { + if err := w.appendEncoded(*batch); err != nil { + fail(err) + } + } + pool <- batch // never blocks: the pool has room for every batch + } + return context.Cause(ctx) +} + +// newBatchPool returns a pool of empty batches. They travel as pointers: +// []byte channels put the heap loop's slice temporaries on the stack. +func newBatchPool(recordSize int) chan *[]byte { + pool := make(chan *[]byte, mergePoolDepth) + for range mergePoolDepth { + b := make([]byte, mergeBatchRecords*recordSize) + pool <- &b + } + return pool +} + +// mergeFiles merges runs through one heap into batches of records on out, +// taking empty batches from pool. It stops at the next batch once ctx is +// canceled. +func mergeFiles(ctx context.Context, runs []sortedRun, recordSize, bufBytes int, out chan<- *[]byte, pool <-chan *[]byte) error { + readers := make([]*runReader, 0, len(runs)) + defer func() { + for _, r := range readers { + r.f.Close() + } + }() + + h := make([]mergeEntry, 0, len(runs)) + for _, run := range runs { + r, err := newRunReader(run, recordSize, bufBytes) + if err != nil { + return err + } + readers = append(readers, r) + h = append(h, mergeEntry{key: r.key(), idx: len(readers) - 1}) + } + n := len(h) + for i := n/2 - 1; i >= 0; i-- { + siftDown(h, i, n) + } + + bp := <-pool + batch, pos := *bp, 0 + for n > 0 { + r := readers[h[0].idx] + pos += copy(batch[pos:], r.record()) + if pos == len(batch) { + out <- bp + if err := ctx.Err(); err != nil { + return err + } + bp = <-pool + batch, pos = *bp, 0 + } + + if r.advance() { + k := r.key() + if k.less(h[0].key) { + return fmt.Errorf("sorted file %s: %w", r.f.Name(), ErrUnsortedInput) + } + h[0].key = k + siftDown(h, 0, n) + } else { + if r.err != nil { + return r.err + } + n-- + if n > 0 { + h[0] = h[n] + siftDown(h, 0, n) + } + } + } + if pos > 0 { + *bp = batch[:pos] + out <- bp + } + return nil +} + +// mergeEntry is one input's heap slot: its current key and its index in the readers. +type mergeEntry struct { + key sortKey + idx int +} + +// siftDown restores the min-heap rooted at i over h[:n]. +func siftDown(h []mergeEntry, i, n int) { + for { + left := 2*i + 1 + if left >= n { + break + } + j := left + if right := left + 1; right < n && h[right].key.less(h[j].key) { + j = right + } + if !h[j].key.less(h[i].key) { + break + } + h[i], h[j] = h[j], h[i] + i = j + } +} + +// runReader reads one input's records directly into an aligned buffer and +// slices them out of it, with no per-record parse or allocation. +type runReader struct { + run sortedRun + f *os.File + buf []byte // aligned for direct reads + recordSize int + cursor int // offset in buf of the current record + valid int // bytes of buf that hold records + fileOff int64 // file offset of buf[0], a multiple of directAlign + err error +} + +// newRunReader opens run, which must have records, and reads its first. +func newRunReader(run sortedRun, recordSize, bufBytes int) (*runReader, error) { + // A refill rereads up to directAlign-1 bytes, so the buffer holds at + // least two aligned blocks. + bufBytes = max(bufBytes, 2*directAlign) + bufBytes += (directAlign - bufBytes%directAlign) % directAlign + f, err := openDirect(run.path) + if err != nil { + return nil, err + } + r := &runReader{run: run, f: f, buf: alignedBuffer(bufBytes), recordSize: recordSize} + if !r.fill(run.start) { + f.Close() + return nil, r.err + } + return r, nil +} + +func (r *runReader) record() []byte { return r.buf[r.cursor : r.cursor+r.recordSize] } +func (r *runReader) key() sortKey { return sortKeyOf(r.buf[r.cursor:]) } + +// advance moves to the next record and reports whether there is one. At an +// error it sets r.err. +func (r *runReader) advance() bool { + r.cursor += r.recordSize + return r.cursor+r.recordSize <= r.valid || r.fill(r.fileOff+int64(r.cursor)) +} + +// fill reads the buffer from the aligned offset at or below off, the file +// offset of the next record, and reports whether that record is there. +func (r *runReader) fill(off int64) bool { + if off == r.run.end { + return false + } + alignedOff := off &^ (directAlign - 1) + if r.valid, r.err = r.run.readAt(r.f, r.buf, alignedOff); r.err != nil { + return false + } + r.fileOff, r.cursor = alignedOff, int(off-alignedOff) + return true +} diff --git a/sorted_merge_test.go b/sorted_merge_test.go new file mode 100644 index 0000000..74e908b --- /dev/null +++ b/sorted_merge_test.go @@ -0,0 +1,285 @@ +package streamhash + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "testing" + "time" +) + +// mergeWithWatchdog merges with bufBytes read buffers, failing the test if +// the merge does not return. +func mergeWithWatchdog(t *testing.T, ctx context.Context, output string, inputs []string, cfg SortedFileConfig, bufBytes int) error { + t.Helper() + return runBuildWithWatchdog(t, 2*time.Minute, func() error { + return mergeSortedFiles(ctx, output, inputs, cfg, bufBytes) + }) +} + +// writeMergeInputs writes a sorted file of random records for each size, and +// returns their paths and all their records. The inputs' metadata lengths +// vary, so their records start at different offsets. +func writeMergeInputs(t *testing.T, dir string, payloadSize int, sizes []int) ([]string, []sortedRecord) { + t.Helper() + rng := newTestRNG(t) + var paths []string + var all []sortedRecord + for i, n := range sizes { + recs := randomSortedRecords(rng, n, payloadSize) + path := filepath.Join(dir, fmt.Sprintf("in%04d.sorted", i)) + meta := bytes.Repeat([]byte{byte(i)}, i*1500%10000) + writeSortedFile(t, path, SortedFileConfig{PayloadSize: payloadSize, Metadata: meta}, recs) + paths = append(paths, path) + all = append(all, recs...) + } + return paths, all +} + +func TestMergeSortedFiles(t *testing.T) { + rng := newTestRNG(t) + randomSizes := func(inputs, maxSize int) []int { + sizes := make([]int, inputs) + for i := range sizes { + sizes[i] = rng.IntN(maxSize + 1) + } + return sizes + } + type mergeCase struct { + name string + payloadSize int + sizes []int + bufBytes int + } + cases := []mergeCase{ + {"one input", 3, []int{20000}, mergeReadBuffer}, + {"inputs smaller than a buffer", 3, []int{1, 2, 100, 3000}, mergeReadBuffer}, + {"empty inputs among others", 4, []int{0, 500, 0, 0, 2000, 0}, mergeReadBuffer}, + {"many inputs", 3, randomSizes(300, 400), mergeReadBuffer}, + {"refills across record boundaries", 3, []int{30000, 25000, 9000}, 2 * directAlign}, + } + for ps := 0; ps <= maxPayloadSize; ps++ { + cases = append(cases, mergeCase{fmt.Sprintf("random sizes, payload=%d", ps), ps, randomSizes(1+rng.IntN(12), 15000), []int{2 * directAlign, mergeReadBuffer}[rng.IntN(2)]}) + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + inputs, want := writeMergeInputs(t, dir, tc.payloadSize, tc.sizes) + slices.SortFunc(want, compareSortedRecords) + output := filepath.Join(dir, "merged.sorted") + cfg := SortedFileConfig{PayloadSize: tc.payloadSize, Metadata: []byte("merged"), BytesPerSync: 64 << 10} + if err := mergeWithWatchdog(t, context.Background(), output, inputs, cfg, tc.bufBytes); err != nil { + t.Fatal(err) + } + h, err := ReadSortedFileHeader(output) + if err != nil { + t.Fatal(err) + } + if h.Count != uint64(len(want)) || h.PayloadSize != tc.payloadSize || string(h.Metadata) != "merged" { + t.Fatalf("header %+v, want %d records with %d-byte payloads and the merge's metadata", h, len(want), tc.payloadSize) + } + // readSortedFile fails on a record out of order. Equal keys may come + // from their inputs in any order, so compare as sorted by payload too. + got := readSortedFile(t, output) + slices.SortFunc(got, compareSortedRecords) + if !slices.Equal(got, want) { + t.Fatal("the merged records are not the inputs' records") + } + }) + } +} + +func TestMergeSortedFilesNoRecords(t *testing.T) { + dir := t.TempDir() + empty, _ := writeMergeInputs(t, dir, 2, []int{0, 0}) + for _, inputs := range [][]string{nil, empty} { + output := filepath.Join(dir, "merged.sorted") + if err := mergeWithWatchdog(t, context.Background(), output, inputs, SortedFileConfig{PayloadSize: 2, Metadata: []byte("m")}, mergeReadBuffer); err != nil { + t.Fatal(err) + } + if h, err := ReadSortedFileHeader(output); err != nil || h.Count != 0 || string(h.Metadata) != "m" { + t.Fatalf("merging %d empty inputs: header %+v, %v", len(inputs), h, err) + } + } +} + +func TestMergeSortedFilesRejectsPayloadSizes(t *testing.T) { + dir := t.TempDir() + three, _ := writeMergeInputs(t, dir, 3, []int{10}) + four := filepath.Join(dir, "four.sorted") + writeSortedFile(t, four, SortedFileConfig{PayloadSize: 4}, randomSortedRecords(newTestRNG(t), 10, 4)) + output := filepath.Join(dir, "merged.sorted") + for _, tc := range []struct { + inputs []string + payloadSize int + }{ + {[]string{three[0], four}, 3}, + {[]string{four, three[0]}, 4}, + {three, 4}, + } { + if err := mergeWithWatchdog(t, context.Background(), output, tc.inputs, SortedFileConfig{PayloadSize: tc.payloadSize}, mergeReadBuffer); err == nil { + t.Fatalf("merging %v into %d-byte payloads: want an error", tc.inputs, tc.payloadSize) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("a rejected merge created the output: %v", err) + } + } +} + +// TestMergeSortedFilesRejectsOutputAmongInputs checks that a merge into one +// of its inputs fails before creating the output, which would truncate the +// input. +func TestMergeSortedFilesRejectsOutputAmongInputs(t *testing.T) { + dir := t.TempDir() + inputs, _ := writeMergeInputs(t, dir, 3, []int{1000, 2000}) + before := readFileBytes(t, inputs[1]) + // inputs[1], spelled differently. + output := dir + string(filepath.Separator) + "." + string(filepath.Separator) + filepath.Base(inputs[1]) + if err := mergeWithWatchdog(t, context.Background(), output, inputs, SortedFileConfig{PayloadSize: 3}, mergeReadBuffer); err == nil { + t.Fatal("a merge into one of its inputs returned no error") + } + if !bytes.Equal(readFileBytes(t, inputs[1]), before) { + t.Fatal("the merge changed its input") + } +} + +// TestMergeSortedFilesRejectsUnsortedInput checks that an input whose records +// are out of order fails the merge, wherever the disorder is. +func TestMergeSortedFilesRejectsUnsortedInput(t *testing.T) { + dir := t.TempDir() + inputs, _ := writeMergeInputs(t, dir, 3, []int{5000, 6000}) + recs := uniqueSortedRecords(newTestRNG(t), 5000, 3) + clean := filepath.Join(dir, "clean.sorted") + writeSortedFile(t, clean, SortedFileConfig{PayloadSize: 3}, recs) + data, err := os.ReadFile(clean) + if err != nil { + t.Fatal(err) + } + // The second swap straddles a refill of a two-block buffer. + for _, i := range []int{0, 431, len(recs) - 2} { + bad := slices.Clone(data) + swapRecords(bad, MinKeySize+3, i) + corrupt := filepath.Join(dir, "corrupt.sorted") + if err := os.WriteFile(corrupt, bad, 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "merged.sorted") + err := mergeWithWatchdog(t, context.Background(), output, append(slices.Clone(inputs), corrupt), SortedFileConfig{PayloadSize: 3}, 2*directAlign) + if !errors.Is(err, ErrUnsortedInput) { + t.Fatalf("records %d and %d swapped: the merge returned %v, want ErrUnsortedInput", i, i+1, err) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("a failed merge left its output: %v", err) + } + } +} + +// TestMergeSortedFilesInputShorterThanItsCount covers an input that shrinks +// after its header was checked. +func TestMergeSortedFilesInputShorterThanItsCount(t *testing.T) { + dir := t.TempDir() + inputs, _ := writeMergeInputs(t, dir, 3, []int{1000}) + w, err := NewSortedFileWriter(filepath.Join(dir, "merged.sorted"), SortedFileConfig{PayloadSize: 3}) + if err != nil { + t.Fatal(err) + } + defer w.Close() + for _, count := range []uint64{1001, 5000} { + runs, _, _, err := readSortedRuns(inputs) + if err != nil { + t.Fatal(err) + } + runs[0].end = runs[0].start + int64(count)*(MinKeySize+3) + err = runBuildWithWatchdog(t, time.Minute, func() error { + return mergeRuns(context.Background(), runs, w, 2*directAlign) + }) + if !errors.Is(err, ErrTruncatedFile) { + t.Fatalf("an input counted %d records but holding 1000: the merge returned %v", count, err) + } + } +} + +func TestMergeSortedFilesCanceled(t *testing.T) { + dir := t.TempDir() + inputs, _ := writeMergeInputs(t, dir, 3, []int{20000, 20000, 20000, 20000}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + output := filepath.Join(dir, "merged.sorted") + if err := mergeWithWatchdog(t, ctx, output, inputs, SortedFileConfig{PayloadSize: 3}, mergeReadBuffer); !errors.Is(err, context.Canceled) { + t.Fatalf("a merge with a canceled context returned %v", err) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("a canceled merge left its output: %v", err) + } + + // Cancel once the merge goroutine has handed over its first batch. + runs, _, _, err := readSortedRuns(inputs) + if err != nil { + t.Fatal(err) + } + const recordSize = MinKeySize + 3 + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + out, pool := make(chan *[]byte, mergeChanDepth), newBatchPool(recordSize) + merged := make(chan error, 1) + go func() { + defer close(out) + merged <- mergeFiles(ctx, runs, recordSize, mergeReadBuffer, out, pool) + }() + err = runBuildWithWatchdog(t, time.Minute, func() error { + pool <- <-out + cancel() + batches := 1 + for b := range out { + batches++ + pool <- b + } + if total := (4*20000 + mergeBatchRecords - 1) / mergeBatchRecords; batches >= total { + return fmt.Errorf("the merge sent all %d batches after it was canceled", batches) + } + return <-merged + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("a merge canceled midway returned %v", err) + } +} + +// BenchmarkMergeSortedFiles merges 90 inputs of 1M records with 3-byte payloads. +func BenchmarkMergeSortedFiles(b *testing.B) { + const inputs, perInput = 90, 1_000_000 + dir := b.TempDir() + rng := newTestRNG(b) + paths := make([]string, inputs) + for i := range paths { + paths[i] = filepath.Join(dir, fmt.Sprintf("in%02d.sorted", i)) + writeSortedFile(b, paths[i], SortedFileConfig{PayloadSize: 3}, stratifiedSortedRecords(rng, perInput, 3)) + } + output := filepath.Join(dir, "merged.sorted") + for b.Loop() { + if err := MergeSortedFiles(context.Background(), output, paths, SortedFileConfig{PayloadSize: 3}); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(inputs*perInput)*float64(b.N)/b.Elapsed().Seconds(), "records/s") +} + +// stratifiedSortedRecords returns n records in key order without sorting +// them: record i's first eight bytes are random within the i-th of n equal +// ranges, so inputs interleave as uniform random keys would. +func stratifiedSortedRecords(rng *rand.Rand, n, payloadSize int) []sortedRecord { + recs := make([]sortedRecord, n) + step := ^uint64(0) / uint64(n) + for i := range recs { + binary.BigEndian.PutUint64(recs[i].key[0:8], uint64(i)*step+rng.Uint64N(step)) + binary.BigEndian.PutUint64(recs[i].key[8:16], rng.Uint64()) + recs[i].payload = rng.Uint64() & payloadMaskFor(payloadSize) + } + return recs +} diff --git a/streamhash-spec.md b/streamhash-spec.md index 2a23765..1d89110 100644 --- a/streamhash-spec.md +++ b/streamhash-spec.md @@ -415,6 +415,42 @@ Offset Bytes Meaning The metadata region starts at `0x66` (RAM index ends at `0x48 + 3×10 = 0x66`; the payload region is empty). The per-block byte sizes shown are illustrative encoded lengths; they come straight from the RAM-index offsets by subtraction: block 0 = `0xA5 − 0 = 165 B`, block 1 = `0x142 − 0xA5 = 0x9D = 157 B`. Block 1 here happens to equal the empty-block Bijection size from §3.6 (157 B) purely by coincidence — there is nothing to reconcile between the 165 B and 157 B figures. The footer therefore sits at `0x66 + 0x142 = 0x1A8`. +### 3.9. Sorted files + +A sorted file is not an index. It holds keys with their payloads in key order, in fixed-width records, as input to an index build. `SortedFileWriter` writes one, and `MergeSortedFiles` merges several that share a payload size into one. + +``` ++--------------------------------------------------+ +| Header (padded to a multiple of 4096 B) | ++--------------------------------------------------+ +| Records (Count × (16 + PayloadSize) B) | ++--------------------------------------------------+ +``` + +**Header:** + +| Offset | Size | Field | Type | Description | +|--------|------|-------|------|-------------| +| 0 | 4 | Magic | uint32_le | `0x53545346` ("STSF") | +| 4 | 2 | Version | uint16_le | `0x0001` | +| 6 | 1 | PayloadSize | uint8 | Payload bytes per record (0–8) | +| 7 | 1 | Reserved | byte | Zero | +| 8 | 8 | Count | uint64_le | Number of records | +| 16 | 4 | MetadataLen | uint32_le | Metadata length, at most 65,536 | +| 20 | MetadataLen | Metadata | bytes | Application-defined, not interpreted | + +Zero bytes pad the header to `RecordsOffset = ceil((20 + MetadataLen) / 4096) × 4096`, so the records start at a 4096-aligned offset, as direct reads (`O_DIRECT`) need. + +**Records.** Record `i` is at `RecordsOffset + i × (16 + PayloadSize)`: + +``` +[Key: 16 B][Payload: PayloadSize B, little-endian] +``` + +`Key` is the first 16 bytes of the key the index uses (§2.1). Records are in non-decreasing bytewise order of `Key`, which is the order of big-endian bytes 0–7, then big-endian bytes 8–15. `fastRange32` is monotonic in the prefix (§2.2), so this is also non-decreasing block order, the order sorted mode needs (§6.1). A key may repeat; an index build rejects it as a duplicate (§7.1). + +**Validation.** A file's size must be exactly `RecordsOffset + Count × (16 + PayloadSize)`. A reader checks this by dividing the size, never by multiplying `Count`, so a corrupt `Count` cannot overflow the arithmetic and pass. The writer writes `Count` after the records, then syncs. §7.5 lists the errors. + --- ## 4. Algorithm: Bijection @@ -868,6 +904,17 @@ An index with `N = 0` keys is not an error: it builds successfully, and every `Q | `ErrChecksumFailed` | footer hash mismatch (from `Verify`) | | `ErrNotFound` | empty block, or fingerprint mismatch | +**Sorted files (§3.9):** + +| Error | Condition | +|-------|-----------| +| `ErrTruncatedFile` | file shorter than its header, or than its `Count` records | +| `ErrKeyCountMismatch` | file longer than its header and `Count` records | +| `ErrInvalidMagic` | magic ≠ `0x53545346` | +| `ErrInvalidVersion` | version ≠ `0x0001` | +| `ErrCorruptedIndex` | PayloadSize > 8, or MetadataLen > 65,536 | +| `ErrUnsortedInput` | a key sorts before the key before it, when it is written, merged or read | + ### 7.6. Integrity and corruption safety A reader **should** bounds-check before trusting an offset: From e8db257cc6df351a59149f86a9282d304a7c4446 Mon Sep 17 00:00:00 2001 From: tamirms Date: Mon, 26 Oct 2026 08:15:50 +0000 Subject: [PATCH 2/2] Build the sorted-file tests without a later change's helper TestMergeSortedFilesRejectsOutputAmongInputs read its input with readFileBytes, a helper the BuildFromSortedFiles change adds, so this change's tests did not build on their own. It now reads the input with os.ReadFile. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DjQed1VX9uGU9PWpAjBiz7 --- sorted_merge_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sorted_merge_test.go b/sorted_merge_test.go index 74e908b..395ebbc 100644 --- a/sorted_merge_test.go +++ b/sorted_merge_test.go @@ -139,13 +139,20 @@ func TestMergeSortedFilesRejectsPayloadSizes(t *testing.T) { func TestMergeSortedFilesRejectsOutputAmongInputs(t *testing.T) { dir := t.TempDir() inputs, _ := writeMergeInputs(t, dir, 3, []int{1000, 2000}) - before := readFileBytes(t, inputs[1]) + before, err := os.ReadFile(inputs[1]) + if err != nil { + t.Fatal(err) + } // inputs[1], spelled differently. output := dir + string(filepath.Separator) + "." + string(filepath.Separator) + filepath.Base(inputs[1]) if err := mergeWithWatchdog(t, context.Background(), output, inputs, SortedFileConfig{PayloadSize: 3}, mergeReadBuffer); err == nil { t.Fatal("a merge into one of its inputs returned no error") } - if !bytes.Equal(readFileBytes(t, inputs[1]), before) { + after, err := os.ReadFile(inputs[1]) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { t.Fatal("the merge changed its input") } }