diff --git a/README.md b/README.md index bfc39e5..dbc9a87 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ if err := builder.Finish(); err != nil { } ``` +Keys that are already in sorted files (written with `SortedFileWriter`, see [`streamhash-spec.md`](streamhash-spec.md) §3.9) build without being merged: `BuildFromSortedFiles(ctx, "index.idx", paths, opts...)` reads the files once, in stripes of blocks that the workers solve in parallel, and produces the same index. It uses about 256 MiB whatever the number of keys or files. + ### Building an index (unsorted input) ```go diff --git a/builder.go b/builder.go index e41207c..2023327 100644 --- a/builder.go +++ b/builder.go @@ -85,17 +85,13 @@ type builder struct { // Use WithWorkers(N) to enable parallel building with N workers. // With workers > 1, blocks are built in parallel while maintaining // the streaming API and O(W × block_size) memory. -func newBuilder(ctx context.Context, output string, totalKeys uint64, opts ...BuildOption) (*builder, error) { +func newBuilder(ctx context.Context, output string, totalKeys uint64, cfg *buildConfig) (*builder, error) { // totalKeys == 0 is allowed: numBlocks() floors at 2, so a zero-key build // emits empty blocks and finalizes to a valid empty index. if totalKeys > maxKeys { return nil, sherr.ErrTooManyKeys } - cfg := defaultBuildConfig() - for _, opt := range opts { - opt(cfg) - } cfg.totalKeys = totalKeys if cfg.payloadSize < 0 || cfg.payloadSize > maxPayloadSize { @@ -389,12 +385,12 @@ type SortedBuilder struct { // totalKeys may be 0: the result is a valid empty index whose every QueryRank // returns ErrNotFound. func NewSortedBuilder(ctx context.Context, output string, totalKeys uint64, opts ...BuildOption) (*SortedBuilder, error) { - b, err := newBuilder(ctx, output, totalKeys, opts...) + b, err := newBuilder(ctx, output, totalKeys, newBuildConfig(opts)) if err != nil { return nil, err } if b.workers > 1 { - b.initParallelWorkers() + b.startPipeline(b.runWorker) b.pendingEntries = b.getEntrySlice() } return &SortedBuilder{b: b}, nil diff --git a/builder_options.go b/builder_options.go index 56b3316..f78a546 100644 --- a/builder_options.go +++ b/builder_options.go @@ -11,7 +11,8 @@ type BuildOption func(*buildConfig) type buildConfig struct { workers int payloadSize int - fingerprintSize int // in bytes + payloadSet bool // WithPayload was given + fingerprintSize int // in bytes globalSeed uint64 userMetadata []byte unsortedTempDir string // temp directory for unsorted partition files @@ -28,6 +29,15 @@ func defaultBuildConfig() *buildConfig { } } +// newBuildConfig applies opts to the default configuration. +func newBuildConfig(opts []BuildOption) *buildConfig { + cfg := defaultBuildConfig() + for _, opt := range opts { + opt(cfg) + } + return cfg +} + // WithWorkers sets the number of parallel workers for block building during Finish. func WithWorkers(n int) BuildOption { return func(c *buildConfig) { @@ -40,6 +50,7 @@ func WithWorkers(n int) BuildOption { func WithPayload(sizeBytes int) BuildOption { return func(c *buildConfig) { c.payloadSize = sizeBytes + c.payloadSet = true } } diff --git a/builder_parallel.go b/builder_parallel.go index c163b2d..b67a89a 100644 --- a/builder_parallel.go +++ b/builder_parallel.go @@ -47,9 +47,11 @@ type blockResult struct { err error } -// initParallelWorkers initializes channels, pools, and starts worker/writer goroutines -// for parallel building. Used by both sorted parallel mode and unsorted parallel mode. -func (b *builder) initParallelWorkers() { +// startPipeline creates the parallel pipeline's channels and pools, and +// starts its writer goroutine and b.workers goroutines running worker, which +// send blockResults on resultChan and return once workerCtx is done. Used by +// sorted parallel mode, unsorted parallel mode and BuildFromSortedFiles. +func (b *builder) startPipeline(worker func() error) { b.workChan = make(chan blockWork, b.workers*workChanBufferMultiplier) b.resultChan = make(chan blockResult, b.workers*workChanBufferMultiplier) b.writerDone = make(chan error, 1) @@ -67,13 +69,12 @@ func (b *builder) initParallelWorkers() { return make([]byte, metadataBufSize) } - // Start worker goroutines. // Wrap in explicit cancel so shutdownWorkers can unblock workers stuck on resultChan. ctx, cancel := context.WithCancel(b.ctx) b.workerCancel = cancel b.workerGroup, b.workerCtx = errgroup.WithContext(ctx) for range b.workers { - b.workerGroup.Go(b.runWorker) + b.workerGroup.Go(worker) } // Start writer goroutine @@ -164,20 +165,76 @@ func (b *builder) dispatchEmptyBlock(blockID uint32) error { return b.dispatchBlockWork(blockID, nil, false) } -// runWorker is the worker goroutine that builds blocks in parallel. -func (b *builder) runWorker() error { - // Create block builder for this worker - blkBuilder, err := newBlockBuilder(b.cfg.algorithm, b.cfg.totalKeys, b.cfg.globalSeed, b.cfg.payloadSize, b.cfg.fingerprintSize) +// blockSolver is a worker's block builder, with the payload buffer it reuses +// across blocks. +type blockSolver struct { + b *builder + blk blockBuilder + payloadsBuf []byte +} + +func (b *builder) newBlockSolver() (*blockSolver, error) { + blk, err := newBlockBuilder(b.cfg.algorithm, b.cfg.totalKeys, b.cfg.globalSeed, b.cfg.payloadSize, b.cfg.fingerprintSize) if err != nil { - return err + return nil, err } + return &blockSolver{b: b, blk: blk}, nil +} +// loadBlock resets blk and adds entries to it, computing their fingerprints +// from k0 and k1. +func (b *builder) loadBlock(blk blockBuilder, entries []routedEntry) { fpSize := b.cfg.fingerprintSize - entrySize := b.cfg.payloadSize + fpSize + blk.Reset() + for _, e := range entries { + blk.AddKey(e.k0, e.k1, e.payload, extractFingerprint(e.k0, e.k1, fpSize)) + } +} - // Pre-allocate reusable payload buffer (grows as needed, reused across blocks) - // Metadata buffer is NOT reused because it's sent through channel - var payloadsBuf []byte +// build solves the loaded block as blockID, writes its payloads after the +// keysBefore keys of the blocks before it, and returns the result for the +// writer. The metadata buffer comes from the pool; the writer returns it. +func (s *blockSolver) build(blockID uint32, keysBefore uint64) blockResult { + b := s.b + entrySize := b.cfg.payloadSize + b.cfg.fingerprintSize + payloadsNeeded := s.blk.KeysAdded() * entrySize + if cap(s.payloadsBuf) < payloadsNeeded { + s.payloadsBuf = make([]byte, payloadsNeeded) + } + payloads := s.payloadsBuf[:payloadsNeeded] + metadataBuf := b.metadataPool.Get().([]byte) + metadataLen, _, numKeys, err := s.blk.BuildSeparatedInto(metadataBuf, payloads) + result := blockResult{blockID: blockID, metadata: metadataBuf[:metadataLen], numKeys: numKeys, err: err} + if err == nil { + // Hash and write the payloads while they are hot in CPU cache. + result.payloadHash = xxhash.Sum64(payloads) + result.err = b.iw.writePayloadsDirect(payloads, keysBefore*uint64(entrySize)) + } + return result +} + +// sendResult hands a block's result to the writer and returns the block's +// error, if any, for the worker to return. A block build error must +// propagate to the errgroup, not just ride along in blockResult.err: +// returning it cancels workerCtx, which releases any peer workers blocked +// sending on resultChan after the writer exits on the first error -- +// otherwise they (and drainParallelPipeline's workerGroup.Wait) deadlock. +// Wait then surfaces this error to the build. +func (b *builder) sendResult(result blockResult) error { + select { + case b.resultChan <- result: + return result.err + case <-b.workerCtx.Done(): + return b.workerCtx.Err() + } +} + +// runWorker is the worker goroutine that builds blocks in parallel. +func (b *builder) runWorker() error { + solver, err := b.newBlockSolver() + if err != nil { + return err + } for work := range b.workChan { select { @@ -190,98 +247,26 @@ func (b *builder) runWorker() error { default: } - var metadataBuf []byte - var numKeys int - var payloadHash uint64 - var buildErr error - - if len(work.entries) == 0 { - // Empty block: reset builder and build empty metadata - blkBuilder.Reset() - metadataBuf = b.metadataPool.Get().([]byte) - var metadataLen int - metadataLen, _, _, buildErr = blkBuilder.BuildSeparatedInto(metadataBuf, nil) - metadataBuf = metadataBuf[:metadataLen] - numKeys = 0 - // Empty block contributes hash of empty slice (deterministic) - payloadHash = xxhash.Sum64(nil) - } else { - // Reset and populate the builder, computing fingerprints from k0/k1 - blkBuilder.Reset() - for _, e := range work.entries { - blkBuilder.AddKey(e.k0, e.k1, e.payload, extractFingerprint(e.k0, e.k1, fpSize)) - } + b.loadBlock(solver.blk, work.entries) - // Signal that entries are no longer being read. This allows the - // flatBuf backing the entries to be safely reused for the next - // partition read (fencing for unsorted parallel mode). - if work.fenceWg != nil { - work.fenceWg.Done() - } - - numKeysInBlock := len(work.entries) - payloadsNeeded := numKeysInBlock * entrySize - - // Reuse payload buffer (grows if needed) - if cap(payloadsBuf) < payloadsNeeded { - payloadsBuf = make([]byte, payloadsNeeded) - } else { - payloadsBuf = payloadsBuf[:payloadsNeeded] - } - - // Get metadata buffer from pool (returned to pool by writer after use) - metadataBuf = b.metadataPool.Get().([]byte) - - var metadataLen int - metadataLen, _, numKeys, buildErr = blkBuilder.BuildSeparatedInto(metadataBuf, payloadsBuf) - if buildErr == nil { - metadataBuf = metadataBuf[:metadataLen] - - // Compute payload hash and write payloads while data is hot in CPU cache - if payloadsNeeded > 0 { - // Hash payload buffer before writing via pwrite - payloadHash = xxhash.Sum64(payloadsBuf[:payloadsNeeded]) - // Write payloads via pwrite, then buffer can be reused - payloadOffset := work.keysBefore * uint64(entrySize) - if werr := b.iw.writePayloadsDirect(payloadsBuf, payloadOffset); werr != nil { - buildErr = werr - } - } else { - // MPHF-only mode: no payloads, hash empty slice - payloadHash = xxhash.Sum64(nil) - } - } - - // Return entry slice to pool only if it was pool-allocated. - // Unsorted mode dispatches flatBuf-backed slices with pooled=false - // and uses fenceWg for lifetime management instead. - if work.pooled { - b.putEntrySlice(work.entries) - } + // Signal that entries are no longer being read. This allows the + // flatBuf backing the entries to be safely reused for the next + // partition read (fencing for unsorted parallel mode). + if work.fenceWg != nil { + work.fenceWg.Done() } - // Send result to writer (metadata + payload hash) - select { - case b.resultChan <- blockResult{ - blockID: work.blockID, - metadata: metadataBuf, - numKeys: numKeys, - payloadHash: payloadHash, - err: buildErr, - }: - case <-b.workerCtx.Done(): - return b.workerCtx.Err() + result := solver.build(work.blockID, work.keysBefore) + + // Return entry slice to pool only if it was pool-allocated. + // Unsorted mode dispatches flatBuf-backed slices with pooled=false + // and uses fenceWg for lifetime management instead. + if work.pooled { + b.putEntrySlice(work.entries) } - // A block build error must propagate to the errgroup, not just ride - // along in blockResult.err. Returning it cancels workerCtx, which - // releases any peer workers blocked sending on resultChan after the - // writer exits on the first error -- otherwise they (and - // drainParallelPipeline's workerGroup.Wait) deadlock. Wait then surfaces - // this error to Finish(). Covers both the sorted and unsorted-parallel - // paths, which share this worker pool and drainParallelPipeline. - if buildErr != nil { - return buildErr + if err := b.sendResult(result); err != nil { + return err } } @@ -396,37 +381,36 @@ func (b *builder) finishParallel() error { return b.drainParallelPipeline() } -// drainParallelPipeline closes the work channel, waits for workers and writer -// to finish, then finalizes the index. +// drainParallelPipeline closes the work channel, waits for the workers and +// the writer to finish, then finalizes the index, unless one of them failed +// or the pipeline was canceled. func (b *builder) drainParallelPipeline() error { // Close work channel to signal workers we're done close(b.workChan) b.workersShutDown.Store(true) // Prevents double-close in Close()/shutdownWorkers() - // Wait for all workers to finish - if werr := b.workerGroup.Wait(); werr != nil { - close(b.resultChan) - // Prefer the writer's error: when the writer initiates the failure it - // cancels workerCtx, so workerGroup.Wait returns a bare context.Canceled - // while the real cause (e.g. an ENOSPC metadata write, or the block - // buildErr) sits in writerDone. Fall back to the worker error only if the - // writer reported none (e.g. a worker setup failure). - cause := <-b.writerDone - if cause == nil { - cause = fmt.Errorf("worker error: %w", werr) - } - return errors.Join(cause, b.cleanup()) - } - - // Close result channel to signal writer we're done + // Read the cancellation before Wait, which cancels workerCtx itself. + canceled := b.workerCtx.Err() + werr := b.workerGroup.Wait() close(b.resultChan) - - // Wait for writer to finish - if err := <-b.writerDone; err != nil { - primaryErr := fmt.Errorf("writer error: %w", err) - return errors.Join(primaryErr, b.cleanup()) + cause := <-b.writerDone + // A block build error reaches writerDone through failWriter and is the + // worker's error too; the writer's copy is the cause as it is, and werr + // is then a bare context.Canceled. The prefixes name where an error of + // the pipeline's own arose. + switch { + case cause != nil && werr == nil: + cause = fmt.Errorf("writer error: %w", cause) + case cause == nil && werr != nil: + cause = fmt.Errorf("worker error: %w", werr) + case cause == nil: + // A canceled pipeline never finalizes: its producer may have stopped + // before dispatching every block. + cause = canceled + } + if cause != nil { + return errors.Join(cause, b.cleanup()) } - return b.finalizeIndex() } diff --git a/builder_unsorted.go b/builder_unsorted.go index 4eff375..628932e 100644 --- a/builder_unsorted.go +++ b/builder_unsorted.go @@ -552,7 +552,7 @@ type UnsortedBuilder struct { // totalKeys may be 0: the result is a valid empty index whose every QueryRank // returns ErrNotFound. func NewUnsortedBuilder(ctx context.Context, output string, totalKeys uint64, tempDir string, opts ...BuildOption) (*UnsortedBuilder, error) { - b, err := newBuilder(ctx, output, totalKeys, opts...) + b, err := newBuilder(ctx, output, totalKeys, newBuildConfig(opts)) if err != nil { return nil, err } diff --git a/builder_unsorted_finish.go b/builder_unsorted_finish.go index f144237..44a9002 100644 --- a/builder_unsorted_finish.go +++ b/builder_unsorted_finish.go @@ -357,7 +357,7 @@ func (ub *UnsortedBuilder) finishUnsortedFastPath() error { } if b.workers > 1 { - b.initParallelWorkers() + b.startPipeline(b.runWorker) } fpSize := b.cfg.fingerprintSize @@ -512,7 +512,7 @@ func (ub *UnsortedBuilder) finishUnsortedSingleThreaded() error { // per-partition channels. func (ub *UnsortedBuilder) finishUnsortedParallel() error { b := ub.b - b.initParallelWorkers() + b.startPipeline(b.runWorker) u := ub.unsortedBuf numParts := u.numPartitions diff --git a/doc.go b/doc.go index d65fbf1..9ada06e 100644 --- a/doc.go +++ b/doc.go @@ -59,7 +59,7 @@ // - Public API: builder.go (NewSortedBuilder), builder_unsorted.go (NewUnsortedBuilder), index.go (Open, QueryRank, QueryBatch, PayloadIndex) // - Configuration: builder_options.go (BuildOption, With* functions) // - Serialization: header.go (header, footer, ramIndexEntry), index_writer.go -// - Sorted files: sorted_file.go (SortedFileWriter, ReadSortedFileHeader, OpenSortedFile), sorted_merge.go (MergeSortedFiles) +// - Sorted files: sorted_file.go (SortedFileWriter, ReadSortedFileHeader, OpenSortedFile), sorted_merge.go (MergeSortedFiles), sorted_build.go (BuildFromSortedFiles) // - Key routing & hashing: key.go (fingerprint extraction, payload packing), prehash.go (PreHash); block routing is FastRange32 over the big-endian prefix, inline in builder.go/index.go // - Algorithm dispatch: algorithm.go (blockBuilder/blockDecoder interfaces, factory functions) // - Block algorithms: internal/bijection/ (EF/GR), internal/ptrhash/ (Cuckoo) diff --git a/sorted_build.go b/sorted_build.go new file mode 100644 index 0000000..1b08601 --- /dev/null +++ b/sorted_build.go @@ -0,0 +1,599 @@ +package streamhash + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math/bits" + "os" + "sync" + + intbits "github.com/stellar/streamhash/internal/bits" +) + +const ( + // sortedBuildReadBudget is the memory for read buffers, and + // sortedBuildStripeBudget for the stripes in flight. + sortedBuildReadBudget = 128 << 20 + sortedBuildStripeBudget = 128 << 20 + + // A read buffer is between these sizes. + sortedBuildMinBuffer = 64 << 10 + sortedBuildMaxBuffer = 4 << 20 + + // sortedBuildReadGoroutines fill the read buffers. + sortedBuildReadGoroutines = 16 + + // A stripe holds about this many keys, within these bounds. + sortedBuildMinStripeKeys = 32 << 10 + sortedBuildMaxStripeKeys = 256 << 10 + + // sortedBuildStripesWaiting is how many gathered stripes may wait for a + // worker, beyond the one each worker is solving. + sortedBuildStripesWaiting = 2 +) + +// BuildFromSortedFiles builds the index at output from the sorted files at +// inputs. The key count is the sum of their record counts, and their payload +// size is the index's. The options are NewSortedBuilder's; WithPayload is not +// needed, and must agree with the files if given. The index is byte-identical +// to one built by adding the same keys to a SortedBuilder in block order. +// +// The files are not merged. Each is checked, then read once, sequentially and +// directly where the filesystem allows it. A file whose keys are out of order +// fails the build, and so does a duplicate key. +// +// Speed: about 6M keys/s per worker (WithWorkers), up to what the disk reads +// at 16+PayloadSize bytes per key. Memory: about 256 MiB, plus each worker's +// block state (about 0.5 MB with bijection, 5 MB with PTRHash), whatever the +// number of keys or files. The build keeps one file open per input. +func BuildFromSortedFiles(ctx context.Context, output string, inputs []string, opts ...BuildOption) error { + return buildFromSortedFiles(ctx, output, inputs, sortedBuildTuning{}, opts...) +} + +// sortedBuildTuning overrides the build's sizing; zero values mean the +// defaults. Tests use it to force stripe and buffer sizes, and to act once a +// stripe is gathered. +type sortedBuildTuning struct { + stripeKeys int + bufBytes int + afterGather func(s int) +} + +func buildFromSortedFiles(ctx context.Context, output string, inputs []string, tune sortedBuildTuning, opts ...BuildOption) error { + if len(inputs) == 0 { + return errors.New("no sorted files to build from") + } + if err := ctx.Err(); err != nil { + return err + } + runs, payloadSize, total, err := readSortedRuns(inputs) + if err != nil { + return err + } + if err := checkOutputNotInput(output, inputs); err != nil { + return err + } + cfg := newBuildConfig(opts) + if cfg.payloadSet && cfg.payloadSize != payloadSize { + return fmt.Errorf("the sorted files have %d-byte payloads, WithPayload asks for %d", payloadSize, cfg.payloadSize) + } + cfg.payloadSize = payloadSize + + // The build's own context lets close stop the reads at once after a failure. + ctx, cancel := context.WithCancel(ctx) + b, err := newBuilder(ctx, output, total, cfg) + if err != nil { + cancel() + return err + } + g := &sortedBuild{b: b, recordSize: MinKeySize + payloadSize, tune: tune} + defer func() { + cancel() + g.close() + }() + if err := g.start(ctx, runs); err != nil { + return errors.Join(err, b.cleanup()) + } + if b.workers > 1 { + return g.buildParallel() + } + return g.buildInline(ctx) +} + +// sortedBuild is the state of one BuildFromSortedFiles call. +type sortedBuild struct { + b *builder + tune sortedBuildTuning + recordSize int + geo stripeGeometry + inFlight int // stripes gathered at a time: one per worker, plus those waiting + stripeCap int // records a stripe buffer is sized for + readers []*fileReader + reqs chan readRequest + readWg sync.WaitGroup + gathered uint64 +} + +// start sizes the stripes and buffers, opens the runs and starts reading +// them, with reads served until ctx is done. +func (g *sortedBuild) start(ctx context.Context, runs []sortedRun) error { + b := g.b + var err error + g.inFlight = b.workers + sortedBuildStripesWaiting + stripeKeys := sortedBuildStripeKeys(g.inFlight, g.recordSize, g.tune.stripeKeys) + if g.geo, err = newStripeGeometry(b.numBlocks, b.cfg.totalKeys, stripeKeys); err != nil { + return err + } + perStripe := int(b.cfg.totalKeys/uint64(b.numBlocks)) * int(g.geo.blocksPerStripe) + g.stripeCap = perStripe + perStripe/16 + 1024 + if len(runs) == 0 { + return nil + } + + bufSize := sortedBuildBufferSize(len(runs), g.recordSize, g.tune.bufBytes) + nbuf := 2 + if 3*len(runs)*bufSize <= sortedBuildReadBudget { + nbuf = 3 + } + g.reqs = make(chan readRequest, len(runs)*nbuf) + for _, run := range runs { + f, err := openDirect(run.path) + if err != nil { + return err + } + r := &fileReader{run: run, f: f, reqs: g.reqs, next: run.start, valid: -1} + for range nbuf { + r.bufs = append(r.bufs, alignedBuffer(bufSize)) + r.done = append(r.done, make(chan readResult, 1)) + } + g.readers = append(g.readers, r) + } + for range min(sortedBuildReadGoroutines, len(runs)*nbuf) { + g.readWg.Go(func() { serveReads(ctx, g.reqs) }) + } + for _, r := range g.readers { + for i := range r.bufs { + r.post(i) + } + } + return nil +} + +// close stops the reads and closes the files. The workers and the writer are +// already stopped: the build drained them or cleaned up. +func (g *sortedBuild) close() { + if g.reqs != nil { + close(g.reqs) + } + g.readWg.Wait() + for _, r := range g.readers { + r.f.Close() + } +} + +// sortedBuildStripeKeys returns how many keys a stripe holds: as many as let +// inFlight stripes, a gathered record and a routed entry per key, fit in the +// budget, within the bounds. +func sortedBuildStripeKeys(inFlight, recordSize, stripeKeys int) int { + if stripeKeys > 0 { + return stripeKeys + } + perKey := recordSize + readEntrySize + return min(max(sortedBuildStripeBudget/(inFlight*perKey), sortedBuildMinStripeKeys), sortedBuildMaxStripeKeys) +} + +// sortedBuildBufferSize returns the read buffer size for files of +// recordSize-byte records: the budget shared between two buffers per file, +// within the limits, and a multiple of both directAlign and recordSize, so +// that every buffer starts and ends at a record. +func sortedBuildBufferSize(files, recordSize, bufBytes int) int { + if bufBytes <= 0 { + bufBytes = min(max(sortedBuildReadBudget/2/files, sortedBuildMinBuffer), sortedBuildMaxBuffer) + } + // directAlign is a power of two, so this is lcm(directAlign, recordSize). + unit := directAlign * (recordSize / (recordSize & -recordSize)) + return max(bufBytes/unit, 1) * unit +} + +// stripeGeometry splits the index's blocks into stripes of consecutive +// blocks, each of about the same number of keys. +type stripeGeometry struct { + numBlocks uint32 + blocksPerStripe uint32 + numStripes int +} + +// newStripeGeometry makes stripes of about stripeKeys keys, and checks that +// every stripe's first prefix is the first FastRange32 maps to its first +// block, so that no block spans two stripes. +func newStripeGeometry(numBlocks uint32, totalKeys uint64, stripeKeys int) (stripeGeometry, error) { + per := uint64(numBlocks) + if totalKeys > 0 { + per = min(max(uint64(stripeKeys)*uint64(numBlocks)/totalKeys, 1), uint64(numBlocks)) + } + geo := stripeGeometry{numBlocks: numBlocks, blocksPerStripe: uint32(per), numStripes: int((uint64(numBlocks) + per - 1) / per)} + for s := 1; s < geo.numStripes; s++ { + first, _ := geo.blocks(s) + p := lowPrefix(first, numBlocks) + if intbits.FastRange32(p, numBlocks) != first || intbits.FastRange32(p-1, numBlocks) != first-1 { + return geo, fmt.Errorf("stripe %d: prefix %#x is not the first of block %d of %d", s, p, first, numBlocks) + } + } + return geo, nil +} + +// blocks returns stripe s's first block and how many blocks it has. +func (geo stripeGeometry) blocks(s int) (first, count uint32) { + first = uint32(s) * geo.blocksPerStripe + return first, min(geo.blocksPerStripe, geo.numBlocks-first) +} + +// bound returns the first prefix past stripe s, unless s is the last stripe. +func (geo stripeGeometry) bound(s int) (uint64, bool) { + if s+1 >= geo.numStripes { + return 0, false + } + first, _ := geo.blocks(s + 1) + return lowPrefix(first, geo.numBlocks), true +} + +// lowPrefix returns the smallest prefix FastRange32 maps to block b of n: +// ceil(b * 2^64 / n). +func lowPrefix(b, n uint32) uint64 { + q, r := bits.Div64(uint64(b), 0, uint64(n)) + if r != 0 { + q++ + } + return q +} + +// A stripe is the records of a run of consecutive blocks, gathered from +// every file. +type stripe struct { + first uint32 // its first block + blocks uint32 // how many blocks it has + keysBefore uint64 // the keys of the stripes before it + data []byte // its records, file by file + runs []int // how many records came from each file +} + +func (g *sortedBuild) newStripe() *stripe { + return &stripe{data: make([]byte, 0, g.stripeCap*g.recordSize), runs: make([]int, len(g.readers))} +} + +// gather copies stripe s's run out of every file into st. +func (g *sortedBuild) gather(ctx context.Context, s int, st *stripe) error { + bound, bounded := g.geo.bound(s) + st.first, st.blocks = g.geo.blocks(s) + st.keysBefore = g.gathered + st.data = st.data[:0] + recordSize := g.recordSize + for i, r := range g.readers { + count := 0 + for { + ok, err := r.ready(ctx) + if err != nil { + return err + } + if !ok { + break + } + rest := r.bufs[r.cur][r.pos:r.valid] + n := len(rest) / recordSize + cut := n + if bounded { + cut = firstAtOrPast(rest, recordSize, n, bound) + } + st.data = append(st.data, rest[:cut*recordSize]...) + r.pos += cut * recordSize + count += cut + if cut < n { + break + } + r.advance() + } + st.runs[i] = count + g.gathered += uint64(count) + } + if g.tune.afterGather != nil { + g.tune.afterGather(s) + } + return nil +} + +// firstAtOrPast returns the index of the first of the n records in buf whose +// prefix is at or past bound, or n. It scans the first records in order, +// since with many files a run is a few dozen records, then gallops. +func firstAtOrPast(buf []byte, recordSize, n int, bound uint64) int { + prefix := func(i int) uint64 { return binary.BigEndian.Uint64(buf[i*recordSize:]) } + i := 0 + for ; i < min(n, 64); i++ { + if prefix(i) >= bound { + return i + } + } + if i == n { + return n + } + lo, hi := i-1, i + for hi < n && prefix(hi) < bound { + lo, hi = hi, hi*2 + } + hi = min(hi, n) + for lo+1 < hi { + mid := (lo + hi) / 2 + if prefix(mid) < bound { + lo = mid + } else { + hi = mid + } + } + return hi +} + +// checkCount checks that the gathered records add up to the files' counts. +func (g *sortedBuild) checkCount() error { + if g.gathered != g.b.cfg.totalKeys { + return fmt.Errorf("%w: read %d records, the sorted files count %d", ErrKeyCountMismatch, g.gathered, g.b.cfg.totalKeys) + } + return nil +} + +// buildInline gathers, routes, solves and commits each stripe in turn on the +// calling goroutine. +func (g *sortedBuild) buildInline(ctx context.Context) error { + b := g.b + st, router := g.newStripe(), g.newStripeRouter() + for s := range g.geo.numStripes { + if err := g.buildStripeInline(ctx, s, st, router); err != nil { + return errors.Join(err, b.cleanup()) + } + } + if err := g.checkCount(); err != nil { + return errors.Join(err, b.cleanup()) + } + return b.finalizeIndex() +} + +// buildStripeInline gathers stripe s into st, routes it, and solves and +// commits its blocks, the empty ones included. +func (g *sortedBuild) buildStripeInline(ctx context.Context, s int, st *stripe, router *stripeRouter) error { + if err := ctx.Err(); err != nil { + return err + } + if err := g.gather(ctx, s, st); err != nil { + return err + } + flat, ends, err := router.route(st) + if err != nil { + return err + } + b, start := g.b, 0 + for _, end := range ends { + b.loadBlock(b.builder, flat[start:end]) + if err := b.buildBlock(); err != nil { + return err + } + start = end + } + return nil +} + +// buildParallel gathers stripes for the workers, which route and solve them +// and hand their blocks to the writer. +func (g *sortedBuild) buildParallel() error { + b := g.b + free := make(chan *stripe, g.inFlight) + for range g.inFlight { + free <- g.newStripe() + } + stripes := make(chan *stripe, g.inFlight) + b.startPipeline(func() error { return g.runStripeWorker(stripes, free) }) + err := g.gatherAll(b.workerCtx, stripes, free) + if err != nil && b.workerCtx.Err() == nil { + // The gather failed on its own, so stop the pipeline. + return errors.Join(err, b.cleanup()) + } + // All gathered, or the pipeline failed or the build was canceled: the + // drain reports the cause, and never finalizes a canceled pipeline. + return b.drainParallelPipeline() +} + +// gatherAll gathers every stripe into a buffer from free and queues it on +// stripes, which it closes when it returns: at the end, at the first error, +// or once ctx is done. +func (g *sortedBuild) gatherAll(ctx context.Context, stripes chan<- *stripe, free <-chan *stripe) error { + defer close(stripes) + for s := range g.geo.numStripes { + var st *stripe + select { + case st = <-free: + case <-ctx.Done(): + return ctx.Err() + } + if err := g.gather(ctx, s, st); err != nil { + return err + } + stripes <- st // never blocks: inFlight stripes exist and stripes holds inFlight + } + return g.checkCount() +} + +// runStripeWorker routes and solves stripes until stripes closes or the +// workers' context is done. +func (g *sortedBuild) runStripeWorker(stripes <-chan *stripe, free chan<- *stripe) error { + b := g.b + solver, err := b.newBlockSolver() + if err != nil { + return err + } + router := g.newStripeRouter() + for { + var st *stripe + select { + case s, ok := <-stripes: + if !ok { + return nil + } + st = s + case <-b.workerCtx.Done(): + return b.workerCtx.Err() + } + flat, ends, err := router.route(st) + if err != nil { + return err + } + first, keysBefore := st.first, st.keysBefore + free <- st // never blocks: the pool has room for every stripe + start := 0 + for i, end := range ends { + b.loadBlock(solver.blk, flat[start:end]) + if err := b.sendResult(solver.build(first+uint32(i), keysBefore+uint64(start))); err != nil { + return err + } + start = end + } + } +} + +// stripeRouter sorts a stripe's records by block. +type stripeRouter struct { + g *sortedBuild + ends []int + flat []routedEntry +} + +func (g *sortedBuild) newStripeRouter() *stripeRouter { + return &stripeRouter{g: g, ends: make([]int, g.geo.blocksPerStripe+1), flat: make([]routedEntry, 0, g.stripeCap)} +} + +// route sorts st's records by block, by a counting sort, into flat, and +// returns flat and where each block ends in it: block i of the stripe is +// flat[ends[i-1]:ends[i]], with ends[-1] taken as 0. A record out of order +// in its file's run, or outside the stripe, fails with ErrUnsortedInput. +func (r *stripeRouter) route(st *stripe) ([]routedEntry, []int, error) { + g := r.g + recordSize, numBlocks := g.recordSize, g.geo.numBlocks + first, blocks := st.first, st.blocks + data := st.data + ends := r.ends[:blocks+1] + clear(ends) + off := 0 + for i, n := range st.runs { + var last sortKey + for end := off + n*recordSize; off < end; off += recordSize { + k := sortKeyOf(data[off : off+MinKeySize : off+MinKeySize]) + if k.less(last) { + return nil, nil, fmt.Errorf("sorted file %s: %w", g.readers[i].run.path, ErrUnsortedInput) + } + last = k + blk := intbits.FastRange32(k.hi, numBlocks) - first + if blk >= blocks { + return nil, nil, fmt.Errorf("sorted file %s: %w: a record is outside its stripe", g.readers[i].run.path, ErrUnsortedInput) + } + ends[blk+1]++ + } + } + for i := 1; i < len(ends); i++ { + ends[i] += ends[i-1] + } + + n := len(data) / recordSize + if cap(r.flat) < n { + r.flat = make([]routedEntry, n) + } + flat := r.flat[:n] + place := func(k0, k1, payload uint64) { + blk := intbits.FastRange32(bits.ReverseBytes64(k0), numBlocks) - first + flat[ends[blk]] = routedEntry{k0: k0, k1: k1, payload: payload} + ends[blk]++ + } + // Every record but the last has 8 readable bytes past its key, so its + // payload is one masked load; the last record decodes its own bytes. + payloadSize := g.b.cfg.payloadSize + mask := ^uint64(0) >> (64 - 8*payloadSize) + off = 0 + for ; off+MinKeySize+8 <= len(data); off += recordSize { + rec := data[off : off+MinKeySize+8 : off+MinKeySize+8] + place(binary.LittleEndian.Uint64(rec[0:8]), binary.LittleEndian.Uint64(rec[8:16]), binary.LittleEndian.Uint64(rec[16:24])&mask) + } + for ; off < len(data); off += recordSize { + rec := data[off : off+recordSize : off+recordSize] + place(binary.LittleEndian.Uint64(rec[0:8]), binary.LittleEndian.Uint64(rec[8:16]), unpackPayloadFromBytes(rec[MinKeySize:], payloadSize)) + } + return flat, ends[:blocks], nil +} + +// fileReader reads one file's records into aligned buffers, which the read +// goroutines fill ahead of the gatherer, the one consumer. +type fileReader struct { + run sortedRun + f *os.File + reqs chan<- readRequest + bufs [][]byte + done []chan readResult // each buffer's read result; never full + next int64 // the file offset of the next read to post + cur int // the buffer being consumed + pos int // bytes of bufs[cur] consumed + valid int // bytes of records in bufs[cur]: 0 past the run's end, -1 until its result is in +} + +type readRequest struct { + r *fileReader + buf int + off int64 +} + +type readResult struct { + n int + err error +} + +// serveReads fills buffers as requested until reqs closes. Once ctx is done +// it answers the requests without reading. +func serveReads(ctx context.Context, reqs <-chan readRequest) { + for q := range reqs { + var res readResult + if res.err = ctx.Err(); res.err == nil { + res.n, res.err = q.r.run.readAt(q.r.f, q.r.bufs[q.buf], q.off) + } + q.r.done[q.buf] <- res + } +} + +// post asks for buffer i to be filled from the file's next unread part, or +// marks it empty once the run is all posted. +func (r *fileReader) post(i int) { + if r.next >= r.run.end { + r.done[i] <- readResult{} + return + } + r.reqs <- readRequest{r: r, buf: i, off: r.next} + r.next += int64(len(r.bufs[i])) +} + +// ready waits for the current buffer's read if its result is not in yet, and +// reports whether the buffer holds records left to consume. +func (r *fileReader) ready(ctx context.Context) (bool, error) { + if r.valid < 0 { + select { + case res := <-r.done[r.cur]: + if res.err != nil { + return false, res.err + } + r.valid = res.n + case <-ctx.Done(): + return false, ctx.Err() + } + } + return r.valid > 0, nil +} + +// advance posts a refill of the consumed buffer and moves on to the next. +func (r *fileReader) advance() { + r.post(r.cur) + r.cur = (r.cur + 1) % len(r.bufs) + r.pos, r.valid = 0, -1 +} diff --git a/sorted_build_test.go b/sorted_build_test.go new file mode 100644 index 0000000..b2008dc --- /dev/null +++ b/sorted_build_test.go @@ -0,0 +1,550 @@ +package streamhash + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "testing" + "time" + + intbits "github.com/stellar/streamhash/internal/bits" +) + +// buildFilesWithWatchdog builds from inputs with tune, failing the test if +// the build does not return. +func buildFilesWithWatchdog(t *testing.T, ctx context.Context, output string, inputs []string, tune sortedBuildTuning, opts ...BuildOption) error { + t.Helper() + return runBuildWithWatchdog(t, 2*time.Minute, func() error { + return buildFromSortedFiles(ctx, output, inputs, tune, opts...) + }) +} + +func readFileBytes(t testing.TB, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data +} + +// referenceIndex builds recs with a SortedBuilder and returns the index +// file's bytes. +func referenceIndex(t testing.TB, path string, recs []sortedRecord, opts ...BuildOption) []byte { + t.Helper() + entries := make([]entry, len(recs)) + for i, r := range recs { + entries[i] = entry{Key: recs[i].key[:], Payload: r.payload} + } + if err := buildFromEntries(context.Background(), path, entries, opts...); err != nil { + t.Fatal(err) + } + return readFileBytes(t, path) +} + +// The layouts in sortedFileLayouts. +const ( + layoutSpread = iota + layoutOneFile + layoutEmptyFiles + layoutRanges +) + +// sortedFileLayouts are ways to spread records over files: file returns +// the file of record i of n, out of files. +var sortedFileLayouts = []struct { + name string + files int + file func(rng *rand.Rand, i, n, files int) int +}{ + {"spread", 24, func(rng *rand.Rand, _, _, files int) int { return rng.IntN(files) }}, + {"one file", 1, func(*rand.Rand, int, int, int) int { return 0 }}, + // The odd-numbered files stay empty. + {"empty files among full", 12, func(rng *rand.Rand, _, _, files int) int { return 2 * rng.IntN(files/2) }}, + // Each file holds a key range, so most stripes have records in one file. + {"ranges", 8, func(_ *rand.Rand, i, n, files int) int { return i * files / n }}, +} + +// writeLayout writes recs, which are in key order, to files under dir as +// layout spreads them, and returns their paths. +func writeLayout(t *testing.T, dir string, rng *rand.Rand, recs []sortedRecord, payloadSize, layout int) []string { + t.Helper() + l := sortedFileLayouts[layout] + parts := make([][]sortedRecord, l.files) + for i, r := range recs { + f := l.file(rng, i, len(recs), l.files) + parts[f] = append(parts[f], r) + } + paths := make([]string, l.files) + for f, part := range parts { + paths[f] = filepath.Join(dir, fmt.Sprintf("layout%d-%02d.sorted", layout, f)) + writeSortedFile(t, paths[f], SortedFileConfig{PayloadSize: payloadSize}, part) + } + return paths +} + +// TestBuildFromSortedFilesEquivalence checks that the build's index is +// byte-identical to a SortedBuilder's, for both algorithms, payload and +// fingerprint sizes, worker counts, stripe sizes and file layouts. +func TestBuildFromSortedFilesEquivalence(t *testing.T) { + const numKeys = 30000 + rng := newTestRNG(t) + // Stripe sizes: the default, which is one stripe at this size; one block + // per stripe; and a few blocks per stripe, with a partial last stripe. + stripeKeys := []int{0, 1, 9000} + workers := []int{1, 2, 4, 16} + for _, c := range []struct{ payloadSize, fpSize int }{{0, 0}, {3, 1}, {5, 3}, {8, 4}} { + recs := uniqueSortedRecords(rng, numKeys, c.payloadSize) + dir := t.TempDir() + files := make([][]string, len(sortedFileLayouts)) + for l := range sortedFileLayouts { + files[l] = writeLayout(t, dir, rng, recs, c.payloadSize, l) + } + for _, algo := range []Algorithm{AlgoBijection, AlgoPTRHash} { + opts := []BuildOption{WithAlgorithm(algo), WithFingerprint(c.fpSize)} + want := referenceIndex(t, filepath.Join(dir, "reference.idx"), recs, append(opts, WithPayload(c.payloadSize))...) + for l, layout := range sortedFileLayouts { + for _, w := range workers { + for _, s := range stripeKeys { + name := fmt.Sprintf("%s/payload=%d/fp=%d/%s/workers=%d/stripe=%d", algo, c.payloadSize, c.fpSize, layout.name, w, s) + t.Run(name, func(t *testing.T) { + output := filepath.Join(dir, "built.idx") + err := buildFilesWithWatchdog(t, context.Background(), output, files[l], sortedBuildTuning{stripeKeys: s}, append(opts, WithWorkers(w))...) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFileBytes(t, output), want) { + t.Fatal("the index differs from the SortedBuilder's") + } + }) + } + } + } + } + } +} + +// TestBuildFromSortedFilesAcrossBufferSeams checks the equivalence with the +// smallest read buffers and many bounded stripes, so that the gatherer's cut +// crosses read-buffer seams and gallops past the records it scans in order. +func TestBuildFromSortedFilesAcrossBufferSeams(t *testing.T) { + rng := newTestRNG(t) + for _, c := range []struct{ payloadSize, fpSize, files, numKeys int }{{0, 0, 1, 60000}, {3, 1, 4, 60000}, {8, 4, 2, 40000}} { + recs := uniqueSortedRecords(rng, c.numKeys, c.payloadSize) + dir := t.TempDir() + parts := make([][]sortedRecord, c.files) + for _, r := range recs { + f := rng.IntN(c.files) + parts[f] = append(parts[f], r) + } + files := make([]string, c.files) + for f, part := range parts { + files[f] = filepath.Join(dir, fmt.Sprintf("in%02d.sorted", f)) + writeSortedFile(t, files[f], SortedFileConfig{PayloadSize: c.payloadSize}, part) + } + for _, algo := range []Algorithm{AlgoBijection, AlgoPTRHash} { + opts := []BuildOption{WithAlgorithm(algo), WithFingerprint(c.fpSize)} + want := referenceIndex(t, filepath.Join(dir, "reference.idx"), recs, append(opts, WithPayload(c.payloadSize))...) + for _, w := range []int{1, 3} { + for _, s := range []int{3000, 9000} { + // The smallest buffers, then three of a 19-byte record's units. + for _, buf := range []int{1, 3 * 77824} { + name := fmt.Sprintf("%s/payload=%d/fp=%d/files=%d/workers=%d/stripe=%d/buf=%d", algo, c.payloadSize, c.fpSize, c.files, w, s, buf) + t.Run(name, func(t *testing.T) { + output := filepath.Join(dir, "built.idx") + err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{stripeKeys: s, bufBytes: buf}, append(opts, WithWorkers(w))...) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFileBytes(t, output), want) { + t.Fatal("the index differs from the SortedBuilder's") + } + }) + } + } + } + } + } +} + +// TestBuildFromSortedFilesSmall covers no keys at all, which is two empty +// blocks, and totals so small that blocks are empty, with and without empty +// files. +func TestBuildFromSortedFilesSmall(t *testing.T) { + rng := newTestRNG(t) + for _, n := range []int{0, 1, 2, 3, 7} { + recs := uniqueSortedRecords(rng, n, 2) + dir := t.TempDir() + opts := []BuildOption{WithPayload(2), WithFingerprint(1)} + want := referenceIndex(t, filepath.Join(dir, "reference.idx"), recs, opts...) + inputs := map[string][]string{ + "one file": writeLayout(t, dir, rng, recs, 2, layoutOneFile), + "empty files": writeLayout(t, dir, rng, recs, 2, layoutEmptyFiles), + } + keys := make([][]byte, len(recs)) + for i := range recs { + keys[i] = recs[i].key[:] + } + for name, files := range inputs { + for _, w := range []int{1, 4} { + for _, s := range []int{0, 1} { + t.Run(fmt.Sprintf("keys=%d/%s/workers=%d/stripe=%d", n, name, w, s), func(t *testing.T) { + output := filepath.Join(dir, "built.idx") + err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{stripeKeys: s}, WithFingerprint(1), WithWorkers(w)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFileBytes(t, output), want) { + t.Fatal("the index differs from the SortedBuilder's") + } + idx, err := Open(output) + if err != nil { + t.Fatal(err) + } + defer idx.Close() + if stats := idx.Stats(); stats.NumKeys != uint64(len(recs)) || stats.NumBlocks != 2 { + t.Fatalf("the index has %d keys in %d blocks, want %d in 2", stats.NumKeys, stats.NumBlocks, len(recs)) + } + verifyMPHF(t, idx, keys) + }) + } + } + } + } +} + +// TestBuildFromSortedFilesBytesPerSync checks that writeback changes nothing +// in the index, even when it starts on nearly every write. +func TestBuildFromSortedFilesBytesPerSync(t *testing.T) { + rng := newTestRNG(t) + recs := uniqueSortedRecords(rng, 50000, 3) + dir := t.TempDir() + files := writeLayout(t, dir, rng, recs, 3, layoutSpread) + for _, w := range []int{1, 4} { + var indexes [][]byte + for _, bytesPerSync := range []int{0, 1} { + output := filepath.Join(dir, fmt.Sprintf("sync%d-%d.idx", bytesPerSync, w)) + err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{stripeKeys: 9000}, WithFingerprint(1), WithWorkers(w), WithBytesPerSync(bytesPerSync)) + if err != nil { + t.Fatal(err) + } + indexes = append(indexes, readFileBytes(t, output)) + } + if !bytes.Equal(indexes[0], indexes[1]) { + t.Fatalf("workers=%d: the index built with WithBytesPerSync differs from the one built without it", w) + } + } +} + +// TestBuildFromSortedFilesPayloadOption checks that WithPayload is accepted +// when it matches the files and rejected when it does not. +func TestBuildFromSortedFilesPayloadOption(t *testing.T) { + rng := newTestRNG(t) + recs := uniqueSortedRecords(rng, 3000, 3) + dir := t.TempDir() + files := writeLayout(t, dir, rng, recs, 3, layoutSpread) + want := referenceIndex(t, filepath.Join(dir, "reference.idx"), recs, WithPayload(3)) + output := filepath.Join(dir, "built.idx") + if err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{}, WithPayload(3)); err != nil { + t.Fatal(err) + } + if !bytes.Equal(readFileBytes(t, output), want) { + t.Fatal("the index built with a matching WithPayload differs from the SortedBuilder's") + } + for _, ps := range []int{0, 4} { + output := filepath.Join(dir, "rejected.idx") + if err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{}, WithPayload(ps)); err == nil { + t.Fatalf("WithPayload(%d) on files with 3-byte payloads: want an error", ps) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("a rejected build left its output: %v", err) + } + } +} + +// TestBuildFromSortedFilesRejectsOutputAmongInputs checks that a build into +// one of its inputs fails before creating the output, which would truncate +// the input. +func TestBuildFromSortedFilesRejectsOutputAmongInputs(t *testing.T) { + rng := newTestRNG(t) + dir := t.TempDir() + files := writeLayout(t, dir, rng, uniqueSortedRecords(rng, 3000, 3), 3, layoutSpread) + before := readFileBytes(t, files[3]) + // files[3], spelled differently. + output := dir + string(filepath.Separator) + "." + string(filepath.Separator) + filepath.Base(files[3]) + if err := buildFilesWithWatchdog(t, context.Background(), output, files, sortedBuildTuning{}); err == nil { + t.Fatal("a build into one of its inputs returned no error") + } + if !bytes.Equal(readFileBytes(t, files[3]), before) { + t.Fatal("the build changed its input") + } +} + +// TestBuildFromSortedFilesRejects covers inputs the build must fail on, at +// one worker and several, and that a failed build removes its output. +func TestBuildFromSortedFilesRejects(t *testing.T) { + const numKeys = 30000 + rng := newTestRNG(t) + dir := t.TempDir() + write := func(name string, payloadSize int, recs []sortedRecord) string { + path := filepath.Join(dir, name+".sorted") + writeSortedFile(t, path, SortedFileConfig{PayloadSize: payloadSize}, recs) + return path + } + // swapped returns a copy of path with records i and i+1 swapped. + swapped := func(path string, recordSize, i int) string { + data := readFileBytes(t, path) + swapRecords(data, recordSize, i) + out := filepath.Join(dir, fmt.Sprintf("swapped%d-%s", i, filepath.Base(path))) + if err := os.WriteFile(out, data, 0o600); err != nil { + t.Fatal(err) + } + return out + } + recs3 := uniqueSortedRecords(rng, numKeys, 3) + clean3 := write("clean3", 3, recs3) + other3 := write("other3", 3, uniqueSortedRecords(rng, 5000, 3)) + recs0 := uniqueSortedRecords(rng, numKeys, 0) + clean0 := write("clean0", 0, recs0) + + truncated := filepath.Join(dir, "truncated.sorted") + if err := os.WriteFile(truncated, readFileBytes(t, clean3)[:sortedFileAlign+100*19+7], 0o600); err != nil { + t.Fatal(err) + } + + // A stripe boundary: with 9000-key stripes the geometry has a few blocks + // per stripe, and the first record of stripe 1 is the first whose prefix + // is at or past the stripe's bound. + numBlocks, err := numBlocksForAlgo(AlgoBijection, uint64(len(recs3)), 3, 0) + if err != nil { + t.Fatal(err) + } + geo, err := newStripeGeometry(numBlocks, uint64(len(recs3)), 9000) + if err != nil { + t.Fatal(err) + } + if geo.numStripes < 3 { + t.Fatalf("%d stripes, want several", geo.numStripes) + } + bound, _ := geo.bound(0) + boundary := slices.IndexFunc(recs3, func(r sortedRecord) bool { return binary.BigEndian.Uint64(r.key[:8]) >= bound }) + if boundary < 1 { + t.Fatalf("no record at or past the first stripe bound %#x", bound) + } + + // The seam between the smallest read buffers of 16-byte records. + seam := sortedBuildBufferSize(1, MinKeySize, 1) / MinKeySize + + withDuplicate := slices.Insert(slices.Clone(recs3), 1000, recs3[1000]) + duplicateIn := write("duplicate", 3, withDuplicate) + duplicateAcross := write("duplicate-across", 3, recs3[1000:1001]) + + for _, tc := range []struct { + name string + inputs []string + tune sortedBuildTuning + want error + }{ + {"no inputs", nil, sortedBuildTuning{}, nil}, + {"mismatched payload sizes", []string{clean3, clean0}, sortedBuildTuning{}, nil}, + {"a truncated file", []string{clean3, truncated}, sortedBuildTuning{}, ErrTruncatedFile}, + {"a record out of order inside a run", []string{other3, swapped(clean3, 19, 777)}, sortedBuildTuning{stripeKeys: 9000}, ErrUnsortedInput}, + {"a record out of order across a read-buffer seam", []string{swapped(clean0, 16, seam-1)}, sortedBuildTuning{bufBytes: 1}, ErrUnsortedInput}, + {"a record outside its stripe", []string{other3, swapped(clean3, 19, boundary-1)}, sortedBuildTuning{stripeKeys: 9000}, ErrUnsortedInput}, + {"a duplicate key within one file", []string{duplicateIn}, sortedBuildTuning{}, ErrDuplicateKey}, + {"a duplicate key across two files", []string{clean3, duplicateAcross}, sortedBuildTuning{}, ErrDuplicateKey}, + } { + for _, w := range []int{1, 4} { + for _, algo := range []Algorithm{AlgoBijection, AlgoPTRHash} { + t.Run(fmt.Sprintf("%s/workers=%d/%s", tc.name, w, algo), func(t *testing.T) { + output := filepath.Join(dir, "rejected.idx") + err := buildFilesWithWatchdog(t, context.Background(), output, tc.inputs, tc.tune, WithWorkers(w), WithAlgorithm(algo)) + if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("the build returned %v, want %v", err, tc.want) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("a failed build left its output: %v", err) + } + }) + } + } + } +} + +// TestBuildFromSortedFilesCanceled checks that a build canceled before it +// starts, or between stripes, returns the cancellation and removes its +// output. +func TestBuildFromSortedFilesCanceled(t *testing.T) { + rng := newTestRNG(t) + recs := uniqueSortedRecords(rng, 30000, 3) + dir := t.TempDir() + files := writeLayout(t, dir, rng, recs, 3, layoutSpread) + for _, w := range []int{1, 4} { + for _, at := range []int{-1, 0, 3} { + t.Run(fmt.Sprintf("workers=%d/stripe=%d", w, at), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tune := sortedBuildTuning{stripeKeys: 1} + if at < 0 { + cancel() + } else { + tune.afterGather = func(s int) { + if s == at { + cancel() + } + } + } + output := filepath.Join(dir, "canceled.idx") + err := buildFilesWithWatchdog(t, ctx, output, files, tune, WithFingerprint(1), WithWorkers(w)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("the canceled build returned %v", err) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("the canceled build left its output: %v", err) + } + }) + } + } +} + +// TestStripeGeometry checks the stripe bounds against FastRange32 and the +// split of blocks into stripes. +func TestStripeGeometry(t *testing.T) { + rng := newTestRNG(t) + for i := range 20000 { + n := uint32(rng.Uint64N(math.MaxUint32-1)) + 2 + b := uint32(rng.Uint64N(uint64(n)-1)) + 1 + if i%4 == 0 { + n = uint32(rng.IntN(1000)) + 2 + b = uint32(rng.IntN(int(n)-1)) + 1 + } + p := lowPrefix(b, n) + if intbits.FastRange32(p, n) != b || intbits.FastRange32(p-1, n) != b-1 { + t.Fatalf("lowPrefix(%d, %d) = %#x maps to %d, and one below to %d", b, n, p, intbits.FastRange32(p, n), intbits.FastRange32(p-1, n)) + } + } + + for _, tc := range []struct { + numBlocks uint32 + totalKeys uint64 + stripeKeys int + blocksPerStripe uint32 + numStripes int + lastStripeBlocks uint32 + }{ + {2, 0, 100000, 2, 1, 2}, + {2, 5, 100000, 2, 1, 2}, + {10, 30000, 9000, 3, 4, 1}, + {10, 30000, 1, 1, 10, 1}, + {10, 30000, 1 << 20, 10, 1, 10}, + {1000, 3_000_000, 32 << 10, 10, 100, 10}, + } { + geo, err := newStripeGeometry(tc.numBlocks, tc.totalKeys, tc.stripeKeys) + if err != nil { + t.Fatal(err) + } + _, last := geo.blocks(geo.numStripes - 1) + _, bounded := geo.bound(geo.numStripes - 1) + if geo.blocksPerStripe != tc.blocksPerStripe || geo.numStripes != tc.numStripes || last != tc.lastStripeBlocks || bounded { + t.Fatalf("%d blocks, %d keys, %d-key stripes: %+v, last stripe of %d blocks, bounded %v", tc.numBlocks, tc.totalKeys, tc.stripeKeys, geo, last, bounded) + } + for s := 0; s+1 < geo.numStripes; s++ { + first, count := geo.blocks(s) + next, _ := geo.blocks(s + 1) + bound, bounded := geo.bound(s) + if !bounded || first+count != next || intbits.FastRange32(bound, geo.numBlocks) != next || intbits.FastRange32(bound-1, geo.numBlocks) != next-1 { + t.Fatalf("stripe %d of %+v: blocks %d+%d, next %d, bound %#x", s, geo, first, count, next, bound) + } + } + } +} + +// TestSortedBuildSizing checks the read buffer and stripe sizes against the +// budgets. +func TestSortedBuildSizing(t *testing.T) { + for recordSize := MinKeySize; recordSize <= MinKeySize+maxPayloadSize; recordSize++ { + // The smallest buffer that holds whole records and whole 4096-byte blocks. + unit := directAlign * (recordSize / (recordSize & -recordSize)) + for _, files := range []int{1, 10, 11, 16, 100, 1000, 1024, 1025, 5000} { + size := sortedBuildBufferSize(files, recordSize, 0) + if size%unit != 0 { + t.Fatalf("%d files of %d-byte records: a %d-byte buffer is not a multiple of %d", files, recordSize, size, unit) + } + want := min(max(sortedBuildReadBudget/2/files, sortedBuildMinBuffer), sortedBuildMaxBuffer) + if size-want >= unit || want-size >= unit { + t.Fatalf("%d files of %d-byte records: a %d-byte buffer, want within %d of %d", files, recordSize, size, unit, want) + } + } + if size := sortedBuildBufferSize(1000, recordSize, 1); size != unit { + t.Fatalf("%d-byte records: the smallest buffer is %d bytes, want %d", recordSize, size, unit) + } + } + for _, tc := range []struct{ inFlight, recordSize, want int }{ + {3, 19, sortedBuildMaxStripeKeys}, + {18, 19, sortedBuildStripeBudget / (18 * (19 + 24))}, + {66, 24, sortedBuildStripeBudget / (66 * (24 + 24))}, + {130, 16, sortedBuildMinStripeKeys}, + } { + if got := sortedBuildStripeKeys(tc.inFlight, tc.recordSize, 0); got != tc.want { + t.Fatalf("%d stripes in flight, %d-byte records: %d-key stripes, want %d", tc.inFlight, tc.recordSize, got, tc.want) + } + } + if got := sortedBuildStripeKeys(4, 19, 777); got != 777 { + t.Fatalf("a forced stripe size came out as %d", got) + } +} + +// BenchmarkBuildFromSortedFiles builds 16M keys with 3-byte payloads and +// 1-byte fingerprints from 16 sorted files, against a SortedBuilder fed the +// same keys in block order from memory. 16M keys make 64 stripes, so that +// at 8 workers the last stripes do not dominate. +func BenchmarkBuildFromSortedFiles(b *testing.B) { + const files, perFile = 16, 1_000_000 + dir := b.TempDir() + rng := newTestRNG(b) + paths := make([]string, files) + var all []sortedRecord + for i := range paths { + paths[i] = filepath.Join(dir, fmt.Sprintf("in%02d.sorted", i)) + recs := stratifiedSortedRecords(rng, perFile, 3) + writeSortedFile(b, paths[i], SortedFileConfig{PayloadSize: 3}, recs) + all = append(all, recs...) + } + slices.SortFunc(all, compareSortedRecords) + opts := []BuildOption{WithPayload(3), WithFingerprint(1)} + output := filepath.Join(dir, "bench.idx") + for _, workers := range []int{1, 4, 8} { + b.Run(fmt.Sprintf("files/workers=%d", workers), func(b *testing.B) { + for b.Loop() { + if err := BuildFromSortedFiles(context.Background(), output, paths, append(opts, WithWorkers(workers))...); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(all))*float64(b.N)/b.Elapsed().Seconds(), "keys/s") + }) + b.Run(fmt.Sprintf("builder/workers=%d", workers), func(b *testing.B) { + for b.Loop() { + sb, err := NewSortedBuilder(context.Background(), output, uint64(len(all)), append(opts, WithWorkers(workers))...) + if err != nil { + b.Fatal(err) + } + for _, r := range all { + if err := sb.AddKey(r.key[:], r.payload); err != nil { + b.Fatal(err) + } + } + if err := sb.Finish(); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(all))*float64(b.N)/b.Elapsed().Seconds(), "keys/s") + }) + } +} diff --git a/streamhash-spec.md b/streamhash-spec.md index 1d89110..676f9ea 100644 --- a/streamhash-spec.md +++ b/streamhash-spec.md @@ -451,6 +451,8 @@ Zero bytes pad the header to `RecordsOffset = ceil((20 + MetadataLen) / 4096) × **Validation.** A file's size must be exactly `RecordsOffset + Count × (16 + PayloadSize)`. A reader checks this by dividing the size, never by multiplying `Count`, so a corrupt `Count` cannot overflow the arithmetic and pass. The writer writes `Count` after the records, then syncs. §7.5 lists the errors. +**Building an index from sorted files.** `BuildFromSortedFiles` builds an index from any number of sorted files that share a payload size, without merging them. The index is byte-identical to the one a `SortedBuilder` builds from the same keys in block order (§6.1), because the order of the keys inside a block does not change the index. The build splits the blocks into **stripes** of `B` consecutive blocks holding about `S` keys each, with `S` between 32K and 256K, chosen so the stripes in flight fit in 128 MiB. Stripe `s` covers blocks `[s·B, (s+1)·B)` and so, by §2.2, the prefixes `[ceil(s·B·2^64 / numBlocks), ceil((s+1)·B·2^64 / numBlocks))`; the last stripe has no upper bound. Every file is in key order, so it holds a stripe's keys as one contiguous run. One goroutine gathers each stripe in turn: in each file it finds where the run ends, with a galloping search, and copies the run out. The files are read once, sequentially and directly where the filesystem allows it, through fixed aligned buffers of 64 KiB to 4 MiB, two or three per file within 128 MiB. Each worker (`WithWorkers`) takes whole stripes: it sorts a stripe's records by block with a counting sort, solves the blocks, and hands them to the writer of §6.3, which commits them in block order. With one worker, the calling goroutine gathers, solves and writes each stripe itself. While sorting, a worker checks that each file's run is in order and that every record's block is in the stripe; a record that fails either check means a file out of order (`ErrUnsortedInput`). A duplicate key fails its block (`ErrDuplicateKey`), and the build fails if the records it read do not add up to the headers' counts (`ErrKeyCountMismatch`). + --- ## 4. Algorithm: Bijection @@ -756,6 +758,7 @@ The framework offers three build paths. All produce **byte-identical** output fo - **Sorted** (`NewSortedBuilder`) — keys arrive in block-sorted prefix order; true streaming, lowest RAM, no temp disk. - **Unsorted** (`NewUnsortedBuilder`) — keys arrive in any order; buffered to temp files, then read back in block order. - **Parallel** (`WithWorkers(n)`) — available for both, building blocks concurrently. +- **From sorted files** (`BuildFromSortedFiles`) — keys come from sorted files (§3.9), read in stripes of blocks and never merged; the same output as sorted mode. All paths share the block model of §2: route each key, accumulate a block's keys, hand the block to the algorithm's solver, write the metadata, advance. @@ -909,11 +912,11 @@ An index with `N = 0` keys is not an error: it builds successfully, and every `Q | Error | Condition | |-------|-----------| | `ErrTruncatedFile` | file shorter than its header, or than its `Count` records | -| `ErrKeyCountMismatch` | file longer than its header and `Count` records | +| `ErrKeyCountMismatch` | file longer than its header and `Count` records; or a build read a different number of records than the files count | | `ErrInvalidMagic` | magic ≠ `0x53545346` | | `ErrInvalidVersion` | version ≠ `0x0001` | | `ErrCorruptedIndex` | PayloadSize > 8, or MetadataLen > 65,536 | -| `ErrUnsortedInput` | a key sorts before the key before it, when it is written, merged or read | +| `ErrUnsortedInput` | a key sorts before the key before it, when it is written, merged, read or built from; or a build finds a record outside its stripe | ### 7.6. Integrity and corruption safety