Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ if err := builder.Finish(); err != nil {
}
```

Keys that are already in sorted files (written with `SortedFileWriter`, see [`streamhash-spec.md`](streamhash-spec.md) §3.9) build without being merged: `BuildFromSortedFiles(ctx, "index.idx", paths, opts...)` reads the files once, in stripes of blocks that the workers solve in parallel, and produces the same index. It uses about 256 MiB whatever the number of keys or files.

### Building an index (unsorted input)

```go
Expand Down
10 changes: 3 additions & 7 deletions builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,13 @@ type builder struct {
// Use WithWorkers(N) to enable parallel building with N workers.
// With workers > 1, blocks are built in parallel while maintaining
// the streaming API and O(W × block_size) memory.
func newBuilder(ctx context.Context, output string, totalKeys uint64, opts ...BuildOption) (*builder, error) {
func newBuilder(ctx context.Context, output string, totalKeys uint64, cfg *buildConfig) (*builder, error) {
// totalKeys == 0 is allowed: numBlocks() floors at 2, so a zero-key build
// emits empty blocks and finalizes to a valid empty index.
if totalKeys > maxKeys {
return nil, sherr.ErrTooManyKeys
}

cfg := defaultBuildConfig()
for _, opt := range opts {
opt(cfg)
}
cfg.totalKeys = totalKeys

if cfg.payloadSize < 0 || cfg.payloadSize > maxPayloadSize {
Expand Down Expand Up @@ -389,12 +385,12 @@ type SortedBuilder struct {
// totalKeys may be 0: the result is a valid empty index whose every QueryRank
// returns ErrNotFound.
func NewSortedBuilder(ctx context.Context, output string, totalKeys uint64, opts ...BuildOption) (*SortedBuilder, error) {
b, err := newBuilder(ctx, output, totalKeys, opts...)
b, err := newBuilder(ctx, output, totalKeys, newBuildConfig(opts))
if err != nil {
return nil, err
}
if b.workers > 1 {
b.initParallelWorkers()
b.startPipeline(b.runWorker)
b.pendingEntries = b.getEntrySlice()
}
return &SortedBuilder{b: b}, nil
Expand Down
13 changes: 12 additions & 1 deletion builder_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ type BuildOption func(*buildConfig)
type buildConfig struct {
workers int
payloadSize int
fingerprintSize int // in bytes
payloadSet bool // WithPayload was given
fingerprintSize int // in bytes
globalSeed uint64
userMetadata []byte
unsortedTempDir string // temp directory for unsorted partition files
Expand All @@ -28,6 +29,15 @@ func defaultBuildConfig() *buildConfig {
}
}

// newBuildConfig applies opts to the default configuration.
func newBuildConfig(opts []BuildOption) *buildConfig {
cfg := defaultBuildConfig()
for _, opt := range opts {
opt(cfg)
}
return cfg
}

// WithWorkers sets the number of parallel workers for block building during Finish.
func WithWorkers(n int) BuildOption {
return func(c *buildConfig) {
Expand All @@ -40,6 +50,7 @@ func WithWorkers(n int) BuildOption {
func WithPayload(sizeBytes int) BuildOption {
return func(c *buildConfig) {
c.payloadSize = sizeBytes
c.payloadSet = true
}
}

Expand Down
234 changes: 109 additions & 125 deletions builder_parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,11 @@ type blockResult struct {
err error
}

// initParallelWorkers initializes channels, pools, and starts worker/writer goroutines
// for parallel building. Used by both sorted parallel mode and unsorted parallel mode.
func (b *builder) initParallelWorkers() {
// startPipeline creates the parallel pipeline's channels and pools, and
// starts its writer goroutine and b.workers goroutines running worker, which
// send blockResults on resultChan and return once workerCtx is done. Used by
// sorted parallel mode, unsorted parallel mode and BuildFromSortedFiles.
func (b *builder) startPipeline(worker func() error) {
b.workChan = make(chan blockWork, b.workers*workChanBufferMultiplier)
b.resultChan = make(chan blockResult, b.workers*workChanBufferMultiplier)
b.writerDone = make(chan error, 1)
Expand All @@ -67,13 +69,12 @@ func (b *builder) initParallelWorkers() {
return make([]byte, metadataBufSize)
}

// Start worker goroutines.
// Wrap in explicit cancel so shutdownWorkers can unblock workers stuck on resultChan.
ctx, cancel := context.WithCancel(b.ctx)
b.workerCancel = cancel
b.workerGroup, b.workerCtx = errgroup.WithContext(ctx)
for range b.workers {
b.workerGroup.Go(b.runWorker)
b.workerGroup.Go(worker)
}

// Start writer goroutine
Expand Down Expand Up @@ -164,20 +165,76 @@ func (b *builder) dispatchEmptyBlock(blockID uint32) error {
return b.dispatchBlockWork(blockID, nil, false)
}

// runWorker is the worker goroutine that builds blocks in parallel.
func (b *builder) runWorker() error {
// Create block builder for this worker
blkBuilder, err := newBlockBuilder(b.cfg.algorithm, b.cfg.totalKeys, b.cfg.globalSeed, b.cfg.payloadSize, b.cfg.fingerprintSize)
// blockSolver is a worker's block builder, with the payload buffer it reuses
// across blocks.
type blockSolver struct {
b *builder
blk blockBuilder
payloadsBuf []byte
}

func (b *builder) newBlockSolver() (*blockSolver, error) {
blk, err := newBlockBuilder(b.cfg.algorithm, b.cfg.totalKeys, b.cfg.globalSeed, b.cfg.payloadSize, b.cfg.fingerprintSize)
if err != nil {
return err
return nil, err
}
return &blockSolver{b: b, blk: blk}, nil
}

// loadBlock resets blk and adds entries to it, computing their fingerprints
// from k0 and k1.
func (b *builder) loadBlock(blk blockBuilder, entries []routedEntry) {
fpSize := b.cfg.fingerprintSize
entrySize := b.cfg.payloadSize + fpSize
blk.Reset()
for _, e := range entries {
blk.AddKey(e.k0, e.k1, e.payload, extractFingerprint(e.k0, e.k1, fpSize))
}
}

// Pre-allocate reusable payload buffer (grows as needed, reused across blocks)
// Metadata buffer is NOT reused because it's sent through channel
var payloadsBuf []byte
// build solves the loaded block as blockID, writes its payloads after the
// keysBefore keys of the blocks before it, and returns the result for the
// writer. The metadata buffer comes from the pool; the writer returns it.
func (s *blockSolver) build(blockID uint32, keysBefore uint64) blockResult {
b := s.b
entrySize := b.cfg.payloadSize + b.cfg.fingerprintSize
payloadsNeeded := s.blk.KeysAdded() * entrySize
if cap(s.payloadsBuf) < payloadsNeeded {
s.payloadsBuf = make([]byte, payloadsNeeded)
}
payloads := s.payloadsBuf[:payloadsNeeded]
metadataBuf := b.metadataPool.Get().([]byte)
metadataLen, _, numKeys, err := s.blk.BuildSeparatedInto(metadataBuf, payloads)
result := blockResult{blockID: blockID, metadata: metadataBuf[:metadataLen], numKeys: numKeys, err: err}
if err == nil {
// Hash and write the payloads while they are hot in CPU cache.
result.payloadHash = xxhash.Sum64(payloads)
result.err = b.iw.writePayloadsDirect(payloads, keysBefore*uint64(entrySize))
}
return result
}

// sendResult hands a block's result to the writer and returns the block's
// error, if any, for the worker to return. A block build error must
// propagate to the errgroup, not just ride along in blockResult.err:
// returning it cancels workerCtx, which releases any peer workers blocked
// sending on resultChan after the writer exits on the first error --
// otherwise they (and drainParallelPipeline's workerGroup.Wait) deadlock.
// Wait then surfaces this error to the build.
func (b *builder) sendResult(result blockResult) error {
select {
case b.resultChan <- result:
return result.err
case <-b.workerCtx.Done():
return b.workerCtx.Err()
}
}

// runWorker is the worker goroutine that builds blocks in parallel.
func (b *builder) runWorker() error {
solver, err := b.newBlockSolver()
if err != nil {
return err
}

for work := range b.workChan {
select {
Expand All @@ -190,98 +247,26 @@ func (b *builder) runWorker() error {
default:
}

var metadataBuf []byte
var numKeys int
var payloadHash uint64
var buildErr error

if len(work.entries) == 0 {
// Empty block: reset builder and build empty metadata
blkBuilder.Reset()
metadataBuf = b.metadataPool.Get().([]byte)
var metadataLen int
metadataLen, _, _, buildErr = blkBuilder.BuildSeparatedInto(metadataBuf, nil)
metadataBuf = metadataBuf[:metadataLen]
numKeys = 0
// Empty block contributes hash of empty slice (deterministic)
payloadHash = xxhash.Sum64(nil)
} else {
// Reset and populate the builder, computing fingerprints from k0/k1
blkBuilder.Reset()
for _, e := range work.entries {
blkBuilder.AddKey(e.k0, e.k1, e.payload, extractFingerprint(e.k0, e.k1, fpSize))
}
b.loadBlock(solver.blk, work.entries)

// Signal that entries are no longer being read. This allows the
// flatBuf backing the entries to be safely reused for the next
// partition read (fencing for unsorted parallel mode).
if work.fenceWg != nil {
work.fenceWg.Done()
}

numKeysInBlock := len(work.entries)
payloadsNeeded := numKeysInBlock * entrySize

// Reuse payload buffer (grows if needed)
if cap(payloadsBuf) < payloadsNeeded {
payloadsBuf = make([]byte, payloadsNeeded)
} else {
payloadsBuf = payloadsBuf[:payloadsNeeded]
}

// Get metadata buffer from pool (returned to pool by writer after use)
metadataBuf = b.metadataPool.Get().([]byte)

var metadataLen int
metadataLen, _, numKeys, buildErr = blkBuilder.BuildSeparatedInto(metadataBuf, payloadsBuf)
if buildErr == nil {
metadataBuf = metadataBuf[:metadataLen]

// Compute payload hash and write payloads while data is hot in CPU cache
if payloadsNeeded > 0 {
// Hash payload buffer before writing via pwrite
payloadHash = xxhash.Sum64(payloadsBuf[:payloadsNeeded])
// Write payloads via pwrite, then buffer can be reused
payloadOffset := work.keysBefore * uint64(entrySize)
if werr := b.iw.writePayloadsDirect(payloadsBuf, payloadOffset); werr != nil {
buildErr = werr
}
} else {
// MPHF-only mode: no payloads, hash empty slice
payloadHash = xxhash.Sum64(nil)
}
}

// Return entry slice to pool only if it was pool-allocated.
// Unsorted mode dispatches flatBuf-backed slices with pooled=false
// and uses fenceWg for lifetime management instead.
if work.pooled {
b.putEntrySlice(work.entries)
}
// Signal that entries are no longer being read. This allows the
// flatBuf backing the entries to be safely reused for the next
// partition read (fencing for unsorted parallel mode).
if work.fenceWg != nil {
work.fenceWg.Done()
}

// Send result to writer (metadata + payload hash)
select {
case b.resultChan <- blockResult{
blockID: work.blockID,
metadata: metadataBuf,
numKeys: numKeys,
payloadHash: payloadHash,
err: buildErr,
}:
case <-b.workerCtx.Done():
return b.workerCtx.Err()
result := solver.build(work.blockID, work.keysBefore)

// Return entry slice to pool only if it was pool-allocated.
// Unsorted mode dispatches flatBuf-backed slices with pooled=false
// and uses fenceWg for lifetime management instead.
if work.pooled {
b.putEntrySlice(work.entries)
}

// A block build error must propagate to the errgroup, not just ride
// along in blockResult.err. Returning it cancels workerCtx, which
// releases any peer workers blocked sending on resultChan after the
// writer exits on the first error -- otherwise they (and
// drainParallelPipeline's workerGroup.Wait) deadlock. Wait then surfaces
// this error to Finish(). Covers both the sorted and unsorted-parallel
// paths, which share this worker pool and drainParallelPipeline.
if buildErr != nil {
return buildErr
if err := b.sendResult(result); err != nil {
return err
}
}

Expand Down Expand Up @@ -396,37 +381,36 @@ func (b *builder) finishParallel() error {
return b.drainParallelPipeline()
}

// drainParallelPipeline closes the work channel, waits for workers and writer
// to finish, then finalizes the index.
// drainParallelPipeline closes the work channel, waits for the workers and
// the writer to finish, then finalizes the index, unless one of them failed
// or the pipeline was canceled.
func (b *builder) drainParallelPipeline() error {
// Close work channel to signal workers we're done
close(b.workChan)
b.workersShutDown.Store(true) // Prevents double-close in Close()/shutdownWorkers()

// Wait for all workers to finish
if werr := b.workerGroup.Wait(); werr != nil {
close(b.resultChan)
// Prefer the writer's error: when the writer initiates the failure it
// cancels workerCtx, so workerGroup.Wait returns a bare context.Canceled
// while the real cause (e.g. an ENOSPC metadata write, or the block
// buildErr) sits in writerDone. Fall back to the worker error only if the
// writer reported none (e.g. a worker setup failure).
cause := <-b.writerDone
if cause == nil {
cause = fmt.Errorf("worker error: %w", werr)
}
return errors.Join(cause, b.cleanup())
}

// Close result channel to signal writer we're done
// Read the cancellation before Wait, which cancels workerCtx itself.
canceled := b.workerCtx.Err()
werr := b.workerGroup.Wait()
close(b.resultChan)

// Wait for writer to finish
if err := <-b.writerDone; err != nil {
primaryErr := fmt.Errorf("writer error: %w", err)
return errors.Join(primaryErr, b.cleanup())
cause := <-b.writerDone
// A block build error reaches writerDone through failWriter and is the
// worker's error too; the writer's copy is the cause as it is, and werr
// is then a bare context.Canceled. The prefixes name where an error of
// the pipeline's own arose.
switch {
case cause != nil && werr == nil:
cause = fmt.Errorf("writer error: %w", cause)
case cause == nil && werr != nil:
cause = fmt.Errorf("worker error: %w", werr)
case cause == nil:
// A canceled pipeline never finalizes: its producer may have stopped
// before dispatching every block.
cause = canceled
}
if cause != nil {
return errors.Join(cause, b.cleanup())
}

return b.finalizeIndex()
}

Expand Down
2 changes: 1 addition & 1 deletion builder_unsorted.go
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ type UnsortedBuilder struct {
// totalKeys may be 0: the result is a valid empty index whose every QueryRank
// returns ErrNotFound.
func NewUnsortedBuilder(ctx context.Context, output string, totalKeys uint64, tempDir string, opts ...BuildOption) (*UnsortedBuilder, error) {
b, err := newBuilder(ctx, output, totalKeys, opts...)
b, err := newBuilder(ctx, output, totalKeys, newBuildConfig(opts))
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions builder_unsorted_finish.go
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ func (ub *UnsortedBuilder) finishUnsortedFastPath() error {
}

if b.workers > 1 {
b.initParallelWorkers()
b.startPipeline(b.runWorker)
}

fpSize := b.cfg.fingerprintSize
Expand Down Expand Up @@ -512,7 +512,7 @@ func (ub *UnsortedBuilder) finishUnsortedSingleThreaded() error {
// per-partition channels.
func (ub *UnsortedBuilder) finishUnsortedParallel() error {
b := ub.b
b.initParallelWorkers()
b.startPipeline(b.runWorker)

u := ub.unsortedBuf
numParts := u.numPartitions
Expand Down
Loading
Loading