Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@
// - Public API: builder.go (NewSortedBuilder), builder_unsorted.go (NewUnsortedBuilder), index.go (Open, QueryRank, QueryBatch, PayloadIndex)
// - Configuration: builder_options.go (BuildOption, With* functions)
// - Serialization: header.go (header, footer, ramIndexEntry), index_writer.go
// - Sorted files: sorted_file.go (SortedFileWriter, ReadSortedFileHeader, OpenSortedFile), sorted_merge.go (MergeSortedFiles)
// - Key routing & hashing: key.go (fingerprint extraction, payload packing), prehash.go (PreHash); block routing is FastRange32 over the big-endian prefix, inline in builder.go/index.go
// - Algorithm dispatch: algorithm.go (blockBuilder/blockDecoder interfaces, factory functions)
// - Block algorithms: internal/bijection/ (EF/GR), internal/ptrhash/ (Cuckoo)
// - Platform: platform_*.go (OS-specific: fallocate), madvise_*.go (mmap read-around advice)
// - Platform: platform_*.go (OS-specific: fallocate), madvise_*.go (mmap read-around advice), odirect*.go (direct reads)
package streamhash
6 changes: 3 additions & 3 deletions internal/sherr/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ var (
ErrIndistinguishableHashes = errors.New("streamhash: indistinguishable hashes in bucket - retry with different globalSeed")
)

// Index errors
// Index and sorted-file errors
var (
ErrInvalidMagic = errors.New("streamhash: invalid magic number")
ErrInvalidVersion = errors.New("streamhash: unsupported version")
ErrChecksumFailed = errors.New("streamhash: file checksum verification failed")
ErrTruncatedFile = errors.New("streamhash: index file is truncated")
ErrCorruptedIndex = errors.New("streamhash: index data is corrupted")
ErrTruncatedFile = errors.New("streamhash: file is truncated")
ErrCorruptedIndex = errors.New("streamhash: data is corrupted")
)

// Query errors
Expand Down
16 changes: 16 additions & 0 deletions odirect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package streamhash

import "unsafe"

// directAlign is the alignment direct reads need of the buffer's address,
// the file offset and the length.
const directAlign = 4096

// alignedBuffer returns a size-byte slice aligned to directAlign. Go's GC does
// not move it, so the alignment holds.
func alignedBuffer(size int) []byte {
backing := make([]byte, size+directAlign)
misalign := int(uintptr(unsafe.Pointer(&backing[0])) % directAlign)
off := (directAlign - misalign) % directAlign
return backing[off : off+size : off+size]
}
21 changes: 21 additions & 0 deletions odirect_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//go:build linux

package streamhash

import (
"errors"
"os"

"golang.org/x/sys/unix"
)

// openDirect opens path for reading with O_DIRECT, so reading a file once
// does not fill the page cache. A filesystem that refuses O_DIRECT gets a
// plain open; reads into aligned buffers at aligned offsets work either way.
func openDirect(path string) (*os.File, error) {
f, err := os.OpenFile(path, os.O_RDONLY|unix.O_DIRECT, 0)
if errors.Is(err, unix.EINVAL) {
return os.Open(path)
}
return f, err
}
11 changes: 11 additions & 0 deletions odirect_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
//go:build !linux

package streamhash

import "os"

// openDirect opens path for reading. Off Linux there is no O_DIRECT, so the
// reads go through the page cache.
func openDirect(path string) (*os.File, error) {
return os.Open(path)
}
2 changes: 1 addition & 1 deletion sentinels.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ var (
ErrBlockOverflow = sherr.ErrBlockOverflow
)

// Index errors.
// Index and sorted-file errors.
var (
ErrInvalidMagic = sherr.ErrInvalidMagic
ErrInvalidVersion = sherr.ErrInvalidVersion
Expand Down
Loading
Loading