diff --git a/README.md b/README.md index 9bbeb11..ba4c286 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,27 @@ Every package takes `WithShards`. `twoq` and `s3fifo` also have options for tuni queues ([what that is worth for `s3fifo`](bench/results/tuning.md)); `tinylfu` tunes its own. +### Batches + +`sieve`, `s3fifo` and `tinylfu` also have `GetMany(keys, values, found)`, which does what +`Get` does for each key. For up to 32 keys at a time it loads the memory that the lookups +start with before it looks any key up, so the CPU fetches it in parallel. + +| Time per key, `GetMany` against a `Get` loop | 1 CPU | 8 CPUs | +|:---|---:|---:| +| 1M `int` entries, 32 keys per call | −48 to −62% | −47 to −53% | +| 1M `int` entries, 8 keys per call | −36 to −49% | −27 to −34% | +| 1M entries with 23-byte string keys, 32 keys per call | −66 to −68% | −56 to −58% | +| Zipf read-through with room for 1M, 32 keys per call | −49 to −55% | −48 to −51% | +| 16K `int` entries, 32 keys per call | −2 to +3% | +1 to +9% | +| 1M `int` entries, 1 key per call | +40 to +86% | +32 to +80% | + +- Ranges over the three caches of medians of three runs of `BenchmarkGetMany` in each + package, on an Apple M3 Max, keys picked at random. The times themselves vary more + between runs than these ratios. +- For a single key, use `Get`. +- `stats.Wrap` passes `GetMany` on to the cache it wraps, and counts a hit or a miss per key. + ### Statistics [`stats.Wrap`](https://pkg.go.dev/github.com/floatdrop/evict/stats) counts the hits, misses diff --git a/internal/cachetest/lookups.go b/internal/cachetest/lookups.go new file mode 100644 index 0000000..7c44155 --- /dev/null +++ b/internal/cachetest/lookups.go @@ -0,0 +1,114 @@ +package cachetest + +import ( + "math/rand/v2" + "strconv" + "strings" +) + +// The functions below make the keys for benchmarks of lookups in caches that +// are too large for the CPU's caches. A sequence of keys to look up lies in +// memory one after the other, so that reading the next key costs next to +// nothing and what is measured is the cache. A string key in a sequence is a +// copy of the key in the cache, as a key that has just come in would be: its +// bytes are at hand, and comparing it with the key in the cache loads those of +// the other one. + +// SeqLen is the length of a sequence of lookups: long enough that a benchmark +// does not come back to the same keys while they are still in the CPU's +// caches. It is a power of two. +const SeqLen = 1 << 22 + +// width is the length of a key: that of "user:profile:1000000000", as in the +// bench module. +const width = 23 + +// names returns n keys of width bytes, cut out of one string. +func names(n int) []string { + var b strings.Builder + b.Grow(width * n) + for i := range n { + b.WriteString("user:profile:") + b.WriteString(strconv.Itoa(1_000_000_000 + i)) + } + all := b.String() + keys := make([]string, n) + for i := range keys { + keys[i] = all[width*i : width*(i+1)] + } + return keys +} + +// copies returns a copy of keys[k] for every k in picks, one after the other. +func copies(keys []string, picks []uint32) []string { + var b strings.Builder + b.Grow(width * len(picks)) + for _, k := range picks { + b.WriteString(keys[k]) + } + all := b.String() + out := make([]string, len(picks)) + for i := range out { + out[i] = all[width*i : width*(i+1)] + } + return out +} + +// uniform returns SeqLen numbers picked at random from [0, n). +func uniform(n int) []uint32 { + rng := rand.New(rand.NewPCG(3, 4)) + picks := make([]uint32, SeqLen) + for i := range picks { + picks[i] = uint32(rng.IntN(n)) + } + return picks +} + +// UniformInts returns the keys 0 to n-1, to fill a cache with, and a sequence +// of lookups of them picked at random. +func UniformInts(n int) (keys, seq []int) { + keys = make([]int, n) + for i := range keys { + keys[i] = i + } + seq = make([]int, SeqLen) + for i, k := range uniform(n) { + seq[i] = int(k) + } + return keys, seq +} + +// UniformStrings is UniformInts with keys of 23 bytes. +func UniformStrings(n int) (keys, seq []string) { + keys = names(n) + return keys, copies(keys, uniform(n)) +} + +// ZipfStrings returns requests as the bench module's ReadThrough makes them +// for a cache of the given capacity: for four times as many keys as it has +// room for, with a popularity that follows a Zipf distribution of skew 1.01. +// The cache is to be read through with the first sixteen requests per entry +// that it has room for, a Get and, if that misses, a Set, so that it holds +// what its policy keeps; seq is the sequence of lookups that follows. +func ZipfStrings(capacity int) (warm, seq []string) { + n := 4 * capacity + all := names(n) + keys := make([]string, n) + // The popular keys are spread over the key space, as they would be. + for i, p := range rand.New(rand.NewPCG(5, 6)).Perm(n) { + keys[i] = all[p] + } + z := rand.NewZipf(rand.New(rand.NewPCG(7, 8)), 1.01, 1, uint64(n-1)) + draw := func(m int) []uint32 { + picks := make([]uint32, m) + for i := range picks { + picks[i] = uint32(z.Uint64()) + } + return picks + } + warm = make([]string, 16*capacity) + for i, k := range draw(len(warm)) { + warm[i] = keys[k] + } + return warm, copies(keys, draw(SeqLen)) +} diff --git a/internal/cslab/cslab.go b/internal/cslab/cslab.go index 40a9642..bf1cdf7 100644 --- a/internal/cslab/cslab.go +++ b/internal/cslab/cslab.go @@ -229,6 +229,43 @@ func (s *Slab[K, V]) Cap() int { return int(s.capacity) } // for writers. func (s *Slab[K, V]) Hash(idx uint32) uint32 { return s.Links[idx].hash } +// Batch is the number of keys whose memory a reader that looks up several +// keys at once touches before it looks any of them up; see [Slab.Touch]. +// Batches of 16 did as well with int keys and about 15% worse with string +// keys, and batches of 8 worse with both. +const Batch = 32 + +// MinBatch is the fewest keys for which touching pays: with fewer, the reader +// should look them up one by one. A batch of 4 keys took about as long as +// looking them up one by one, a batch of 2 about 1.6 times as long, and a +// batch of 1 more than twice as long. +const MinBatch = 4 + +// Touch loads the slot of the index at which a lookup of hash starts, so that +// it is in the CPU's cache when the lookup comes, and returns it. It is for +// readers that look up several keys: they touch the slots of all of them +// first, then the nodes with [Slab.TouchNode], and the CPU fetches what they +// touch from memory at the same time instead of one after another. That +// takes loads that do not depend on each other and come one right after the +// other; a reader that touches one key ahead of its lookups gains nothing. +// +// The word that Touch returns may be out of date by the time it is used. It +// is good for TouchNode and nothing else. +func (s *Slab[K, V]) Touch(hash uint32) uint64 { + return atomic.LoadUint64(&s.slots[uint32(uint64(hash)*uint64(s.size)>>32)]) +} + +// TouchNode loads the node that word, which [Slab.Touch] returned for hash, +// points to, if it has the hash: the cache line with its Meta, which the Key +// and Value usually share. A word that is out of date points to a node all +// the same, since nodes are never taken away, so the worst it can do is load +// the wrong one. +func (s *Slab[K, V]) TouchNode(hash uint32, word uint64) { + if word>>32 == uint64(hash) { + atomic.LoadUint32(&s.Nodes[uint32(word)].meta) + } +} + // Lookup returns the value that is stored under key. The hash must be the one // the key was inserted with. Before it returns the value, Lookup counts the // read in the Meta of the node, unless the Meta has reached limit already: a @@ -867,6 +904,8 @@ func (s *Slab[K, V]) Unlink(idx uint32) { // see the note in package lru. Instantiating the methods here sees to that. var ( _ = (*Slab[int, int]).Lookup + _ = (*Slab[int, int]).Touch + _ = (*Slab[int, int]).TouchNode _ = (*Slab[int, int]).Find _ = (*Slab[int, int]).Insert _ = (*Slab[int, int]).Replace diff --git a/internal/inlinecheck/inline_test.go b/internal/inlinecheck/inline_test.go index d0fd5d4..eebf4c6 100644 --- a/internal/inlinecheck/inline_test.go +++ b/internal/inlinecheck/inline_test.go @@ -25,13 +25,13 @@ import ( ) // hot matches the functions that every operation on a cache goes through. -var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` + +var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|GetMany|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` + `/c?slab\.\(\*Slab\[.*\]\)\.(Find|Lookup|Insert|Replace|Remove)$`) // slow matches the calls that must not show up in them: functions that are // small enough to be inlined, and are, as long as the compiler knows them. var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` + - `slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred)$|` + + `slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred|Touch|TouchNode)$|` + `cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$|cslab\.\(\*Slot\)\.Unpin$|/s3fifo\.(tagOf|ghostOf)$|` + `/tinylfu\.(tagOf|kept|keep|printOf|printIn)$|/tinylfu\.\(\*climber\)\.(hit|miss)$|` + `/stats\.\(\*Cache\[.*\]\)\.stripe$`) @@ -45,9 +45,9 @@ var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` + // it builds in. A lock is taken only when a read cannot be sure, and calls // to it are fine. var ( - read = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`) + read = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|GetMany|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`) readSlow = regexp.MustCompile(`^sync/atomic\.|^github\.com/floatdrop/evict/[^\[]*$|` + - `cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`) + `cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred|Touch|TouchNode)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`) ) func TestHotPathsAreInlined(t *testing.T) { @@ -145,13 +145,25 @@ func TestHotPathsAreInlined(t *testing.T) { } } } + for _, pkg := range []string{"sieve", "s3fifo", "tinylfu", "stats"} { + found := false + for name := range seen { + found = found || strings.Contains(name, "/"+pkg+".(*Cache[") && strings.HasSuffix(name, ").GetMany") + } + if !found { + t.Errorf("the compiler's output has no code for %s.Cache.GetMany", pkg) + } + } for _, want := range [][2]string{ {"sieve.(*Cache[", "Get"}, {"sieve.(*Cache[", "Peek"}, + {"sieve.(*Cache[", "GetMany"}, {"s3fifo.(*Cache[", "Get"}, {"s3fifo.(*Cache[", "Peek"}, + {"s3fifo.(*Cache[", "GetMany"}, {"tinylfu.(*Cache[", "Get"}, {"tinylfu.(*Cache[", "Peek"}, + {"tinylfu.(*Cache[", "GetMany"}, {"cslab.(*Slab[", "Lookup"}, } { found := false diff --git a/internal/inlinecheck/probe/s3fifo/probe.go b/internal/inlinecheck/probe/s3fifo/probe.go index 348d460..b5e0016 100644 --- a/internal/inlinecheck/probe/s3fifo/probe.go +++ b/internal/inlinecheck/probe/s3fifo/probe.go @@ -17,5 +17,8 @@ func Use(c *s3fifo.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/sieve/probe.go b/internal/inlinecheck/probe/sieve/probe.go index 42a0092..5383786 100644 --- a/internal/inlinecheck/probe/sieve/probe.go +++ b/internal/inlinecheck/probe/sieve/probe.go @@ -17,5 +17,8 @@ func Use(c *sieve.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/stats/probe.go b/internal/inlinecheck/probe/stats/probe.go index d9f6166..bcee219 100644 --- a/internal/inlinecheck/probe/stats/probe.go +++ b/internal/inlinecheck/probe/stats/probe.go @@ -17,5 +17,8 @@ func Use(c *stats.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/statswrap/probe.go b/internal/inlinecheck/probe/statswrap/probe.go index 9a81900..8fa25fa 100644 --- a/internal/inlinecheck/probe/statswrap/probe.go +++ b/internal/inlinecheck/probe/statswrap/probe.go @@ -13,5 +13,8 @@ type Cache[K comparable, V any] struct{ C *stats.Cache[K, V] } // Get reads from the counted cache. func (c Cache[K, V]) Get(key K) (V, bool) { return c.C.Get(key) } +// GetMany reads several keys from the counted cache. +func (c Cache[K, V]) GetMany(keys []K, values []V, found []bool) { c.C.GetMany(keys, values, found) } + // Set writes to the counted cache. func (c Cache[K, V]) Set(key K, value V) { c.C.Set(key, value) } diff --git a/internal/inlinecheck/probe/statswrapuser/probe.go b/internal/inlinecheck/probe/statswrapuser/probe.go index 30fe528..24e802a 100644 --- a/internal/inlinecheck/probe/statswrapuser/probe.go +++ b/internal/inlinecheck/probe/statswrapuser/probe.go @@ -14,5 +14,8 @@ type Value struct{ Size, Hits int } // Use keeps the compiler from discarding the methods of the cache. func Use(c wrap.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/tinylfu/probe.go b/internal/inlinecheck/probe/tinylfu/probe.go index c0cabac..ae3a9cb 100644 --- a/internal/inlinecheck/probe/tinylfu/probe.go +++ b/internal/inlinecheck/probe/tinylfu/probe.go @@ -17,5 +17,8 @@ func Use(c *tinylfu.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/wrap/wrap.go b/internal/inlinecheck/probe/wrap/wrap.go index 0176f61..8e13428 100644 --- a/internal/inlinecheck/probe/wrap/wrap.go +++ b/internal/inlinecheck/probe/wrap/wrap.go @@ -32,4 +32,19 @@ func (c Caches[K, V]) Read(key K) (found int) { return found } +// ReadMany looks up keys in every cache, all of them together, and returns +// how often they were found. +func (c Caches[K, V]) ReadMany(keys []K) (found int) { + values, ok := make([]V, len(keys)), make([]bool, len(keys)) + for _, getMany := range []func([]K, []V, []bool){c.Sieve.GetMany, c.S3FIFO.GetMany, c.TinyLFU.GetMany} { + getMany(keys, values, ok) + for _, ok := range ok { + if ok { + found++ + } + } + } + return found +} + func has[V any](_ V, ok bool) bool { return ok } diff --git a/internal/inlinecheck/probe/wrapuser/wrapuser.go b/internal/inlinecheck/probe/wrapuser/wrapuser.go index 8e09ce1..966445b 100644 --- a/internal/inlinecheck/probe/wrapuser/wrapuser.go +++ b/internal/inlinecheck/probe/wrapuser/wrapuser.go @@ -11,4 +11,4 @@ type Key struct{ Tenant, ID string } type Value struct{ Size, Hits int } // Use keeps the compiler from discarding the methods of the caches. -func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) } +func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) + c.ReadMany([]Key{k, k}) } diff --git a/s3fifo/export_test.go b/s3fifo/export_test.go index 439aa52..fcdf76a 100644 --- a/s3fifo/export_test.go +++ b/s3fifo/export_test.go @@ -75,3 +75,16 @@ func Twins[V any](c *Cache[int, V]) (first, second int) { seen[hash] = key } } + +// Reads returns the reads that the entry for key has counted, and whether +// there is one. +func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + if idx := s.slab.Find(uint32(h), key); idx != 0 { + return s.slab.Nodes[idx].Meta(), true + } + return 0, false +} diff --git a/s3fifo/getmany_test.go b/s3fifo/getmany_test.go new file mode 100644 index 0000000..35a8530 --- /dev/null +++ b/s3fifo/getmany_test.go @@ -0,0 +1,340 @@ +package s3fifo_test + +import ( + "fmt" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/s3fifo" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := s3fifo.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := s3fifo.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + viaMany := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + for k := 2 * capacity; k < 3*capacity; k++ { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + getMain, getSmall := viaGet.Queues() + manyMain, manySmall := viaMany.Queues() + if !slices.Equal(getMain, manyMain) || !slices.Equal(getSmall, manySmall) { + t.Errorf("after evictions, the queues differ:\nGet: %v %v\nGetMany: %v %v", getMain, getSmall, manyMain, manySmall) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := s3fifo.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := s3fifo.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*s3fifo.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*s3fifo.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*s3fifo.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*s3fifo.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := s3fifo.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *s3fifo.Cache[K, int], lookups []K) { + c = s3fifo.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *s3fifo.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/s3fifo/s3fifo.go b/s3fifo/s3fifo.go index 2fb31dd..6dbee86 100644 --- a/s3fifo/s3fifo.go +++ b/s3fifo/s3fifo.go @@ -116,12 +116,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -380,6 +380,78 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 51% less time than with Get with 32 keys per call on one CPU, and +// 48% less on eight; with 8 keys per call, 36% and 32% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 9% of Get. For a +// single key, call Get: through GetMany it took 40% and 32% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("s3fifo: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], maxFrequency, false) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(min(node.Meta()+1, maxFrequency)) + value, ok = node.Value, true + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without counting the read. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -662,6 +734,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/s3fifo/s3fifo_test.go b/s3fifo/s3fifo_test.go index 54935ab..6146a02 100644 --- a/s3fifo/s3fifo_test.go +++ b/s3fifo/s3fifo_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/s3fifo" ) @@ -1004,6 +1005,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *s3fifo.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *s3fifo.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *s3fifo.Cache[int, *cachetest.Blob], key int)) { diff --git a/sieve/export_test.go b/sieve/export_test.go index 30f2945..042f3e2 100644 --- a/sieve/export_test.go +++ b/sieve/export_test.go @@ -58,3 +58,16 @@ func Twins[V any](c *Cache[int, V]) (first, second int) { seen[hash] = key } } + +// Reads returns the mark of the entry for key, which is visited if it was read +// since the hand last came by, and whether there is one. +func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + if idx := s.slab.Find(uint32(h), key); idx != 0 { + return s.slab.Nodes[idx].Meta(), true + } + return 0, false +} diff --git a/sieve/getmany_test.go b/sieve/getmany_test.go new file mode 100644 index 0000000..2ab1e8f --- /dev/null +++ b/sieve/getmany_test.go @@ -0,0 +1,339 @@ +package sieve_test + +import ( + "fmt" + "maps" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/sieve" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := sieve.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := sieve.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := sieve.New[int, int](capacity, sieve.WithShards(1)) + viaMany := sieve.New[int, int](capacity, sieve.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + for k := 2 * capacity; k < 3*capacity; k++ { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + if a, b := slices.Sorted(maps.Keys(maps.Collect(viaGet.All()))), slices.Sorted(maps.Keys(maps.Collect(viaMany.All()))); !slices.Equal(a, b) { + t.Errorf("after evictions, the caches hold different keys:\nGet: %v\nGetMany: %v", a, b) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := sieve.New[int, int](capacity, sieve.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := sieve.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := sieve.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*sieve.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*sieve.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*sieve.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*sieve.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := sieve.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *sieve.Cache[K, int], lookups []K) { + c = sieve.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *sieve.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/sieve/sieve.go b/sieve/sieve.go index bf98b46..784d80a 100644 --- a/sieve/sieve.go +++ b/sieve/sieve.go @@ -79,12 +79,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -212,6 +212,78 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 62% less time than with Get with 32 keys per call on one CPU, and +// 53% less on eight; with 8 keys per call, 49% and 27% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 2% of Get. For a +// single key, call Get: through GetMany it took 42% and 36% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("sieve: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], visited, false) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(visited) + value, ok = node.Value, true + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without marking the entry as used. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -431,6 +503,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/sieve/sieve_test.go b/sieve/sieve_test.go index 3767d3c..846c85a 100644 --- a/sieve/sieve_test.go +++ b/sieve/sieve_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/sieve" ) @@ -506,6 +507,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *sieve.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *sieve.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *sieve.Cache[int, *cachetest.Blob], key int)) { diff --git a/stats/getmany_test.go b/stats/getmany_test.go new file mode 100644 index 0000000..aa71b41 --- /dev/null +++ b/stats/getmany_test.go @@ -0,0 +1,138 @@ +package stats_test + +import ( + "math/rand/v2" + "slices" + "testing" + + "github.com/floatdrop/evict" + "github.com/floatdrop/evict/lru" + "github.com/floatdrop/evict/s3fifo" + "github.com/floatdrop/evict/stats" +) + +// batcher is a cache with a GetMany of its own, which says how often it was +// called. +type batcher struct { + evict.Cache[int, int] + calls int +} + +func (b *batcher) GetMany(keys, values []int, found []bool) { + b.calls++ + for i, k := range keys { + values[i], found[i] = b.Get(k) + } +} + +// GetMany has to find what Peek finds, and count a hit or a miss for every +// key, whether the cache has a GetMany of its own or not. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 500 + for name, cache := range map[string]evict.Cache[int, int]{ + "lru": lru.New[int, int](capacity), + "s3fifo": s3fifo.New[int, int](capacity), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + c := stats.Wrap(cache) + for k := range capacity / 2 { + c.Set(k, -k) + } + + rng := rand.New(rand.NewPCG(1, 2)) + var hits, misses uint64 + for _, n := range []int{0, 1, 3, 4, 31, 32, 33, 100} { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + v, ok := c.Peek(k) + if values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + if ok { + hits++ + } else { + misses++ + } + } + } + + if s := c.Stats(); s.Hits != hits || s.Misses != misses { + t.Errorf("%d hits and %d misses counted, want %d and %d", s.Hits, s.Misses, hits, misses) + } + }) + } +} + +// A cache that has a GetMany of its own is asked with it. +func TestGetManyForwards(t *testing.T) { + t.Parallel() + + b := &batcher{Cache: lru.New[int, int](10)} + c := stats.Wrap[int, int](b) + c.Set(1, 1) + values, found := make([]int, 2), make([]bool, 2) + c.GetMany([]int{1, 2}, values, found) + + if b.calls != 1 { + t.Errorf("the cache's GetMany was called %d times, want once", b.calls) + } + if s := c.Stats(); s.Hits != 1 || s.Misses != 1 { + t.Errorf("%d hits and %d misses counted, want 1 and 1", s.Hits, s.Misses) + } +} + +// Slices that are too short for the keys make GetMany panic before it looks +// anything up or counts it. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := stats.Wrap(lru.New[int, int](10)) + c.Set(1, 1) + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany([]int{1, 2, 3}, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + if s := c.Stats(); s.Hits != 0 || s.Misses != 0 { + t.Errorf("%d hits and %d misses counted before the panic", s.Hits, s.Misses) + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + for name, cache := range map[string]evict.Cache[int, int]{ + "lru": lru.New[int, int](100), + "s3fifo": s3fifo.New[int, int](100), + } { + t.Run(name, func(t *testing.T) { + c := stats.Wrap(cache) + keys := make([]int, 40) + for i := range keys { + keys[i] = i + c.Set(i, i) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } + }) + } +} diff --git a/stats/stats.go b/stats/stats.go index dfb11d7..bd6fdc2 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -11,6 +11,7 @@ // What is counted: // // - Get: a hit if it finds the key, a miss if it does not. +// - GetMany: the same for each key. // - Set: an eviction if it pushed out the entry of another key to make // room, or if the cache has no capacity and handed the new entry back. // Replacing the entry of a key is not an eviction. @@ -57,8 +58,8 @@ const cacheLine = 128 // of one snapshot may be from slightly different moments. A count is never // less than that of a snapshot taken before it. type Snapshot struct { - // Hits and Misses count the calls to Get that found their key, and - // those that did not. + // Hits and Misses count the keys that Get and GetMany found, and those + // that they did not. Hits, Misses uint64 // Evictions counts the entries that Set pushed out to make room, and @@ -70,8 +71,8 @@ type Snapshot struct { Len, Cap int } -// HitRatio returns the share of calls to Get that found their key, from 0 to -// 1, or 0 if there were none. +// HitRatio returns the share of the keys looked up with Get and GetMany that +// were found, from 0 to 1, or 0 if there were none. func (s Snapshot) HitRatio() float64 { if s.Hits+s.Misses == 0 { return 0 @@ -96,11 +97,18 @@ func (s Snapshot) Sub(prev Snapshot) Snapshot { // the cache it wraps are, as they are for every cache in this module. type Cache[K comparable, V any] struct { cache evict.Cache[K, V] + many manyGetter[K, V] // cache, if it has a GetMany stripes []stripe shift uint8 // 64 minus log2 of the number of stripes zero bool // the cache has no capacity, and hands every entry back } +// manyGetter is a cache that looks up several keys at once, as sieve, s3fifo +// and tinylfu do. +type manyGetter[K comparable, V any] interface { + GetMany(keys []K, values []V, found []bool) +} + // A stripe holds the counts of the goroutines that pick it, in a cache line // of its own. // @@ -127,8 +135,10 @@ func Wrap[K comparable, V any](cache evict.Cache[K, V]) *Cache[K, V] { // Sixteen stripes per CPU make it unlikely that two goroutines that run // at the same time pick the same one. n := 1 << bits.Len(uint(16*runtime.GOMAXPROCS(0)-1)) + many, _ := cache.(manyGetter[K, V]) return &Cache[K, V]{ cache: cache, + many: many, stripes: make([]stripe, n), shift: uint8(64 - bits.TrailingZeros(uint(n))), zero: cache.Cap() == 0, @@ -162,6 +172,40 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany looks up each of keys, stores the value it finds for keys[i] in +// values[i] and whether it found one in found[i], and counts a hit or a miss +// for each key. It panics if values or found is shorter than keys. +// +// If the cache has a GetMany of its own, as sieve, s3fifo and tinylfu do, +// GetMany uses it: for a cache that is too large for the CPU's caches it is +// faster than Get for each key. Otherwise it calls Get for each key. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + if len(values) < len(keys) || len(found) < len(keys) { + panic("stats: GetMany: values and found must be at least as long as keys") + } + if c.many != nil { + c.many.GetMany(keys, values, found) + } else { + for i, key := range keys { + values[i], found[i] = c.cache.Get(key) + } + } + + hits := 0 + for _, ok := range found[:len(keys)] { + if ok { + hits++ + } + } + s := c.stripe() + if hits > 0 { + atomic.AddUint64(&s.hits, uint64(hits)) + } + if misses := len(keys) - hits; misses > 0 { + atomic.AddUint64(&s.misses, uint64(misses)) + } +} + // Peek returns the value stored under key, and counts nothing. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { return c.cache.Peek(key) } diff --git a/tinylfu/export_test.go b/tinylfu/export_test.go index 9e232c6..18e95e2 100644 --- a/tinylfu/export_test.go +++ b/tinylfu/export_test.go @@ -142,3 +142,17 @@ func Apart[V any](c *Cache[int, V], n int, others ...int) []int { } return keys } + +// Hits returns the hits that the window of the cache has been told of or will +// be: those that readers counted in their slots, and those that shards counted +// under their locks. +func (c *Cache[K, V]) Hits() uint64 { + hits := c.window.readers.Hits() + c.window.hits.Load() + for i := range c.shards { + s := &c.shards[i] + s.mu.Lock() + hits += s.hits + s.mu.Unlock() + } + return hits +} diff --git a/tinylfu/getmany_test.go b/tinylfu/getmany_test.go new file mode 100644 index 0000000..1f51f86 --- /dev/null +++ b/tinylfu/getmany_test.go @@ -0,0 +1,337 @@ +package tinylfu_test + +import ( + "fmt" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/tinylfu" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := tinylfu.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + viaMany := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + // Which entries go next depends on a sketch that is seeded at + // random, so the two caches cannot be compared on that. What + // the window that sizes the small queue hears of can. + if a, b := viaGet.Hits(), viaMany.Hits(); a != b { + t.Errorf("%d hits counted after Get, %d after GetMany", a, b) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := tinylfu.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*tinylfu.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*tinylfu.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*tinylfu.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*tinylfu.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := tinylfu.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *tinylfu.Cache[K, int], lookups []K) { + c = tinylfu.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *tinylfu.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/tinylfu/tinylfu.go b/tinylfu/tinylfu.go index 31eee3b..88eaf3a 100644 --- a/tinylfu/tinylfu.go +++ b/tinylfu/tinylfu.go @@ -94,12 +94,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -307,6 +307,79 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 48% less time than with Get with 32 keys per call on one CPU, and +// 47% less on eight; with 8 keys per call, 36% and 34% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 2% of Get. For a +// single key, call Get: through GetMany it took 86% and 80% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("tinylfu: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], maxCount, true) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(min(node.Meta()+1, maxCount)) + value, ok = node.Value, true + s.hits++ + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without counting the read. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -652,6 +725,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/tinylfu/tinylfu_test.go b/tinylfu/tinylfu_test.go index de955d5..58fbb35 100644 --- a/tinylfu/tinylfu_test.go +++ b/tinylfu/tinylfu_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/internal/sketch" "github.com/floatdrop/evict/lru" "github.com/floatdrop/evict/tinylfu" @@ -999,6 +1000,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *tinylfu.Cache[int, *cachetest.Blob], key int)) {