From 46f52851a62d16e041eedd1b4437f9425921d828 Mon Sep 17 00:00:00 2001 From: Vsevolod Strukchinsky Date: Thu, 12 Nov 2026 21:27:54 +0500 Subject: [PATCH] Add GetMany to sieve, s3fifo, tinylfu and stats GetMany(keys, values, found) does what Get does for each key. For up to 32 keys at a time, while at least four are left, it hashes them, loads the index slot that each lookup starts at and the node it points to, and only then looks them up, so that the CPU fetches the memory of all of them in parallel. The last few keys of a call go through Get. With a million entries and keys picked at random, a key took 47 to 62% less time than with a Get loop with 32 keys per call, on one CPU and on eight, and 27 to 49% less with 8 keys per call. With 16K entries, which fit in the CPU's caches, it is within 10% of Get. A single key is faster with Get: through GetMany it took 32 to 86% longer, for a reason not found. Medians of three runs of BenchmarkGetMany on an Apple M3 Max. The gain needs the keys in hand at once: touching one key ahead of each Get does nothing, and the CAS that pins a node is not what slows Get. stats.Wrap passes GetMany on to the cache it wraps if it has one, and calls Get for each key otherwise; it counts a hit or a miss per key. Each package checks that GetMany counts reads and hits as Get does, also when it has to take the lock, that it helps reclaim, that short slices make it panic before it does anything, and that it allocates nothing. The inline check covers it, directly and through a generic wrapper. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 ++ internal/cachetest/lookups.go | 114 ++++++ internal/cslab/cslab.go | 39 ++ internal/inlinecheck/inline_test.go | 20 +- internal/inlinecheck/probe/s3fifo/probe.go | 3 + internal/inlinecheck/probe/sieve/probe.go | 3 + internal/inlinecheck/probe/stats/probe.go | 3 + internal/inlinecheck/probe/statswrap/probe.go | 3 + .../inlinecheck/probe/statswrapuser/probe.go | 3 + internal/inlinecheck/probe/tinylfu/probe.go | 3 + internal/inlinecheck/probe/wrap/wrap.go | 15 + .../inlinecheck/probe/wrapuser/wrapuser.go | 2 +- s3fifo/export_test.go | 13 + s3fifo/getmany_test.go | 340 ++++++++++++++++++ s3fifo/s3fifo.go | 85 ++++- s3fifo/s3fifo_test.go | 9 + sieve/export_test.go | 13 + sieve/getmany_test.go | 339 +++++++++++++++++ sieve/sieve.go | 85 ++++- sieve/sieve_test.go | 9 + stats/getmany_test.go | 138 +++++++ stats/stats.go | 52 ++- tinylfu/export_test.go | 14 + tinylfu/getmany_test.go | 337 +++++++++++++++++ tinylfu/tinylfu.go | 86 ++++- tinylfu/tinylfu_test.go | 9 + 26 files changed, 1731 insertions(+), 27 deletions(-) create mode 100644 internal/cachetest/lookups.go create mode 100644 s3fifo/getmany_test.go create mode 100644 sieve/getmany_test.go create mode 100644 stats/getmany_test.go create mode 100644 tinylfu/getmany_test.go diff --git a/README.md b/README.md index 9bbeb11..ba4c286 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,27 @@ Every package takes `WithShards`. `twoq` and `s3fifo` also have options for tuni queues ([what that is worth for `s3fifo`](bench/results/tuning.md)); `tinylfu` tunes its own. +### Batches + +`sieve`, `s3fifo` and `tinylfu` also have `GetMany(keys, values, found)`, which does what +`Get` does for each key. For up to 32 keys at a time it loads the memory that the lookups +start with before it looks any key up, so the CPU fetches it in parallel. + +| Time per key, `GetMany` against a `Get` loop | 1 CPU | 8 CPUs | +|:---|---:|---:| +| 1M `int` entries, 32 keys per call | −48 to −62% | −47 to −53% | +| 1M `int` entries, 8 keys per call | −36 to −49% | −27 to −34% | +| 1M entries with 23-byte string keys, 32 keys per call | −66 to −68% | −56 to −58% | +| Zipf read-through with room for 1M, 32 keys per call | −49 to −55% | −48 to −51% | +| 16K `int` entries, 32 keys per call | −2 to +3% | +1 to +9% | +| 1M `int` entries, 1 key per call | +40 to +86% | +32 to +80% | + +- Ranges over the three caches of medians of three runs of `BenchmarkGetMany` in each + package, on an Apple M3 Max, keys picked at random. The times themselves vary more + between runs than these ratios. +- For a single key, use `Get`. +- `stats.Wrap` passes `GetMany` on to the cache it wraps, and counts a hit or a miss per key. + ### Statistics [`stats.Wrap`](https://pkg.go.dev/github.com/floatdrop/evict/stats) counts the hits, misses diff --git a/internal/cachetest/lookups.go b/internal/cachetest/lookups.go new file mode 100644 index 0000000..7c44155 --- /dev/null +++ b/internal/cachetest/lookups.go @@ -0,0 +1,114 @@ +package cachetest + +import ( + "math/rand/v2" + "strconv" + "strings" +) + +// The functions below make the keys for benchmarks of lookups in caches that +// are too large for the CPU's caches. A sequence of keys to look up lies in +// memory one after the other, so that reading the next key costs next to +// nothing and what is measured is the cache. A string key in a sequence is a +// copy of the key in the cache, as a key that has just come in would be: its +// bytes are at hand, and comparing it with the key in the cache loads those of +// the other one. + +// SeqLen is the length of a sequence of lookups: long enough that a benchmark +// does not come back to the same keys while they are still in the CPU's +// caches. It is a power of two. +const SeqLen = 1 << 22 + +// width is the length of a key: that of "user:profile:1000000000", as in the +// bench module. +const width = 23 + +// names returns n keys of width bytes, cut out of one string. +func names(n int) []string { + var b strings.Builder + b.Grow(width * n) + for i := range n { + b.WriteString("user:profile:") + b.WriteString(strconv.Itoa(1_000_000_000 + i)) + } + all := b.String() + keys := make([]string, n) + for i := range keys { + keys[i] = all[width*i : width*(i+1)] + } + return keys +} + +// copies returns a copy of keys[k] for every k in picks, one after the other. +func copies(keys []string, picks []uint32) []string { + var b strings.Builder + b.Grow(width * len(picks)) + for _, k := range picks { + b.WriteString(keys[k]) + } + all := b.String() + out := make([]string, len(picks)) + for i := range out { + out[i] = all[width*i : width*(i+1)] + } + return out +} + +// uniform returns SeqLen numbers picked at random from [0, n). +func uniform(n int) []uint32 { + rng := rand.New(rand.NewPCG(3, 4)) + picks := make([]uint32, SeqLen) + for i := range picks { + picks[i] = uint32(rng.IntN(n)) + } + return picks +} + +// UniformInts returns the keys 0 to n-1, to fill a cache with, and a sequence +// of lookups of them picked at random. +func UniformInts(n int) (keys, seq []int) { + keys = make([]int, n) + for i := range keys { + keys[i] = i + } + seq = make([]int, SeqLen) + for i, k := range uniform(n) { + seq[i] = int(k) + } + return keys, seq +} + +// UniformStrings is UniformInts with keys of 23 bytes. +func UniformStrings(n int) (keys, seq []string) { + keys = names(n) + return keys, copies(keys, uniform(n)) +} + +// ZipfStrings returns requests as the bench module's ReadThrough makes them +// for a cache of the given capacity: for four times as many keys as it has +// room for, with a popularity that follows a Zipf distribution of skew 1.01. +// The cache is to be read through with the first sixteen requests per entry +// that it has room for, a Get and, if that misses, a Set, so that it holds +// what its policy keeps; seq is the sequence of lookups that follows. +func ZipfStrings(capacity int) (warm, seq []string) { + n := 4 * capacity + all := names(n) + keys := make([]string, n) + // The popular keys are spread over the key space, as they would be. + for i, p := range rand.New(rand.NewPCG(5, 6)).Perm(n) { + keys[i] = all[p] + } + z := rand.NewZipf(rand.New(rand.NewPCG(7, 8)), 1.01, 1, uint64(n-1)) + draw := func(m int) []uint32 { + picks := make([]uint32, m) + for i := range picks { + picks[i] = uint32(z.Uint64()) + } + return picks + } + warm = make([]string, 16*capacity) + for i, k := range draw(len(warm)) { + warm[i] = keys[k] + } + return warm, copies(keys, draw(SeqLen)) +} diff --git a/internal/cslab/cslab.go b/internal/cslab/cslab.go index 40a9642..bf1cdf7 100644 --- a/internal/cslab/cslab.go +++ b/internal/cslab/cslab.go @@ -229,6 +229,43 @@ func (s *Slab[K, V]) Cap() int { return int(s.capacity) } // for writers. func (s *Slab[K, V]) Hash(idx uint32) uint32 { return s.Links[idx].hash } +// Batch is the number of keys whose memory a reader that looks up several +// keys at once touches before it looks any of them up; see [Slab.Touch]. +// Batches of 16 did as well with int keys and about 15% worse with string +// keys, and batches of 8 worse with both. +const Batch = 32 + +// MinBatch is the fewest keys for which touching pays: with fewer, the reader +// should look them up one by one. A batch of 4 keys took about as long as +// looking them up one by one, a batch of 2 about 1.6 times as long, and a +// batch of 1 more than twice as long. +const MinBatch = 4 + +// Touch loads the slot of the index at which a lookup of hash starts, so that +// it is in the CPU's cache when the lookup comes, and returns it. It is for +// readers that look up several keys: they touch the slots of all of them +// first, then the nodes with [Slab.TouchNode], and the CPU fetches what they +// touch from memory at the same time instead of one after another. That +// takes loads that do not depend on each other and come one right after the +// other; a reader that touches one key ahead of its lookups gains nothing. +// +// The word that Touch returns may be out of date by the time it is used. It +// is good for TouchNode and nothing else. +func (s *Slab[K, V]) Touch(hash uint32) uint64 { + return atomic.LoadUint64(&s.slots[uint32(uint64(hash)*uint64(s.size)>>32)]) +} + +// TouchNode loads the node that word, which [Slab.Touch] returned for hash, +// points to, if it has the hash: the cache line with its Meta, which the Key +// and Value usually share. A word that is out of date points to a node all +// the same, since nodes are never taken away, so the worst it can do is load +// the wrong one. +func (s *Slab[K, V]) TouchNode(hash uint32, word uint64) { + if word>>32 == uint64(hash) { + atomic.LoadUint32(&s.Nodes[uint32(word)].meta) + } +} + // Lookup returns the value that is stored under key. The hash must be the one // the key was inserted with. Before it returns the value, Lookup counts the // read in the Meta of the node, unless the Meta has reached limit already: a @@ -867,6 +904,8 @@ func (s *Slab[K, V]) Unlink(idx uint32) { // see the note in package lru. Instantiating the methods here sees to that. var ( _ = (*Slab[int, int]).Lookup + _ = (*Slab[int, int]).Touch + _ = (*Slab[int, int]).TouchNode _ = (*Slab[int, int]).Find _ = (*Slab[int, int]).Insert _ = (*Slab[int, int]).Replace diff --git a/internal/inlinecheck/inline_test.go b/internal/inlinecheck/inline_test.go index d0fd5d4..eebf4c6 100644 --- a/internal/inlinecheck/inline_test.go +++ b/internal/inlinecheck/inline_test.go @@ -25,13 +25,13 @@ import ( ) // hot matches the functions that every operation on a cache goes through. -var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` + +var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|GetMany|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` + `/c?slab\.\(\*Slab\[.*\]\)\.(Find|Lookup|Insert|Replace|Remove)$`) // slow matches the calls that must not show up in them: functions that are // small enough to be inlined, and are, as long as the compiler knows them. var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` + - `slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred)$|` + + `slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred|Touch|TouchNode)$|` + `cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$|cslab\.\(\*Slot\)\.Unpin$|/s3fifo\.(tagOf|ghostOf)$|` + `/tinylfu\.(tagOf|kept|keep|printOf|printIn)$|/tinylfu\.\(\*climber\)\.(hit|miss)$|` + `/stats\.\(\*Cache\[.*\]\)\.stripe$`) @@ -45,9 +45,9 @@ var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` + // it builds in. A lock is taken only when a read cannot be sure, and calls // to it are fine. var ( - read = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`) + read = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|GetMany|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`) readSlow = regexp.MustCompile(`^sync/atomic\.|^github\.com/floatdrop/evict/[^\[]*$|` + - `cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`) + `cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred|Touch|TouchNode)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`) ) func TestHotPathsAreInlined(t *testing.T) { @@ -145,13 +145,25 @@ func TestHotPathsAreInlined(t *testing.T) { } } } + for _, pkg := range []string{"sieve", "s3fifo", "tinylfu", "stats"} { + found := false + for name := range seen { + found = found || strings.Contains(name, "/"+pkg+".(*Cache[") && strings.HasSuffix(name, ").GetMany") + } + if !found { + t.Errorf("the compiler's output has no code for %s.Cache.GetMany", pkg) + } + } for _, want := range [][2]string{ {"sieve.(*Cache[", "Get"}, {"sieve.(*Cache[", "Peek"}, + {"sieve.(*Cache[", "GetMany"}, {"s3fifo.(*Cache[", "Get"}, {"s3fifo.(*Cache[", "Peek"}, + {"s3fifo.(*Cache[", "GetMany"}, {"tinylfu.(*Cache[", "Get"}, {"tinylfu.(*Cache[", "Peek"}, + {"tinylfu.(*Cache[", "GetMany"}, {"cslab.(*Slab[", "Lookup"}, } { found := false diff --git a/internal/inlinecheck/probe/s3fifo/probe.go b/internal/inlinecheck/probe/s3fifo/probe.go index 348d460..b5e0016 100644 --- a/internal/inlinecheck/probe/s3fifo/probe.go +++ b/internal/inlinecheck/probe/s3fifo/probe.go @@ -17,5 +17,8 @@ func Use(c *s3fifo.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/sieve/probe.go b/internal/inlinecheck/probe/sieve/probe.go index 42a0092..5383786 100644 --- a/internal/inlinecheck/probe/sieve/probe.go +++ b/internal/inlinecheck/probe/sieve/probe.go @@ -17,5 +17,8 @@ func Use(c *sieve.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/stats/probe.go b/internal/inlinecheck/probe/stats/probe.go index d9f6166..bcee219 100644 --- a/internal/inlinecheck/probe/stats/probe.go +++ b/internal/inlinecheck/probe/stats/probe.go @@ -17,5 +17,8 @@ func Use(c *stats.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/statswrap/probe.go b/internal/inlinecheck/probe/statswrap/probe.go index 9a81900..8fa25fa 100644 --- a/internal/inlinecheck/probe/statswrap/probe.go +++ b/internal/inlinecheck/probe/statswrap/probe.go @@ -13,5 +13,8 @@ type Cache[K comparable, V any] struct{ C *stats.Cache[K, V] } // Get reads from the counted cache. func (c Cache[K, V]) Get(key K) (V, bool) { return c.C.Get(key) } +// GetMany reads several keys from the counted cache. +func (c Cache[K, V]) GetMany(keys []K, values []V, found []bool) { c.C.GetMany(keys, values, found) } + // Set writes to the counted cache. func (c Cache[K, V]) Set(key K, value V) { c.C.Set(key, value) } diff --git a/internal/inlinecheck/probe/statswrapuser/probe.go b/internal/inlinecheck/probe/statswrapuser/probe.go index 30fe528..24e802a 100644 --- a/internal/inlinecheck/probe/statswrapuser/probe.go +++ b/internal/inlinecheck/probe/statswrapuser/probe.go @@ -14,5 +14,8 @@ type Value struct{ Size, Hits int } // Use keeps the compiler from discarding the methods of the cache. func Use(c wrap.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/tinylfu/probe.go b/internal/inlinecheck/probe/tinylfu/probe.go index c0cabac..ae3a9cb 100644 --- a/internal/inlinecheck/probe/tinylfu/probe.go +++ b/internal/inlinecheck/probe/tinylfu/probe.go @@ -17,5 +17,8 @@ func Use(c *tinylfu.Cache[Key, Value], k Key, v Value) (Value, bool) { c.Set(k, v) c.Peek(k) c.Delete(k) + var values [2]Value + var found [2]bool + c.GetMany([]Key{k, k}, values[:], found[:]) return c.Get(k) } diff --git a/internal/inlinecheck/probe/wrap/wrap.go b/internal/inlinecheck/probe/wrap/wrap.go index 0176f61..8e13428 100644 --- a/internal/inlinecheck/probe/wrap/wrap.go +++ b/internal/inlinecheck/probe/wrap/wrap.go @@ -32,4 +32,19 @@ func (c Caches[K, V]) Read(key K) (found int) { return found } +// ReadMany looks up keys in every cache, all of them together, and returns +// how often they were found. +func (c Caches[K, V]) ReadMany(keys []K) (found int) { + values, ok := make([]V, len(keys)), make([]bool, len(keys)) + for _, getMany := range []func([]K, []V, []bool){c.Sieve.GetMany, c.S3FIFO.GetMany, c.TinyLFU.GetMany} { + getMany(keys, values, ok) + for _, ok := range ok { + if ok { + found++ + } + } + } + return found +} + func has[V any](_ V, ok bool) bool { return ok } diff --git a/internal/inlinecheck/probe/wrapuser/wrapuser.go b/internal/inlinecheck/probe/wrapuser/wrapuser.go index 8e09ce1..966445b 100644 --- a/internal/inlinecheck/probe/wrapuser/wrapuser.go +++ b/internal/inlinecheck/probe/wrapuser/wrapuser.go @@ -11,4 +11,4 @@ type Key struct{ Tenant, ID string } type Value struct{ Size, Hits int } // Use keeps the compiler from discarding the methods of the caches. -func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) } +func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) + c.ReadMany([]Key{k, k}) } diff --git a/s3fifo/export_test.go b/s3fifo/export_test.go index 439aa52..fcdf76a 100644 --- a/s3fifo/export_test.go +++ b/s3fifo/export_test.go @@ -75,3 +75,16 @@ func Twins[V any](c *Cache[int, V]) (first, second int) { seen[hash] = key } } + +// Reads returns the reads that the entry for key has counted, and whether +// there is one. +func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + if idx := s.slab.Find(uint32(h), key); idx != 0 { + return s.slab.Nodes[idx].Meta(), true + } + return 0, false +} diff --git a/s3fifo/getmany_test.go b/s3fifo/getmany_test.go new file mode 100644 index 0000000..35a8530 --- /dev/null +++ b/s3fifo/getmany_test.go @@ -0,0 +1,340 @@ +package s3fifo_test + +import ( + "fmt" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/s3fifo" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := s3fifo.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := s3fifo.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + viaMany := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + for k := 2 * capacity; k < 3*capacity; k++ { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + getMain, getSmall := viaGet.Queues() + manyMain, manySmall := viaMany.Queues() + if !slices.Equal(getMain, manyMain) || !slices.Equal(getSmall, manySmall) { + t.Errorf("after evictions, the queues differ:\nGet: %v %v\nGetMany: %v %v", getMain, getSmall, manyMain, manySmall) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := s3fifo.New[int, int](capacity, s3fifo.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := s3fifo.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := s3fifo.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*s3fifo.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*s3fifo.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*s3fifo.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*s3fifo.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := s3fifo.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *s3fifo.Cache[K, int], lookups []K) { + c = s3fifo.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *s3fifo.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/s3fifo/s3fifo.go b/s3fifo/s3fifo.go index 2fb31dd..6dbee86 100644 --- a/s3fifo/s3fifo.go +++ b/s3fifo/s3fifo.go @@ -116,12 +116,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -380,6 +380,78 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 51% less time than with Get with 32 keys per call on one CPU, and +// 48% less on eight; with 8 keys per call, 36% and 32% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 9% of Get. For a +// single key, call Get: through GetMany it took 40% and 32% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("s3fifo: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], maxFrequency, false) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(min(node.Meta()+1, maxFrequency)) + value, ok = node.Value, true + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without counting the read. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -662,6 +734,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/s3fifo/s3fifo_test.go b/s3fifo/s3fifo_test.go index 54935ab..6146a02 100644 --- a/s3fifo/s3fifo_test.go +++ b/s3fifo/s3fifo_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/s3fifo" ) @@ -1004,6 +1005,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *s3fifo.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *s3fifo.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *s3fifo.Cache[int, *cachetest.Blob], key int)) { diff --git a/sieve/export_test.go b/sieve/export_test.go index 30f2945..042f3e2 100644 --- a/sieve/export_test.go +++ b/sieve/export_test.go @@ -58,3 +58,16 @@ func Twins[V any](c *Cache[int, V]) (first, second int) { seen[hash] = key } } + +// Reads returns the mark of the entry for key, which is visited if it was read +// since the hand last came by, and whether there is one. +func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + if idx := s.slab.Find(uint32(h), key); idx != 0 { + return s.slab.Nodes[idx].Meta(), true + } + return 0, false +} diff --git a/sieve/getmany_test.go b/sieve/getmany_test.go new file mode 100644 index 0000000..2ab1e8f --- /dev/null +++ b/sieve/getmany_test.go @@ -0,0 +1,339 @@ +package sieve_test + +import ( + "fmt" + "maps" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/sieve" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := sieve.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := sieve.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := sieve.New[int, int](capacity, sieve.WithShards(1)) + viaMany := sieve.New[int, int](capacity, sieve.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + for k := 2 * capacity; k < 3*capacity; k++ { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + if a, b := slices.Sorted(maps.Keys(maps.Collect(viaGet.All()))), slices.Sorted(maps.Keys(maps.Collect(viaMany.All()))); !slices.Equal(a, b) { + t.Errorf("after evictions, the caches hold different keys:\nGet: %v\nGetMany: %v", a, b) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := sieve.New[int, int](capacity, sieve.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := sieve.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := sieve.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*sieve.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*sieve.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*sieve.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*sieve.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := sieve.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *sieve.Cache[K, int], lookups []K) { + c = sieve.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *sieve.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/sieve/sieve.go b/sieve/sieve.go index bf98b46..784d80a 100644 --- a/sieve/sieve.go +++ b/sieve/sieve.go @@ -79,12 +79,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -212,6 +212,78 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 62% less time than with Get with 32 keys per call on one CPU, and +// 53% less on eight; with 8 keys per call, 49% and 27% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 2% of Get. For a +// single key, call Get: through GetMany it took 42% and 36% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("sieve: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], visited, false) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(visited) + value, ok = node.Value, true + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without marking the entry as used. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -431,6 +503,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/sieve/sieve_test.go b/sieve/sieve_test.go index 3767d3c..846c85a 100644 --- a/sieve/sieve_test.go +++ b/sieve/sieve_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/sieve" ) @@ -506,6 +507,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *sieve.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *sieve.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *sieve.Cache[int, *cachetest.Blob], key int)) { diff --git a/stats/getmany_test.go b/stats/getmany_test.go new file mode 100644 index 0000000..aa71b41 --- /dev/null +++ b/stats/getmany_test.go @@ -0,0 +1,138 @@ +package stats_test + +import ( + "math/rand/v2" + "slices" + "testing" + + "github.com/floatdrop/evict" + "github.com/floatdrop/evict/lru" + "github.com/floatdrop/evict/s3fifo" + "github.com/floatdrop/evict/stats" +) + +// batcher is a cache with a GetMany of its own, which says how often it was +// called. +type batcher struct { + evict.Cache[int, int] + calls int +} + +func (b *batcher) GetMany(keys, values []int, found []bool) { + b.calls++ + for i, k := range keys { + values[i], found[i] = b.Get(k) + } +} + +// GetMany has to find what Peek finds, and count a hit or a miss for every +// key, whether the cache has a GetMany of its own or not. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 500 + for name, cache := range map[string]evict.Cache[int, int]{ + "lru": lru.New[int, int](capacity), + "s3fifo": s3fifo.New[int, int](capacity), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + c := stats.Wrap(cache) + for k := range capacity / 2 { + c.Set(k, -k) + } + + rng := rand.New(rand.NewPCG(1, 2)) + var hits, misses uint64 + for _, n := range []int{0, 1, 3, 4, 31, 32, 33, 100} { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + v, ok := c.Peek(k) + if values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + if ok { + hits++ + } else { + misses++ + } + } + } + + if s := c.Stats(); s.Hits != hits || s.Misses != misses { + t.Errorf("%d hits and %d misses counted, want %d and %d", s.Hits, s.Misses, hits, misses) + } + }) + } +} + +// A cache that has a GetMany of its own is asked with it. +func TestGetManyForwards(t *testing.T) { + t.Parallel() + + b := &batcher{Cache: lru.New[int, int](10)} + c := stats.Wrap[int, int](b) + c.Set(1, 1) + values, found := make([]int, 2), make([]bool, 2) + c.GetMany([]int{1, 2}, values, found) + + if b.calls != 1 { + t.Errorf("the cache's GetMany was called %d times, want once", b.calls) + } + if s := c.Stats(); s.Hits != 1 || s.Misses != 1 { + t.Errorf("%d hits and %d misses counted, want 1 and 1", s.Hits, s.Misses) + } +} + +// Slices that are too short for the keys make GetMany panic before it looks +// anything up or counts it. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := stats.Wrap(lru.New[int, int](10)) + c.Set(1, 1) + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany([]int{1, 2, 3}, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + if s := c.Stats(); s.Hits != 0 || s.Misses != 0 { + t.Errorf("%d hits and %d misses counted before the panic", s.Hits, s.Misses) + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + for name, cache := range map[string]evict.Cache[int, int]{ + "lru": lru.New[int, int](100), + "s3fifo": s3fifo.New[int, int](100), + } { + t.Run(name, func(t *testing.T) { + c := stats.Wrap(cache) + keys := make([]int, 40) + for i := range keys { + keys[i] = i + c.Set(i, i) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } + }) + } +} diff --git a/stats/stats.go b/stats/stats.go index dfb11d7..bd6fdc2 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -11,6 +11,7 @@ // What is counted: // // - Get: a hit if it finds the key, a miss if it does not. +// - GetMany: the same for each key. // - Set: an eviction if it pushed out the entry of another key to make // room, or if the cache has no capacity and handed the new entry back. // Replacing the entry of a key is not an eviction. @@ -57,8 +58,8 @@ const cacheLine = 128 // of one snapshot may be from slightly different moments. A count is never // less than that of a snapshot taken before it. type Snapshot struct { - // Hits and Misses count the calls to Get that found their key, and - // those that did not. + // Hits and Misses count the keys that Get and GetMany found, and those + // that they did not. Hits, Misses uint64 // Evictions counts the entries that Set pushed out to make room, and @@ -70,8 +71,8 @@ type Snapshot struct { Len, Cap int } -// HitRatio returns the share of calls to Get that found their key, from 0 to -// 1, or 0 if there were none. +// HitRatio returns the share of the keys looked up with Get and GetMany that +// were found, from 0 to 1, or 0 if there were none. func (s Snapshot) HitRatio() float64 { if s.Hits+s.Misses == 0 { return 0 @@ -96,11 +97,18 @@ func (s Snapshot) Sub(prev Snapshot) Snapshot { // the cache it wraps are, as they are for every cache in this module. type Cache[K comparable, V any] struct { cache evict.Cache[K, V] + many manyGetter[K, V] // cache, if it has a GetMany stripes []stripe shift uint8 // 64 minus log2 of the number of stripes zero bool // the cache has no capacity, and hands every entry back } +// manyGetter is a cache that looks up several keys at once, as sieve, s3fifo +// and tinylfu do. +type manyGetter[K comparable, V any] interface { + GetMany(keys []K, values []V, found []bool) +} + // A stripe holds the counts of the goroutines that pick it, in a cache line // of its own. // @@ -127,8 +135,10 @@ func Wrap[K comparable, V any](cache evict.Cache[K, V]) *Cache[K, V] { // Sixteen stripes per CPU make it unlikely that two goroutines that run // at the same time pick the same one. n := 1 << bits.Len(uint(16*runtime.GOMAXPROCS(0)-1)) + many, _ := cache.(manyGetter[K, V]) return &Cache[K, V]{ cache: cache, + many: many, stripes: make([]stripe, n), shift: uint8(64 - bits.TrailingZeros(uint(n))), zero: cache.Cap() == 0, @@ -162,6 +172,40 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany looks up each of keys, stores the value it finds for keys[i] in +// values[i] and whether it found one in found[i], and counts a hit or a miss +// for each key. It panics if values or found is shorter than keys. +// +// If the cache has a GetMany of its own, as sieve, s3fifo and tinylfu do, +// GetMany uses it: for a cache that is too large for the CPU's caches it is +// faster than Get for each key. Otherwise it calls Get for each key. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + if len(values) < len(keys) || len(found) < len(keys) { + panic("stats: GetMany: values and found must be at least as long as keys") + } + if c.many != nil { + c.many.GetMany(keys, values, found) + } else { + for i, key := range keys { + values[i], found[i] = c.cache.Get(key) + } + } + + hits := 0 + for _, ok := range found[:len(keys)] { + if ok { + hits++ + } + } + s := c.stripe() + if hits > 0 { + atomic.AddUint64(&s.hits, uint64(hits)) + } + if misses := len(keys) - hits; misses > 0 { + atomic.AddUint64(&s.misses, uint64(misses)) + } +} + // Peek returns the value stored under key, and counts nothing. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { return c.cache.Peek(key) } diff --git a/tinylfu/export_test.go b/tinylfu/export_test.go index 9e232c6..18e95e2 100644 --- a/tinylfu/export_test.go +++ b/tinylfu/export_test.go @@ -142,3 +142,17 @@ func Apart[V any](c *Cache[int, V], n int, others ...int) []int { } return keys } + +// Hits returns the hits that the window of the cache has been told of or will +// be: those that readers counted in their slots, and those that shards counted +// under their locks. +func (c *Cache[K, V]) Hits() uint64 { + hits := c.window.readers.Hits() + c.window.hits.Load() + for i := range c.shards { + s := &c.shards[i] + s.mu.Lock() + hits += s.hits + s.mu.Unlock() + } + return hits +} diff --git a/tinylfu/getmany_test.go b/tinylfu/getmany_test.go new file mode 100644 index 0000000..1f51f86 --- /dev/null +++ b/tinylfu/getmany_test.go @@ -0,0 +1,337 @@ +package tinylfu_test + +import ( + "fmt" + "math/rand/v2" + "slices" + "strconv" + "sync" + "testing" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/tinylfu" +) + +// GetMany has to find what Peek finds, key by key, for any number of keys +// around that of a batch, also when a key comes up more than once. +func TestGetMany(t *testing.T) { + t.Parallel() + + const capacity = 1000 + c := tinylfu.New[int, int](capacity) + rng := rand.New(rand.NewPCG(1, 2)) + for range 3 * capacity { + k := rng.IntN(2 * capacity) + c.Set(k, -k) + } + + // Lengths that leave every number of keys from 0 to MinBatch at the end + // of a call, where they go through Get. + for _, n := range []int{ + 0, 1, 2, 3, cslab.MinBatch, cslab.Batch - 1, cslab.Batch, cslab.Batch + 1, + cslab.Batch + cslab.MinBatch - 1, 2*cslab.Batch + 3, 3*cslab.Batch + 5, capacity, + } { + keys := make([]int, n) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, n), make([]bool, n) + c.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := c.Peek(k); values[i] != v || found[i] != ok { + t.Fatalf("GetMany of %d keys: key %d gave %d, %t; Peek gives %d, %t", n, k, values[i], found[i], v, ok) + } + } + } +} + +// values may be keys itself: each key is looked up before its value is +// written over it. +func TestGetManyInPlace(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](1000) + for k := range 500 { + c.Set(k, -k) + } + rng := rand.New(rand.NewPCG(5, 6)) + keys := make([]int, 2*cslab.Batch+3) + for i := range keys { + keys[i] = rng.IntN(1000) + } + asked := slices.Clone(keys) + found := make([]bool, len(keys)) + c.GetMany(keys, keys, found) + for i, k := range asked { + if v, ok := c.Peek(k); keys[i] != v || found[i] != ok { + t.Fatalf("key %d gave %d, %t; Peek gives %d, %t", k, keys[i], found[i], v, ok) + } + } +} + +// GetMany has to count reads as Get does, also those of a key that comes up +// several times, past the limit, and also when it has to take the lock: what +// the cache evicts next depends on them. +func TestGetManyCountsReads(t *testing.T) { + t.Parallel() + + for _, locked := range []bool{false, true} { + t.Run("locked="+strconv.FormatBool(locked), func(t *testing.T) { + t.Parallel() + + const capacity = 200 + viaGet := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + viaMany := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + for k := range capacity { + viaGet.Set(k, k) + viaMany.Set(k, k) + } + + // Half of the keys are not there, and most come up more than + // once. The last few are too few to load ahead, and go through + // Get. + rng := rand.New(rand.NewPCG(3, 4)) + keys := make([]int, 5*cslab.Batch+cslab.MinBatch-1) + for i := range keys { + keys[i] = rng.IntN(2 * capacity) + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + + // With every slot of the readers taken, a lookup cannot pin + // anything and has to take the lock. + releaseGet, releaseMany := func() {}, func() {} + if locked { + releaseGet, releaseMany = viaGet.OccupyReaders(0), viaMany.OccupyReaders(0) + } + viaMany.GetMany(keys, values, found) + for i, k := range keys { + if v, ok := viaGet.Get(k); values[i] != v || found[i] != ok { + t.Fatalf("key %d: GetMany gave %d, %t; Get gives %d, %t", k, values[i], found[i], v, ok) + } + } + releaseGet() + releaseMany() + + for k := range 2 * capacity { + a, aok := viaGet.Reads(k) + b, bok := viaMany.Reads(k) + if a != b || aok != bok { + t.Errorf("key %d: %d reads, %t after Get; %d, %t after GetMany", k, a, aok, b, bok) + } + } + + // Which entries go next depends on a sketch that is seeded at + // random, so the two caches cannot be compared on that. What + // the window that sizes the small queue hears of can. + if a, b := viaGet.Hits(), viaMany.Hits(); a != b { + t.Errorf("%d hits counted after Get, %d after GetMany", a, b) + } + }) + } +} + +// GetMany reads while others write. Keys that nobody touches have to be found +// with their values, others with a value that was stored under them if at +// all, and the race detector has to find nothing. +func TestGetManyConcurrent(t *testing.T) { + t.Parallel() + + // The writers never hold so many keys that the cache has to evict. + const ( + capacity = 1024 + stable = 64 + changing = 512 + ) + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + for k := range stable { + c.Set(k, -k) + } + + var writers, readers sync.WaitGroup + done := make(chan struct{}) + for w := range 2 { + writers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(w), 9)) + for { + select { + case <-done: + return + default: + } + k := stable + rng.IntN(changing) + if rng.IntN(4) == 0 { + c.Delete(k) + } else { + c.Set(k, -k) + } + } + }) + } + for r := range 4 { + readers.Go(func() { + rng := rand.New(rand.NewPCG(uint64(r), 7)) + n := cslab.Batch + 5 + keys, values, found := make([]int, n), make([]int, n), make([]bool, n) + for range 2000 { + for i := range keys { + keys[i] = rng.IntN(stable + changing) + } + c.GetMany(keys, values, found) + for i, k := range keys { + if k < stable && !found[i] || found[i] && values[i] != -k { + t.Errorf("key %d gave %d, %t", k, values[i], found[i]) + return + } + } + } + }) + } + readers.Wait() + close(done) + writers.Wait() +} + +// Slices that are too short for the keys must make GetMany panic before it +// does anything: before it writes past their ends, which there is room for in +// the arrays behind them, and before it counts any read. +func TestGetManyShortSlices(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](16) + keys := []int{1, 2, 3} + for _, k := range keys { + c.Set(k, k) + } + values, found := make([]int, 8), make([]bool, 8) + func() { + defer func() { + if recover() == nil { + t.Error("GetMany with values and found shorter than keys did not panic") + } + }() + c.GetMany(keys, values[:2], found[:2]) + }() + + if slices.ContainsFunc(values, func(v int) bool { return v != 0 }) || slices.Contains(found, true) { + t.Errorf("GetMany wrote %v and %v before it panicked", values, found) + } + for _, k := range keys { + if reads, _ := c.Reads(k); reads != 0 { + t.Errorf("key %d has %d reads after GetMany panicked", k, reads) + } + } +} + +// TestGetManyNoAllocs is not parallel: counting allocations needs the process +// to itself. +func TestGetManyNoAllocs(t *testing.T) { + c := tinylfu.New[string, int](1000) + keys := make([]string, 3*cslab.Batch) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + if i%2 == 0 { + c.Set(keys[i], i) + } + } + values, found := make([]int, len(keys)), make([]bool, len(keys)) + if allocs := testing.AllocsPerRun(100, func() { c.GetMany(keys, values, found) }); allocs != 0 { + t.Errorf("GetMany made %v allocations, want 0", allocs) + } +} + +// The caches and keys of BenchmarkGetMany, made once. Every key has been read +// often enough beforehand that reads no longer change the entries, so that +// the first variant to run does not pay for that. +var ( + benchSmall = sync.OnceValues(func() (*tinylfu.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 14)) }) + benchLarge = sync.OnceValues(func() (*tinylfu.Cache[int, int], []int) { return filled(cachetest.UniformInts(1 << 20)) }) + benchNames = sync.OnceValues(func() (*tinylfu.Cache[string, int], []string) { + return filled(cachetest.UniformStrings(1 << 20)) + }) + benchZipf = sync.OnceValues(func() (*tinylfu.Cache[string, int], []string) { + const capacity = 1 << 20 + warm, seq := cachetest.ZipfStrings(capacity) + c := tinylfu.New[string, int](capacity) + for i, k := range warm { + if _, ok := c.Get(k); !ok { + c.Set(k, i) + } + } + for _, k := range seq { + c.Get(k) + } + return c, seq + }) +) + +// filled returns a cache that holds keys, with room for twice as many so that +// no shard has to evict any. +func filled[K comparable](keys, seq []K) (c *tinylfu.Cache[K, int], lookups []K) { + c = tinylfu.New[K, int](2 * len(keys)) + for i, k := range keys { + c.Set(k, i) + } + for range 4 { + for _, k := range keys { + c.Get(k) + } + } + return c, seq +} + +// BenchmarkGetMany looks keys up, on every CPU at once: with Get one after the +// other, and with GetMany, in calls of the given number of keys. The keys are +// all there, but for those of zipf, which are requested as in the bench +// module's ReadThrough. ns/key is the time per key, and hits the share of keys +// found. +func BenchmarkGetMany(b *testing.B) { + b.Run("int/entries=16K/keys=32", func(b *testing.B) { c, seq := benchSmall(); benchGetMany(b, c, seq, 32) }) + for _, n := range []int{1, 8, 32} { + b.Run(fmt.Sprintf("int/entries=1M/keys=%d", n), func(b *testing.B) { c, seq := benchLarge(); benchGetMany(b, c, seq, n) }) + } + b.Run("string/entries=1M/keys=32", func(b *testing.B) { c, seq := benchNames(); benchGetMany(b, c, seq, 32) }) + b.Run("zipf/capacity=1M/keys=32", func(b *testing.B) { c, seq := benchZipf(); benchGetMany(b, c, seq, 32) }) +} + +// benchGetMany looks up seq in calls of n keys, a power of two. +func benchGetMany[K comparable](b *testing.B, c *tinylfu.Cache[K, int], seq []K, n int) { + b.Helper() + mask := len(seq) - 1 + + run := func(b *testing.B, many bool) { + b.Helper() + var ( + mu sync.Mutex + hits int + ) + b.RunParallel(func(pb *testing.PB) { + values, found := make([]int, n), make([]bool, n) + i, h := rand.Int()&^(n-1), 0 + for pb.Next() { + keys := seq[i&mask : i&mask+n] + if many { + c.GetMany(keys, values, found) + } else { + for j, key := range keys { + values[j], found[j] = c.Get(key) + } + } + for _, ok := range found { + if ok { + h++ + } + } + i += n + } + mu.Lock() + hits += h + mu.Unlock() + }) + keys := float64(b.N * n) + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/keys, "ns/key") + b.ReportMetric(float64(hits)/keys, "hits") + } + b.Run("get", func(b *testing.B) { run(b, false) }) + b.Run("many", func(b *testing.B) { run(b, true) }) +} diff --git a/tinylfu/tinylfu.go b/tinylfu/tinylfu.go index 31eee3b..88eaf3a 100644 --- a/tinylfu/tinylfu.go +++ b/tinylfu/tinylfu.go @@ -94,12 +94,12 @@ import ( // [Cache.All]. A Cache must be created with [New]; the zero value is of no // use. // -// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key -// is not found while an entry is on its way out of the same shard, and when -// more goroutines read at the same moment than the cache was made for, the -// read takes the lock of the shard to be sure. Writes are spread over shards -// that are locked independently, and each shard evicts on its own. See -// [WithShards]. +// [Cache.Get], [Cache.GetMany] and [Cache.Peek] take no lock, with rare +// exceptions: when a key is not found while an entry is on its way out of the +// same shard, and when more goroutines read at the same moment than the cache +// was made for, the read takes the lock of the shard to be sure. Writes are +// spread over shards that are locked independently, and each shard evicts on +// its own. See [WithShards]. type Cache[K comparable, V any] struct { seed maphash.Seed mask uint32 // number of shards minus one @@ -307,6 +307,79 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { return value, ok } +// GetMany does what Get does for each of keys, one after the other, and +// stores the value it finds for keys[i] in values[i] and whether it found one +// in found[i]. It panics if values or found is shorter than keys. values and +// found may each be keys itself, but must not otherwise overlap it. A key that +// cannot be hashed makes GetMany panic, as it makes Get panic, but possibly +// before the keys just ahead of it have been looked up. +// +// For a cache that is too large for the CPU's caches, GetMany is faster than +// calling Get for each key. It takes up to 32 keys at a time, and before it +// looks any of them up, it loads the slot of the index at which each lookup +// starts and the node that the slot points to, so that the CPU can fetch them +// from memory in parallel. When fewer than four keys are left, it looks them +// up with Get, one by one. +// +// In BenchmarkGetMany, with a million entries and keys picked at random, a +// key took 48% less time than with Get with 32 keys per call on one CPU, and +// 47% less on eight; with 8 keys per call, 36% and 34% less. With 16,384 +// entries, which fit in the CPU's caches, it was within 2% of Get. For a +// single key, call Get: through GetMany it took 86% and 80% longer. These are +// medians of three runs on an Apple M3 Max, and vary from run to run. +func (c *Cache[K, V]) GetMany(keys []K, values []V, found []bool) { + // values[:len(keys)] would only be checked against the capacity, and + // a slice that is too short would be written past its end. + if len(values) < len(keys) || len(found) < len(keys) { + panic("tinylfu: GetMany: values and found must be at least as long as keys") + } + + var hashes, words [cslab.Batch]uint64 + for len(keys) >= cslab.MinBatch { + n := min(len(keys), cslab.Batch) + + for i := range n { + hashes[i] = maphash.Comparable(c.seed, keys[i]) + } + for i := range n { + h := hashes[i] + words[i] = c.shards[uint32(h>>32)&c.mask].slab.Touch(uint32(h)) + } + for i := range n { + h := hashes[i] + c.shards[uint32(h>>32)&c.mask].slab.TouchNode(uint32(h), words[i]) + } + + // From here on, as in Get. + for i := range n { + h := hashes[i] + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + value, ok, certain := s.slab.Lookup(hash, keys[i], maxCount, true) + if s.slab.Overdue() { + s.help() + } + if !certain { + s.mu.Lock() + if idx := s.slab.Find(hash, keys[i]); idx != 0 { + node := &s.slab.Nodes[idx] + node.SetMeta(min(node.Meta()+1, maxCount)) + value, ok = node.Value, true + s.hits++ + } + s.mu.Unlock() + } + values[i], found[i] = value, ok + } + + keys, values, found = keys[n:], values[n:], found[n:] + } + + // Too few keys are left for loading ahead to pay. + for i, key := range keys { + values[i], found[i] = c.Get(key) + } +} + // Peek returns the value stored under key without counting the read. func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) @@ -652,6 +725,7 @@ func (c *Cache[K, V]) All() iter.Seq2[K, V] { var ( _ = (*Cache[int, int]).Get _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).GetMany _ = (*Cache[int, int]).Set _ = (*Cache[int, int]).Delete _ = (*Cache[int, int]).Len diff --git a/tinylfu/tinylfu_test.go b/tinylfu/tinylfu_test.go index de955d5..58fbb35 100644 --- a/tinylfu/tinylfu_test.go +++ b/tinylfu/tinylfu_test.go @@ -15,6 +15,7 @@ import ( "github.com/floatdrop/evict" "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/cslab" "github.com/floatdrop/evict/internal/sketch" "github.com/floatdrop/evict/lru" "github.com/floatdrop/evict/tinylfu" @@ -999,6 +1000,14 @@ func TestReadersHelp(t *testing.T) { t.Parallel() testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) }) + t.Run("GetMany", func(t *testing.T) { + t.Parallel() + // Enough keys that GetMany loads ahead rather than call Get. + testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { + keys := slices.Repeat([]int{key}, cslab.MinBatch) + c.GetMany(keys, make([]*cachetest.Blob, len(keys)), make([]bool, len(keys))) + }) + }) } func testReadersHelp(t *testing.T, read func(c *tinylfu.Cache[int, *cachetest.Blob], key int)) {