Add GetMany to sieve, s3fifo, tinylfu and stats

README.md

@@ -75,6 +75,27 @@ Every package takes `WithShards`. `twoq` and `s3fifo` also have options for tuni
 queues ([what that is worth for `s3fifo`](bench/results/tuning.md)); `tinylfu` tunes its
 own.
 
+### Batches
+
+`sieve`, `s3fifo` and `tinylfu` also have `GetMany(keys, values, found)`, which does what
+`Get` does for each key. For up to 32 keys at a time it loads the memory that the lookups
+start with before it looks any key up, so the CPU fetches it in parallel.
+
+| Time per key, `GetMany` against a `Get` loop | 1 CPU | 8 CPUs |
+|:---|---:|---:|
+| 1M `int` entries, 32 keys per call | −48 to −62% | −47 to −53% |
+| 1M `int` entries, 8 keys per call | −36 to −49% | −27 to −34% |
+| 1M entries with 23-byte string keys, 32 keys per call | −66 to −68% | −56 to −58% |
+| Zipf read-through with room for 1M, 32 keys per call | −49 to −55% | −48 to −51% |
+| 16K `int` entries, 32 keys per call | −2 to +3% | +1 to +9% |
+| 1M `int` entries, 1 key per call | +40 to +86% | +32 to +80% |
+
+- Ranges over the three caches of medians of three runs of `BenchmarkGetMany` in each
+  package, on an Apple M3 Max, keys picked at random. The times themselves vary more
+  between runs than these ratios.
+- For a single key, use `Get`.
+- `stats.Wrap` passes `GetMany` on to the cache it wraps, and counts a hit or a miss per key.
+
 ### Statistics
 
 [`stats.Wrap`](https://pkg.go.dev/github.com/floatdrop/evict/stats) counts the hits, misses

internal/cachetest/lookups.go

@@ -0,0 +1,114 @@
+package cachetest
+
+import (
+	"math/rand/v2"
+	"strconv"
+	"strings"
+)
+
+// The functions below make the keys for benchmarks of lookups in caches that
+// are too large for the CPU's caches. A sequence of keys to look up lies in
+// memory one after the other, so that reading the next key costs next to
+// nothing and what is measured is the cache. A string key in a sequence is a
+// copy of the key in the cache, as a key that has just come in would be: its
+// bytes are at hand, and comparing it with the key in the cache loads those of
+// the other one.
+
+// SeqLen is the length of a sequence of lookups: long enough that a benchmark
+// does not come back to the same keys while they are still in the CPU's
+// caches. It is a power of two.
+const SeqLen = 1 << 22
+
+// width is the length of a key: that of "user:profile:1000000000", as in the
+// bench module.
+const width = 23
+
+// names returns n keys of width bytes, cut out of one string.
+func names(n int) []string {
+	var b strings.Builder
+	b.Grow(width * n)
+	for i := range n {
+		b.WriteString("user:profile:")
+		b.WriteString(strconv.Itoa(1_000_000_000 + i))
+	}
+	all := b.String()
+	keys := make([]string, n)
+	for i := range keys {
+		keys[i] = all[width*i : width*(i+1)]
+	}
+	return keys
+}
+
+// copies returns a copy of keys[k] for every k in picks, one after the other.
+func copies(keys []string, picks []uint32) []string {
+	var b strings.Builder
+	b.Grow(width * len(picks))
+	for _, k := range picks {
+		b.WriteString(keys[k])
+	}
+	all := b.String()
+	out := make([]string, len(picks))
+	for i := range out {
+		out[i] = all[width*i : width*(i+1)]
+	}
+	return out
+}
+
+// uniform returns SeqLen numbers picked at random from [0, n).
+func uniform(n int) []uint32 {
+	rng := rand.New(rand.NewPCG(3, 4))
+	picks := make([]uint32, SeqLen)
+	for i := range picks {
+		picks[i] = uint32(rng.IntN(n))
+	}
+	return picks
+}
+
+// UniformInts returns the keys 0 to n-1, to fill a cache with, and a sequence
+// of lookups of them picked at random.
+func UniformInts(n int) (keys, seq []int) {
+	keys = make([]int, n)
+	for i := range keys {
+		keys[i] = i
+	}
+	seq = make([]int, SeqLen)
+	for i, k := range uniform(n) {
+		seq[i] = int(k)
+	}
+	return keys, seq
+}
+
+// UniformStrings is UniformInts with keys of 23 bytes.
+func UniformStrings(n int) (keys, seq []string) {
+	keys = names(n)
+	return keys, copies(keys, uniform(n))
+}
+
+// ZipfStrings returns requests as the bench module's ReadThrough makes them
+// for a cache of the given capacity: for four times as many keys as it has
+// room for, with a popularity that follows a Zipf distribution of skew 1.01.
+// The cache is to be read through with the first sixteen requests per entry
+// that it has room for, a Get and, if that misses, a Set, so that it holds
+// what its policy keeps; seq is the sequence of lookups that follows.
+func ZipfStrings(capacity int) (warm, seq []string) {
+	n := 4 * capacity
+	all := names(n)
+	keys := make([]string, n)
+	// The popular keys are spread over the key space, as they would be.
+	for i, p := range rand.New(rand.NewPCG(5, 6)).Perm(n) {
+		keys[i] = all[p]
+	}
+	z := rand.NewZipf(rand.New(rand.NewPCG(7, 8)), 1.01, 1, uint64(n-1))
+	draw := func(m int) []uint32 {
+		picks := make([]uint32, m)
+		for i := range picks {
+			picks[i] = uint32(z.Uint64())
+		}
+		return picks
+	}
+	warm = make([]string, 16*capacity)
+	for i, k := range draw(len(warm)) {
+		warm[i] = keys[k]
+	}
+	return warm, copies(keys, draw(SeqLen))
+}

internal/cslab/cslab.go

@@ -229,6 +229,43 @@ func (s *Slab[K, V]) Cap() int { return int(s.capacity) }
 // for writers.
 func (s *Slab[K, V]) Hash(idx uint32) uint32 { return s.Links[idx].hash }
 
+// Batch is the number of keys whose memory a reader that looks up several
+// keys at once touches before it looks any of them up; see [Slab.Touch].
+// Batches of 16 did as well with int keys and about 15% worse with string
+// keys, and batches of 8 worse with both.
+const Batch = 32
+
+// MinBatch is the fewest keys for which touching pays: with fewer, the reader
+// should look them up one by one. A batch of 4 keys took about as long as
+// looking them up one by one, a batch of 2 about 1.6 times as long, and a
+// batch of 1 more than twice as long.
+const MinBatch = 4
+
+// Touch loads the slot of the index at which a lookup of hash starts, so that
+// it is in the CPU's cache when the lookup comes, and returns it. It is for
+// readers that look up several keys: they touch the slots of all of them
+// first, then the nodes with [Slab.TouchNode], and the CPU fetches what they
+// touch from memory at the same time instead of one after another. That
+// takes loads that do not depend on each other and come one right after the
+// other; a reader that touches one key ahead of its lookups gains nothing.
+//
+// The word that Touch returns may be out of date by the time it is used. It
+// is good for TouchNode and nothing else.
+func (s *Slab[K, V]) Touch(hash uint32) uint64 {
+	return atomic.LoadUint64(&s.slots[uint32(uint64(hash)*uint64(s.size)>>32)])
+}
+
+// TouchNode loads the node that word, which [Slab.Touch] returned for hash,
+// points to, if it has the hash: the cache line with its Meta, which the Key
+// and Value usually share. A word that is out of date points to a node all
+// the same, since nodes are never taken away, so the worst it can do is load
+// the wrong one.
+func (s *Slab[K, V]) TouchNode(hash uint32, word uint64) {
+	if word>>32 == uint64(hash) {
+		atomic.LoadUint32(&s.Nodes[uint32(word)].meta)
+	}
+}
+
 // Lookup returns the value that is stored under key. The hash must be the one
 // the key was inserted with. Before it returns the value, Lookup counts the
 // read in the Meta of the node, unless the Meta has reached limit already: a
@@ -867,6 +904,8 @@ func (s *Slab[K, V]) Unlink(idx uint32) {
 // see the note in package lru. Instantiating the methods here sees to that.
 var (
 	_ = (*Slab[int, int]).Lookup
+	_ = (*Slab[int, int]).Touch
+	_ = (*Slab[int, int]).TouchNode
 	_ = (*Slab[int, int]).Find
 	_ = (*Slab[int, int]).Insert
 	_ = (*Slab[int, int]).Replace

internal/inlinecheck/inline_test.go

@@ -25,13 +25,13 @@ import (
 )
 
 // hot matches the functions that every operation on a cache goes through.
-var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` +
+var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu|stats)\.\(\*(Cache|shard)\[.*\]\)\.(Get|GetMany|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` +
 	`/c?slab\.\(\*Slab\[.*\]\)\.(Find|Lookup|Insert|Replace|Remove)$`)
 
 // slow matches the calls that must not show up in them: functions that are
 // small enough to be inlined, and are, as long as the compiler knows them.
 var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` +
-	`slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred)$|` +
+	`slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash|pin|preferred|Touch|TouchNode)$|` +
 	`cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$|cslab\.\(\*Slot\)\.Unpin$|/s3fifo\.(tagOf|ghostOf)$|` +
 	`/tinylfu\.(tagOf|kept|keep|printOf|printIn)$|/tinylfu\.\(\*climber\)\.(hit|miss)$|` +
 	`/stats\.\(\*Cache\[.*\]\)\.stripe$`)
@@ -45,9 +45,9 @@ var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` +
 // it builds in. A lock is taken only when a read cannot be sure, and calls
 // to it are fine.
 var (
-	read     = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`)
+	read     = regexp.MustCompile(`/(sieve|s3fifo|tinylfu)\.\(\*Cache\[.*\]\)\.(Get|GetMany|Peek)$|/cslab\.\(\*Slab\[.*\]\)\.Lookup$`)
 	readSlow = regexp.MustCompile(`^sync/atomic\.|^github\.com/floatdrop/evict/[^\[]*$|` +
-		`cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`)
+		`cslab\.\(\*Slab\[.*\]\)\.(Overdue|pin|preferred|Touch|TouchNode)$|cslab\.\(\*Node\[.*\]\)\.(Meta|SetMeta)$`)
 )
 
 func TestHotPathsAreInlined(t *testing.T) {
@@ -145,13 +145,25 @@ func TestHotPathsAreInlined(t *testing.T) {
 			}
 		}
 	}
+	for _, pkg := range []string{"sieve", "s3fifo", "tinylfu", "stats"} {
+		found := false
+		for name := range seen {
+			found = found || strings.Contains(name, "/"+pkg+".(*Cache[") && strings.HasSuffix(name, ").GetMany")
+		}
+		if !found {
+			t.Errorf("the compiler's output has no code for %s.Cache.GetMany", pkg)
+		}
+	}
 	for _, want := range [][2]string{
 		{"sieve.(*Cache[", "Get"},
 		{"sieve.(*Cache[", "Peek"},
+		{"sieve.(*Cache[", "GetMany"},
 		{"s3fifo.(*Cache[", "Get"},
 		{"s3fifo.(*Cache[", "Peek"},
+		{"s3fifo.(*Cache[", "GetMany"},
 		{"tinylfu.(*Cache[", "Get"},
 		{"tinylfu.(*Cache[", "Peek"},
+		{"tinylfu.(*Cache[", "GetMany"},
 		{"cslab.(*Slab[", "Lookup"},
 	} {
 		found := false

internal/inlinecheck/probe/s3fifo/probe.go

@@ -17,5 +17,8 @@ func Use(c *s3fifo.Cache[Key, Value], k Key, v Value) (Value, bool) {
 	c.Set(k, v)
 	c.Peek(k)
 	c.Delete(k)
+	var values [2]Value
+	var found [2]bool
+	c.GetMany([]Key{k, k}, values[:], found[:])
 	return c.Get(k)
 }

internal/inlinecheck/probe/sieve/probe.go

@@ -17,5 +17,8 @@ func Use(c *sieve.Cache[Key, Value], k Key, v Value) (Value, bool) {
 	c.Set(k, v)
 	c.Peek(k)
 	c.Delete(k)
+	var values [2]Value
+	var found [2]bool
+	c.GetMany([]Key{k, k}, values[:], found[:])
 	return c.Get(k)
 }

internal/inlinecheck/probe/stats/probe.go

@@ -17,5 +17,8 @@ func Use(c *stats.Cache[Key, Value], k Key, v Value) (Value, bool) {
 	c.Set(k, v)
 	c.Peek(k)
 	c.Delete(k)
+	var values [2]Value
+	var found [2]bool
+	c.GetMany([]Key{k, k}, values[:], found[:])
 	return c.Get(k)
 }

internal/inlinecheck/probe/statswrap/probe.go

@@ -13,5 +13,8 @@ type Cache[K comparable, V any] struct{ C *stats.Cache[K, V] }
 // Get reads from the counted cache.
 func (c Cache[K, V]) Get(key K) (V, bool) { return c.C.Get(key) }
 
+// GetMany reads several keys from the counted cache.
+func (c Cache[K, V]) GetMany(keys []K, values []V, found []bool) { c.C.GetMany(keys, values, found) }
+
 // Set writes to the counted cache.
 func (c Cache[K, V]) Set(key K, value V) { c.C.Set(key, value) }

internal/inlinecheck/probe/statswrapuser/probe.go

@@ -14,5 +14,8 @@ type Value struct{ Size, Hits int }
 // Use keeps the compiler from discarding the methods of the cache.
 func Use(c wrap.Cache[Key, Value], k Key, v Value) (Value, bool) {
 	c.Set(k, v)
+	var values [2]Value
+	var found [2]bool
+	c.GetMany([]Key{k, k}, values[:], found[:])
 	return c.Get(k)
 }

internal/inlinecheck/probe/tinylfu/probe.go

@@ -17,5 +17,8 @@ func Use(c *tinylfu.Cache[Key, Value], k Key, v Value) (Value, bool) {
 	c.Set(k, v)
 	c.Peek(k)
 	c.Delete(k)
+	var values [2]Value
+	var found [2]bool
+	c.GetMany([]Key{k, k}, values[:], found[:])
 	return c.Get(k)
 }

internal/inlinecheck/probe/wrap/wrap.go

@@ -32,4 +32,19 @@ func (c Caches[K, V]) Read(key K) (found int) {
 	return found
 }
 
+// ReadMany looks up keys in every cache, all of them together, and returns
+// how often they were found.
+func (c Caches[K, V]) ReadMany(keys []K) (found int) {
+	values, ok := make([]V, len(keys)), make([]bool, len(keys))
+	for _, getMany := range []func([]K, []V, []bool){c.Sieve.GetMany, c.S3FIFO.GetMany, c.TinyLFU.GetMany} {
+		getMany(keys, values, ok)
+		for _, ok := range ok {
+			if ok {
+				found++
+			}
+		}
+	}
+	return found
+}
+
 func has[V any](_ V, ok bool) bool { return ok }

internal/inlinecheck/probe/wrapuser/wrapuser.go

@@ -11,4 +11,4 @@ type Key struct{ Tenant, ID string }
 type Value struct{ Size, Hits int }
 
 // Use keeps the compiler from discarding the methods of the caches.
-func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) }
+func Use(c wrap.Caches[Key, Value], k Key) int { return c.Read(k) + c.ReadMany([]Key{k, k}) }

s3fifo/export_test.go

@@ -75,3 +75,16 @@ func Twins[V any](c *Cache[int, V]) (first, second int) {
 		seen[hash] = key
 	}
 }
+
+// Reads returns the reads that the entry for key has counted, and whether
+// there is one.
+func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) {
+	h := maphash.Comparable(c.seed, key)
+	s := &c.shards[uint32(h>>32)&c.mask]
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if idx := s.slab.Find(uint32(h), key); idx != 0 {
+		return s.slab.Nodes[idx].Meta(), true
+	}
+	return 0, false
+}