From e0d396b2acab9e782bbd1758d19ddc3092e1b0ca Mon Sep 17 00:00:00 2001 From: Vsevolod Strukchinsky Date: Wed, 11 Nov 2026 21:27:18 +0500 Subject: [PATCH 1/4] Let readers tally their hits in the slots they pin with Slab.Lookup takes a tally argument. When it is set, a lookup that finds its key adds one to a counter in the slot of the Readers that it pinned the node with, which is in the cache line that a reader writes to anyway. Readers.Hits adds the counters up. Unlike the Meta of a node, the tally knows no limit, so a cache can tell how many requests its readers answered without their writing to anything that all of them share. s3fifo and sieve do not tally. Co-Authored-By: Claude Opus 5.5 --- internal/cslab/cslab.go | 12 +++- internal/cslab/cslab_test.go | 103 +++++++++++++++++++++++++++-------- internal/cslab/readers.go | 20 ++++++- s3fifo/s3fifo.go | 4 +- sieve/sieve.go | 4 +- 5 files changed, 113 insertions(+), 30 deletions(-) diff --git a/internal/cslab/cslab.go b/internal/cslab/cslab.go index a0e60d3..c688db6 100644 --- a/internal/cslab/cslab.go +++ b/internal/cslab/cslab.go @@ -222,11 +222,16 @@ func (s *Slab[K, V]) Hash(idx uint32) uint32 { return s.Links[idx].hash } // its Meta at that very moment: that one may come out one too high or one // too low, though never above the limit. // +// If tally is set, a lookup that finds its key also adds one to the hits of +// the slot it pinned the node with, which is a count that, unlike the Meta of +// a node, knows no limit; see [Readers.Hits]. A lookup that finds nothing, or +// does not get to pin anything, adds nothing. +// // Lookup is for readers. If it finds no value, certain says whether the key // is known to be absent. It is not if entries were on the move in the // meantime, or if no slot was to be had for pinning a node. The caller should // then ask again as a writer, with [Slab.Find]. -func (s *Slab[K, V]) Lookup(hash uint32, key K, limit uint32) (value V, found, certain bool) { +func (s *Slab[K, V]) Lookup(hash uint32, key K, limit uint32, tally bool) (value V, found, certain bool) { seq := s.seq.Load() slots, mask := s.slots, s.mask @@ -275,6 +280,11 @@ func (s *Slab[K, V]) Lookup(hash uint32, key K, limit uint32) (value V, found, c slot.Unpin() if k == key { + // The slot is no longer this reader's, but the count is + // atomic, and its line is still in this CPU's cache. + if tally { + slot.hits.Add(1) + } return value, true, true } // The node belongs to another key with the same hash, which is diff --git a/internal/cslab/cslab_test.go b/internal/cslab/cslab_test.go index e8ee550..5c81f58 100644 --- a/internal/cslab/cslab_test.go +++ b/internal/cslab/cslab_test.go @@ -49,7 +49,7 @@ func check[K, V comparable](t *testing.T, s *Slab[K, V], want map[K]V) { if got := s.Find(hash, n.Key); got != idx { t.Fatalf("Find(%v) = %d, want %d", n.Key, got, idx) } - if got, found, certain := s.Lookup(hash, n.Key, 0); got != n.Value || !found || !certain { + if got, found, certain := s.Lookup(hash, n.Key, 0, false); got != n.Value || !found || !certain { t.Fatalf("Lookup(%v) = %v, %t, %t, want %v", n.Key, got, found, certain, n.Value) } } @@ -164,7 +164,7 @@ func TestInit(t *testing.T) { t.Errorf("Init(%d, %d): list %d = %v, want empty", tt.capacity, tt.heads, head, got) } } - if v, found, certain := s.Lookup(42, 42, 1); v != 0 || found || !certain { + if v, found, certain := s.Lookup(42, 42, 1, false); v != 0 || found || !certain { t.Errorf("Init(%d, %d): Lookup on empty slab = %d, %t, %t", tt.capacity, tt.heads, v, found, certain) } check(t, &s, map[int]int{}) @@ -197,7 +197,7 @@ func TestInsertReplaceRemove(t *testing.T) { if got := s.Find(1, "x"); got != 0 { t.Errorf(`Find("x") = %d, want 0`, got) } - if got, found, certain := s.Lookup(1, "x", 1); got != 0 || found || !certain { + if got, found, certain := s.Lookup(1, "x", 1, false); got != 0 || found || !certain { t.Errorf(`Lookup("x") = %d, %t, %t, want a certain miss`, got, found, certain) } now := func() uint64 { return s.readers.time.Load() } @@ -647,14 +647,14 @@ func TestLookupWithoutSlot(t *testing.T) { s.PushFront(0, s.Insert(1, 10, 100)) release := readers.Occupy(0) - if v, found, certain := s.Lookup(1, 10, 1); v != 0 || found || certain { + if v, found, certain := s.Lookup(1, 10, 1, false); v != 0 || found || certain { t.Errorf("Lookup of a key that is there = %d, %t, %t without a slot to be had, want an uncertain miss", v, found, certain) } // The same hash, another key: only a look at the node could tell. - if v, found, certain := s.Lookup(1, 11, 1); v != 0 || found || certain { + if v, found, certain := s.Lookup(1, 11, 1, false); v != 0 || found || certain { t.Errorf("Lookup of a key with the hash of another = %d, %t, %t without a slot to be had, want an uncertain miss", v, found, certain) } - if v, found, certain := s.Lookup(2, 20, 1); v != 0 || found || !certain { + if v, found, certain := s.Lookup(2, 20, 1, false); v != 0 || found || !certain { t.Errorf("Lookup of a key that is not there = %d, %t, %t, want a certain miss", v, found, certain) } @@ -668,12 +668,12 @@ func TestLookupWithoutSlot(t *testing.T) { } release() - if v, found, certain := s.Lookup(1, 10, 1); v != 99 || !found || !certain { + if v, found, certain := s.Lookup(1, 10, 1, false); v != 99 || !found || !certain { t.Fatalf("Lookup = %d, %t, %t after the slots were given back", v, found, certain) } // A key with the same hash is told apart, and the value of the other // one does not leak out. - if v, found, certain := s.Lookup(1, 11, 1); v != 0 || found || !certain { + if v, found, certain := s.Lookup(1, 11, 1, false); v != 0 || found || !certain { t.Errorf("Lookup of a key with the hash of another = %d, %t, %t, want a certain miss", v, found, certain) } // No lookup leaves a node pinned. @@ -712,7 +712,7 @@ func TestLookupWhileReplaced(t *testing.T) { wg.Go(func() { n, u := 0, 0 for !stop.Load() { - switch _, found, certain := s.Lookup(7, 7, 1); { + switch _, found, certain := s.Lookup(7, 7, 1, false); { case !found && certain: t.Error("Lookup is certain that the key is missing, and it was there all along") return @@ -748,25 +748,25 @@ func TestLookupCounts(t *testing.T) { meta := &s.Nodes[idx].Meta for range 3 { - s.Lookup(1, 10, 0) + s.Lookup(1, 10, 0, false) } if got := meta.Load(); got != 0 { t.Errorf("Meta = %d after lookups with a limit of 0, want it untouched", got) } for i := range 5 { - s.Lookup(1, 10, 3) + s.Lookup(1, 10, 3, false) if got, want := meta.Load(), uint32(min(i+1, 3)); got != want { t.Errorf("Meta = %d after %d lookups with a limit of 3, want %d", got, i+1, want) } } // A limit below what has been counted leaves it alone as well. - s.Lookup(1, 10, 1) + s.Lookup(1, 10, 1, false) if got := meta.Load(); got != 3 { t.Errorf("Meta = %d after a lookup with a limit of 1, want 3", got) } // A miss counts nothing. - s.Lookup(1, 11, 3) - s.Lookup(2, 10, 3) + s.Lookup(1, 11, 3, false) + s.Lookup(2, 10, 3, false) if got := meta.Load(); got != 3 { t.Errorf("Meta = %d after misses, want 3", got) } @@ -784,34 +784,89 @@ func TestLookupCounts(t *testing.T) { return [2]uint32{same.Nodes[first].Meta.Load(), same.Nodes[second].Meta.Load()} } - if v, ok, _ := same.Lookup(7, 2, 3); !ok || v != 20 { + if v, ok, _ := same.Lookup(7, 2, 3, false); !ok || v != 20 { t.Fatalf("Lookup of the second of two keys with one hash = %d, %v", v, ok) } if got, want := counts(), [2]uint32{0, 1}; got != want { t.Errorf("counts = %v after a read of the second key, want %v", got, want) } - if _, ok, certain := same.Lookup(7, 3, 3); ok || !certain { + if _, ok, certain := same.Lookup(7, 3, 3, false); ok || !certain { t.Fatalf("Lookup of a key that is absent = %v, certain %v", ok, certain) } if got, want := counts(), [2]uint32{0, 1}; got != want { t.Errorf("counts = %v after a read of a key that is absent, want %v", got, want) } // Nor does a count that is there already go missing. - same.Lookup(7, 1, 3) - same.Lookup(7, 1, 3) - same.Lookup(7, 2, 3) - same.Lookup(7, 3, 3) + same.Lookup(7, 1, 3, false) + same.Lookup(7, 1, 3, false) + same.Lookup(7, 2, 3, false) + same.Lookup(7, 3, 3, false) if got, want := counts(), [2]uint32{2, 2}; got != want { t.Errorf("counts = %v after two reads of the first key and one more of the second, want %v", got, want) } // The same with a count that is at the limit of the read. - same.Lookup(7, 2, 2) - same.Lookup(7, 3, 2) + same.Lookup(7, 2, 2, false) + same.Lookup(7, 3, 2, false) if got, want := counts(), [2]uint32{2, 2}; got != want { t.Errorf("counts = %v after reads with a limit that has been reached, want %v", got, want) } } +// TestLookupTallies checks that the readers' tally of hits counts the lookups +// that found their key and were asked to count, and only those. +func TestLookupTallies(t *testing.T) { + t.Parallel() + + r := NewReaders() + var s, other Slab[int, int] + s.Init(8, 1, r) + other.Init(8, 1, r) + for _, sl := range []*Slab[int, int]{&s, &other} { + for _, key := range []int{1, 2} { + // Two keys with one hash: a lookup of one passes the other. + sl.PushFront(0, sl.Insert(7, key, 10*key)) + } + } + + s.Lookup(7, 1, 3, true) + s.Lookup(7, 2, 0, true) // Whatever the limit of the count on the node. + s.Lookup(7, 1, 3, false) + s.Lookup(7, 3, 3, true) // Absent, past two nodes with the same hash. + s.Lookup(8, 1, 3, true) // Absent, and no node with its hash. + other.Lookup(7, 2, 3, true) + if got := r.Hits(); got != 3 { + t.Errorf("Hits() = %d after three lookups that found their key and were to be tallied, want 3", got) + } + + // A reader that finds no slot to pin with reads as a writer instead, + // and the tally is the caller's business then. + release := r.Occupy(0) + if _, found, certain := s.Lookup(7, 1, 3, true); found || certain { + t.Fatalf("Lookup without a slot = %t, certain %t", found, certain) + } + release() + if got := r.Hits(); got != 3 { + t.Errorf("Hits() = %d after a lookup that found no slot, want 3", got) + } + + // Many readers at once lose no hit, whichever slots they share. + var wg sync.WaitGroup + for g := range 16 { + wg.Go(func() { + for i := range 1000 { + if _, found, _ := s.Lookup(7, 1+(g+i)%2, 3, true); !found { + t.Error("Lookup did not find a key with no writer about") + return + } + } + }) + } + wg.Wait() + if got := r.Hits(); got != 3+16*1000 { + t.Errorf("Hits() = %d after 16,000 more hits, want %d", got, 3+16*1000) + } +} + // TestReleasedMark checks that Released waits for what had left by the time // of its mark, and for nothing that left later. func TestReleasedMark(t *testing.T) { @@ -1406,7 +1461,7 @@ func TestConcurrentReaders(t *testing.T) { key := rng.IntN(keys) before := present[key].Load() - v, found, certain := s.Lookup(hashOf(key), key, 1) + v, found, certain := s.Lookup(hashOf(key), key, 1, false) if found && (v.a != key || v.c != v.a^v.b^sum) { t.Errorf("Lookup(%d) found the value %v", key, v) @@ -1460,7 +1515,7 @@ func TestNoAllocs(t *testing.T) { s.PushFront(0, s.Insert(uint32(i%3), key, value)) } for i, key := range keys { - s.Lookup(uint32(i%3), key, 1) + s.Lookup(uint32(i%3), key, 1, false) idx := s.Replace(s.Find(uint32(i%3), key), key, value) s.Unlink(idx) diff --git a/internal/cslab/readers.go b/internal/cslab/readers.go index ccda57e..4196548 100644 --- a/internal/cslab/readers.go +++ b/internal/cslab/readers.go @@ -87,7 +87,13 @@ type Slot struct { // that its reader is looking at: the number of the slab in the upper // half, which is never zero, and the index of the node in the lower. pin atomic.Uint64 - _ [cacheLine - 8]byte + + // hits counts the lookups that found their key with this slot and were + // asked to say so; see [Slab.Lookup]. Whoever has used the slot has + // added to it. It is in the line that a reader writes to anyway. + hits atomic.Uint64 + + _ [cacheLine - 16]byte } // NewReaders returns a Readers for a new cache. Its slots take up a few @@ -166,6 +172,18 @@ func (r *Readers) PinAny(node uint64) *Slot { // Unpin clears the slot. The node it named is no longer safe to look at. func (s *Slot) Unpin() { s.pin.Store(0) } +// Hits returns the number of lookups, by any slab of this Readers, that have +// found their key and were asked to tally it. It looks at every slot, which +// takes a while: a slot to every 128 bytes that [NewReaders] speaks of. A +// lookup that is under way may or may not have been counted. +func (r *Readers) Hits() uint64 { + var hits uint64 + for i := range r.slots { + hits += r.slots[i].hits.Load() + } + return hits +} + // Occupy takes the slots that are free and returns a function that gives them // back. In between, nobody can pin a node, and whoever wants to read has to // do it under a lock. If leave is not zero, one slot in every so many is left diff --git a/s3fifo/s3fifo.go b/s3fifo/s3fifo.go index d783659..446ab98 100644 --- a/s3fifo/s3fifo.go +++ b/s3fifo/s3fifo.go @@ -352,7 +352,7 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { // two of them may count as one. That is close enough. An entry that is // read all the time has reached the limit, and is not written to // again: its cache line stays shared between the CPUs. - value, ok, certain := s.slab.Lookup(hash, key, maxFrequency) + value, ok, certain := s.slab.Lookup(hash, key, maxFrequency, false) if s.slab.Overdue() { s.help() } @@ -378,7 +378,7 @@ func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) - value, ok, certain := s.slab.Lookup(hash, key, 0) + value, ok, certain := s.slab.Lookup(hash, key, 0, false) if s.slab.Overdue() { s.help() } diff --git a/sieve/sieve.go b/sieve/sieve.go index 3d9a3fa..e07983b 100644 --- a/sieve/sieve.go +++ b/sieve/sieve.go @@ -184,7 +184,7 @@ func (c *Cache[K, V]) Get(key K) (value V, ok bool) { // The lookup marks the entry, unless it is marked already: popular // entries are, and not writing to them again keeps their cache line // shared between the CPUs that read it. - value, ok, certain := s.slab.Lookup(hash, key, visited) + value, ok, certain := s.slab.Lookup(hash, key, visited, false) if s.slab.Overdue() { s.help() } @@ -210,7 +210,7 @@ func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { h := maphash.Comparable(c.seed, key) s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) - value, ok, certain := s.slab.Lookup(hash, key, 0) + value, ok, certain := s.slab.Lookup(hash, key, 0, false) if s.slab.Overdue() { s.help() } From 619257f99a5687d2023391b40d315092c8307682 Mon Sep 17 00:00:00 2001 From: Vsevolod Strukchinsky Date: Wed, 11 Nov 2026 21:27:18 +0500 Subject: [PATCH 2/4] Add a frequency sketch internal/sketch is the count-min sketch of TinyLFU in Caffeine's layout: four 4-bit counters per key, in four words of one 64-byte block, sixteen counters to a word and a word for every entry of the cache. Counters stop at fifteen and are halved, with the tally of what they have counted, whenever that tally reaches the sample. Co-Authored-By: Claude Opus 5.5 --- internal/sketch/sketch.go | 163 ++++++++++++++++ internal/sketch/sketch_test.go | 346 +++++++++++++++++++++++++++++++++ 2 files changed, 509 insertions(+) create mode 100644 internal/sketch/sketch.go create mode 100644 internal/sketch/sketch_test.go diff --git a/internal/sketch/sketch.go b/internal/sketch/sketch.go new file mode 100644 index 0000000..54d776e --- /dev/null +++ b/internal/sketch/sketch.go @@ -0,0 +1,163 @@ +// Package sketch provides the memory that a cache has of how often keys have +// been asked for of late, whether it holds them or not: the frequency sketch +// of TinyLFU. +// +// It is a count-min sketch. A key has four counters among many, picked by its +// hash, and shares each of them with other keys. Counting a request adds to +// all four, and the lowest of the four is the estimate: it is never too low, +// and too high only if every one of the four has been pushed up by other +// keys. A counter has four bits and stops at fifteen, and all counters are +// halved every so often, so that what was popular once does not stay so for +// good. +// +// The layout is the one that Caffeine uses: sixteen counters to a word, eight +// words to a block, and the four counters of a key in four different words of +// one block, which is one cache line. +// +// TinyLFU is described in "TinyLFU: A Highly Efficient Cache Admission Policy" +// by Einziger, Friedman and Manes (ACM Transactions on Storage, 2017). +package sketch + +import "math/bits" + +// Most is as far as a counter goes. +const Most = 15 + +// A word holds sixteen counters and a block eight words. +const ( + wordsPerBlock = 8 + + // odd has the lowest bit of every counter in a word, and halved all the + // bits that a counter may have set once it has been halved. + odd = 0x1111111111111111 + halved = 0x7777777777777777 +) + +// Sketch counts requests for keys, by their hashes. It does not allocate once +// it has been initialized. A Sketch is not safe for concurrent use. +type Sketch struct { + table []uint64 + mask uint64 // number of blocks minus one + + // size is what has been added since the counters were last halved, + // and sample how much may be added before they are halved again. + size, sample int +} + +// Init readies the sketch for a cache of the given capacity and empties it. +// It gets a word for every entry of the cache, which is sixteen counters, +// rounded up to a whole number of blocks that is a power of two. It halves its +// counters, and with them its tally of the requests counted, whenever that +// tally reaches sample: the first time after sample requests, and after about +// half as many from then on. It panics if capacity is negative or sample is +// less than one. +func (s *Sketch) Init(capacity, sample int) { + if capacity < 0 || sample < 1 { + panic("sketch: invalid capacity or sample") + } + + blocks := uint64(1) + if words := uint64(capacity); words > wordsPerBlock { + blocks = 1 << bits.Len64((words-1)/wordsPerBlock) + } + s.table = make([]uint64, blocks*wordsPerBlock) + s.mask = blocks - 1 + s.size, s.sample = 0, sample +} + +// spread scatters the bits of a hash, so that keys whose hashes have bits in +// common, as those of one shard of a cache have, do not share their counters +// for it. +func spread(x uint64) uint64 { + x = (x ^ x>>30) * 0xbf58476d1ce4e5b9 + x = (x ^ x>>27) * 0x94d049bb133111eb + return x ^ x>>31 +} + +// A place is where one of the four counters of a key is: the word, and how +// far into it the counter begins. +type place struct { + word uint64 + shift uint +} + +// places returns where the counters of the key with the given hash are. The +// upper half of the scattered hash picks the block. Of the lower half, each +// byte picks a counter: its lowest bit one of two words that are that +// counter's alone, and the next four bits the counter in the word. +func (s *Sketch) places(hash uint64) [4]place { + h := spread(hash) + block := (h >> 32 & s.mask) * wordsPerBlock + var p [4]place + for i := range uint64(4) { + b := h >> (8 * i) + p[i] = place{word: block + 2*i + b&1, shift: uint(b >> 1 & 15 * 4)} + } + return p +} + +// Estimate returns how often the key with the given hash has been counted of +// late, or more: the lowest of its four counters. +func (s *Sketch) Estimate(hash uint64) int { + least := uint64(Most) + for _, p := range s.places(hash) { + least = min(least, s.table[p.word]>>p.shift&Most) + } + return int(least) +} + +// Add counts n more requests for the key with the given hash. A counter that +// has reached [Most] stays there, and if all four have, nothing is counted. +func (s *Sketch) Add(hash uint64, n int) { + if n <= 0 { + return + } + added := false + for _, p := range s.places(hash) { + if count := s.table[p.word] >> p.shift & Most; count < Most { + s.table[p.word] += min(uint64(n), Most-count) << p.shift + added = true + } + } + if !added { + return + } + + s.size += n + if s.size >= s.sample { + s.halve() + } +} + +// halve halves every counter, rounding down, and with them the tally of what +// has been added, less what the rounding took: a quarter of a request for +// every counter that was odd, since a request is in four counters. +func (s *Sketch) halve() { + odds := 0 + for i, word := range s.table { + odds += bits.OnesCount64(word & odd) + s.table[i] = word >> 1 & halved + } + s.size = (s.size - odds/4) / 2 +} + +// Shared reports whether the keys with the two hashes share any of their +// counters. It is for the tests of a cache, which need keys that do not: of +// those the sketch knows exactly how often each one was counted. +func (s *Sketch) Shared(a, b uint64) bool { + pa, pb := s.places(a), s.places(b) + for i := range pa { + // The counters of a key are in four pairs of words, one in each, + // so only those with the same number can be the same. + if pa[i] == pb[i] { + return true + } + } + return false +} + +// Reset empties the sketch. +func (s *Sketch) Reset() { + clear(s.table) + s.size = 0 +} diff --git a/internal/sketch/sketch_test.go b/internal/sketch/sketch_test.go new file mode 100644 index 0000000..366c1dc --- /dev/null +++ b/internal/sketch/sketch_test.go @@ -0,0 +1,346 @@ +package sketch + +import ( + "math/rand/v2" + "testing" +) + +// model is a Sketch at its simplest: a counter in a byte for every place +// that a key can have one, and the arithmetic spelled out. +type model struct { + s *Sketch // for where the counters of a key are, which is not the model's business + counters map[[2]uint64]int + size int +} + +func (m *model) estimate(hash uint64) int { + least := Most + for _, p := range m.s.places(hash) { + least = min(least, m.counters[[2]uint64{p.word, uint64(p.shift)}]) + } + return least +} + +func (m *model) add(hash uint64, n int) { + if n <= 0 { + return + } + added := false + for _, p := range m.s.places(hash) { + at := [2]uint64{p.word, uint64(p.shift)} + if m.counters[at] < Most { + m.counters[at] = min(m.counters[at]+n, Most) + added = true + } + } + if !added { + return + } + m.size += n + if m.size >= m.s.sample { + odds := 0 + for at, count := range m.counters { + odds += count % 2 + m.counters[at] = count / 2 + } + m.size = (m.size - odds/4) / 2 + } +} + +func TestInit(t *testing.T) { + t.Parallel() + + // A word for every entry, in whole blocks, of which there is a power of + // two. + for capacity, words := range map[int]int{ + 0: 8, 1: 8, 8: 8, 9: 16, 16: 16, 17: 32, 64: 64, 65: 128, 1000: 1024, 1024: 1024, 1025: 2048, 100_000: 131_072, + } { + var s Sketch + s.Init(capacity, 10) + if len(s.table) != words || int(s.mask) != words/wordsPerBlock-1 { + t.Errorf("capacity %d: %d words in %d blocks, want %d words", capacity, len(s.table), s.mask+1, words) + } + } + + // Init empties a sketch that has been in use, the tally of what was + // added included. + var s Sketch + s.Init(64, 1000) + s.Add(1, 5) + s.Add(2, 7) + s.Init(64, 1000) + if s.size != 0 || s.Estimate(1) != 0 || s.Estimate(2) != 0 { + t.Errorf("after Init again: %d additions on record and estimates of %d and %d", s.size, s.Estimate(1), s.Estimate(2)) + } + + for name, init := range map[string]func(){ + "a negative capacity": func() { new(Sketch).Init(-1, 10) }, + "no sample": func() { new(Sketch).Init(10, 0) }, + } { + func() { + defer func() { + if recover() == nil { + t.Errorf("Init with %s did not panic", name) + } + }() + init() + }() + } +} + +// TestPlaces checks that the four counters of a key are in four words of one +// block, each with a pair of words to itself, and that keys spread over all +// there is, also when their hashes differ in a few bits only. +func TestPlaces(t *testing.T) { + t.Parallel() + + var s Sketch + s.Init(1024, 10) + + blocks, words, shifts := map[uint64]bool{}, map[uint64]bool{}, map[uint]bool{} + // first has the places that the keys of a block have their first + // counter in. + first := map[uint64]map[place]bool{} + for key := range uint64(20_000) { + // The hashes of the keys of a shard: the upper bits are those that + // picked the shard, and the same for all. + p := s.places(0xabcd<<48 | key) + block := p[0].word / wordsPerBlock + for i, at := range p { + if at.word/wordsPerBlock != block || at.word%wordsPerBlock/2 != uint64(i) { + t.Fatalf("key %d: counter %d is in word %d, want one of the pair %d of block %d", key, i, at.word, i, block) + } + if at.shift%4 != 0 || at.shift > 60 { + t.Fatalf("key %d: counter %d begins at bit %d", key, i, at.shift) + } + words[at.word], shifts[at.shift] = true, true + } + blocks[block] = true + if first[block] == nil { + first[block] = map[place]bool{} + } + first[block][p[0]] = true + } + // What picks the block of a key must not pick its counters as well, or + // the keys of a block would all have the same ones. Some 150 keys to a + // block leave few of the 32 places for a counter unused. + for block, places := range first { + if len(places) < 24 { + t.Fatalf("the keys of block %d have their first counter in %d places, want most of 32", block, len(places)) + } + } + if len(blocks) != 128 || len(words) != 1024 || len(shifts) != 16 { + t.Errorf("20,000 keys reach %d blocks, %d words and %d places in a word; want 128, 1024 and 16", + len(blocks), len(words), len(shifts)) + } +} + +// TestCounting follows single keys: what is added is what is estimated, up to +// the most that a counter holds, and a key that was never added is not there. +func TestCounting(t *testing.T) { + t.Parallel() + + var s Sketch + s.Init(1024, 1<<30) + + if got := s.Estimate(1); got != 0 { + t.Fatalf("Estimate of a key in an empty sketch = %d", got) + } + s.Add(1, 1) + s.Add(1, 2) + s.Add(1, 0) + s.Add(1, -5) + if got := s.Estimate(1); got != 3 { + t.Errorf("Estimate = %d after 1 and 2 were added, want 3", got) + } + if got := s.Estimate(2); got != 0 { + t.Errorf("Estimate of another key = %d, want 0", got) + } + s.Add(1, 100) + if got := s.Estimate(1); got != Most { + t.Errorf("Estimate = %d after far more was added than a counter holds, want %d", got, Most) + } + // Nothing spills over into the counters next to them. + for i, word := range s.table { + for shift := 0; shift < 64; shift += 4 { + if count := word >> shift & Most; count != 0 && count != Most { + t.Fatalf("the counter at bit %d of word %d is %d", shift, i, count) + } + } + } + size := s.size + s.Add(1, 1) + if s.size != size { + t.Errorf("a request that no counter had room for was counted: %d additions, then %d", size, s.size) + } + + s.Reset() + if got := s.Estimate(1); got != 0 || s.size != 0 { + t.Errorf("after Reset the estimate is %d and %d additions are on record", got, s.size) + } +} + +// TestHalving checks that the counters are halved when the sample is full, +// and the tally of additions with them. +func TestHalving(t *testing.T) { + t.Parallel() + + var s Sketch + s.Init(1024, 20) + s.Add(1, 7) + s.Add(2, 6) + s.Add(3, 6) // 19 so far. + if s.Estimate(1) != 7 || s.Estimate(2) != 6 || s.size != 19 { + t.Fatalf("before the sample is full: estimates %d and %d, %d additions", s.Estimate(1), s.Estimate(2), s.size) + } + s.Add(4, 3) // 22, which is over the sample. + // Seven becomes three, six three, and three one. The eight counters of + // the first and the last key were odd: rounding took two requests, a + // quarter of one from each of them. That leaves 20 to be halved. + if got := [4]int{s.Estimate(1), s.Estimate(2), s.Estimate(3), s.Estimate(4)}; got != [4]int{3, 3, 3, 1} { + t.Errorf("after halving the estimates are %v, want [3 3 3 1]", got) + } + if s.size != 10 { + t.Errorf("after halving %d additions are on record, want 10", s.size) + } +} + +// TestAgainstModel adds and estimates at random, with few enough places for +// keys to share counters and a sample that fills up many times. +func TestAgainstModel(t *testing.T) { + t.Parallel() + + for _, tt := range []struct{ capacity, sample, keys int }{ + {1, 50, 40}, {8, 100, 200}, {64, 640, 300}, {64, 100, 3000}, {1000, 10_000, 5000}, + } { + var s Sketch + s.Init(tt.capacity, tt.sample) + m := model{s: &s, counters: map[[2]uint64]int{}} + rng := rand.New(rand.NewPCG(uint64(tt.capacity), uint64(tt.sample))) + + for step := range 100_000 { + // Some keys are far more popular than others. + hash := uint64(rng.IntN(tt.keys)) * 0x9e3779b97f4a7c15 + if rng.IntN(4) == 0 { + hash = uint64(rng.IntN(1 + tt.keys/50)) + } + if rng.IntN(3) == 0 { + if got, want := s.Estimate(hash), m.estimate(hash); got != want { + t.Fatalf("capacity %d, step %d: Estimate(%#x) = %d, want %d", tt.capacity, step, hash, got, want) + } + continue + } + n := 1 + rng.IntN(3) + s.Add(hash, n) + m.add(hash, n) + if s.size != m.size { + t.Fatalf("capacity %d, step %d: %d additions on record, want %d", tt.capacity, step, s.size, m.size) + } + } + } +} + +// TestNeverTooLow checks the promise of a count-min sketch: until the +// counters are halved for the first time, no key is estimated at less than +// it was counted, as far as a counter goes. And with as many keys as it was +// made for, it is seldom too high. +func TestNeverTooLow(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + capacity, keys int + exact float64 // the share of estimates that are to be right + }{ + {256, 2000, 0}, // Eight times the keys that it was made for. + {2048, 2000, 0.95}, + } { + var s Sketch + s.Init(tt.capacity, 1<<30) + rng := rand.New(rand.NewPCG(3, 4)) + counted := map[uint64]int{} + for range 5000 { + hash := uint64(rng.IntN(tt.keys)) * 0x9e3779b97f4a7c15 + s.Add(hash, 1) + counted[hash]++ + } + exact := 0 + for hash, n := range counted { + got := s.Estimate(hash) + if got < min(n, Most) { + t.Fatalf("capacity %d: a key that was counted %d times is estimated at %d", tt.capacity, n, got) + } + if got == min(n, Most) { + exact++ + } + } + if float64(exact) < tt.exact*float64(len(counted)) { + t.Errorf("capacity %d: %d of %d estimates are exact, want %.0f%% at least", tt.capacity, exact, len(counted), 100*tt.exact) + } + } +} + +// TestShared checks that two keys are said to share a counter exactly when +// counting one of them shows in a counter of the other. +func TestShared(t *testing.T) { + t.Parallel() + + var s Sketch + s.Init(16, 1<<30) // Two blocks: keys meet often. + rng := rand.New(rand.NewPCG(7, 8)) + shared := 0 + for range 20_000 { + a, b := rng.Uint64(), rng.Uint64() + s.Reset() + s.Add(a, 1) + touched := false + for _, p := range s.places(b) { + touched = touched || s.table[p.word]>>p.shift&Most != 0 + } + if got := s.Shared(a, b); got != touched || got != s.Shared(b, a) { + t.Fatalf("Shared(%#x, %#x) = %t and %t the other way round, want %t", a, b, got, s.Shared(b, a), touched) + } + if touched { + shared++ + } + } + if !s.Shared(1, 1) { + t.Error("a key does not share its counters with itself") + } + // Half the pairs are in the same block, and of those about one in + // eight has a counter in common. + if shared < 800 || shared > 1700 { + t.Errorf("%d of 20,000 pairs of keys share a counter, want about 1,200", shared) + } +} + +// TestNoAllocs is not parallel: counting allocations needs the process to +// itself. +func TestNoAllocs(t *testing.T) { + var s Sketch + s.Init(1000, 500) + hash, step, sum := uint64(0), uint64(0x9e3779b97f4a7c15), 0 + allocs := testing.AllocsPerRun(1, func() { + for range 10_000 { + hash += step + s.Add(hash, 1) + sum += s.Estimate(hash - 10*step) + } + s.Reset() + }) + if allocs != 0 { + t.Errorf("%v allocations, want 0", allocs) + } + _ = sum +} + +func BenchmarkAddAndEstimate(b *testing.B) { + var s Sketch + s.Init(1<<20, 10<<20) + hash, step, sum := uint64(0), uint64(0x9e3779b97f4a7c15), 0 + for b.Loop() { + hash += step + s.Add(hash, 1) + sum += s.Estimate(hash - 1000*step) + } + _ = sum +} From 5cc2e7feb1eec5760c76cdf1431c56a3639ea917 Mon Sep 17 00:00:00 2001 From: Vsevolod Strukchinsky Date: Wed, 11 Nov 2026 21:27:18 +0500 Subject: [PATCH 3/4] Add tinylfu: TinyLFU admission over S3-FIFO's queues A new entry waits in a small queue, as in S3-FIFO. When its turn comes it gets into the main queue only if the sketch has its key down as asked for more often of late than the entry that the main queue would evict next; otherwise it goes. The main queue gives an entry that was read another round, up to three in a row. Reads take no lock: they count on the entry up to three, and the next write that comes by the entry tells the sketch, which only writers touch. The size of the small queue is found by Caffeine's hill climber, one for the whole cache. It goes by the share of requests answered, counted exactly: readers tally their hits in their slots, and shards hand on their misses, in batches when there are several of them. It starts at a hundredth of the room and stays between that and four fifths. The cache is compared step by step with a model of the policy, fuzzed against it, and checked under the race detector, also while the window moves and Clear runs. Co-Authored-By: Claude Opus 5.5 --- Makefile | 1 + evict.go | 8 +- internal/inlinecheck/inline_test.go | 7 +- internal/inlinecheck/probe/tinylfu/probe.go | 21 + tinylfu/bench_test.go | 139 +++ tinylfu/climber.go | 226 ++++ tinylfu/climber_test.go | 187 ++++ tinylfu/example_test.go | 47 + tinylfu/export_test.go | 144 +++ tinylfu/shards_test.go | 560 +++++++++ tinylfu/tinylfu.go | 653 +++++++++++ tinylfu/tinylfu_test.go | 1122 +++++++++++++++++++ 12 files changed, 3111 insertions(+), 4 deletions(-) create mode 100644 internal/inlinecheck/probe/tinylfu/probe.go create mode 100644 tinylfu/bench_test.go create mode 100644 tinylfu/climber.go create mode 100644 tinylfu/climber_test.go create mode 100644 tinylfu/example_test.go create mode 100644 tinylfu/export_test.go create mode 100644 tinylfu/shards_test.go create mode 100644 tinylfu/tinylfu.go create mode 100644 tinylfu/tinylfu_test.go diff --git a/Makefile b/Makefile index 3bf19e5..df443f5 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ fuzz: ## Fuzz every target for FUZZTIME (default 30s) each. $(GO) test -run='^$$' -fuzz=FuzzAgainstModel -fuzztime=$(FUZZTIME) ./twoq $(GO) test -run='^$$' -fuzz=FuzzAgainstModel -fuzztime=$(FUZZTIME) ./sieve $(GO) test -run='^$$' -fuzz=FuzzAgainstModel -fuzztime=$(FUZZTIME) ./s3fifo + $(GO) test -run='^$$' -fuzz=FuzzAgainstModel -fuzztime=$(FUZZTIME) ./tinylfu cd bench && $(GO) test -run='^$$' -fuzz=FuzzRead -fuzztime=$(FUZZTIME) ./trace .PHONY: bench diff --git a/evict.go b/evict.go index f7c6bc7..b635479 100644 --- a/evict.go +++ b/evict.go @@ -10,6 +10,11 @@ // - [github.com/floatdrop/evict/s3fifo] implements S3-FIFO, which lets new // entries prove themselves in a small queue before they get into the // main one, and reads without taking a lock as well. +// - [github.com/floatdrop/evict/tinylfu] admits with TinyLFU: an entry gets +// past the small queue only if its key has been asked for more often of +// late than the entry it would replace, by a record that the cache keeps +// of keys whether it holds them or not. It finds the size of the small +// queue by itself, and reads without taking a lock as well. // // Every cache is safe for concurrent use and keeps its entries in arrays that // it allocates when it is created and addresses by 32-bit indices. Getting, @@ -42,7 +47,8 @@ package evict import "iter" // Cache is the behaviour common to every cache in this module. It is -// implemented by *lru.Cache, *twoq.Cache, *sieve.Cache and *s3fifo.Cache. +// implemented by *lru.Cache, *twoq.Cache, *sieve.Cache, *s3fifo.Cache and +// *tinylfu.Cache. // // All methods are safe for concurrent use. type Cache[K comparable, V any] interface { diff --git a/internal/inlinecheck/inline_test.go b/internal/inlinecheck/inline_test.go index f85e37e..8dffdad 100644 --- a/internal/inlinecheck/inline_test.go +++ b/internal/inlinecheck/inline_test.go @@ -25,14 +25,15 @@ import ( ) // hot matches the functions that every operation on a cache goes through. -var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo)\.\(\*(Cache|shard)\[.*\]\)\.(Get|Peek|Set|set|evict|Delete)$|` + +var hot = regexp.MustCompile(`/(lru|twoq|sieve|s3fifo|tinylfu)\.\(\*(Cache|shard)\[.*\]\)\.(Get|Peek|Set|set|evict|Delete|look|victim|promote|asked)$|` + `/c?slab\.\(\*Slab\[.*\]\)\.(Find|Lookup|Insert|Replace|Remove)$`) // slow matches the calls that must not show up in them: functions that are // small enough to be inlined, and are, as long as the compiler knows them. var slow = regexp.MustCompile(`^sync\.\(\*(RW)?Mutex\)\.|^sync/atomic\.|` + `slab\.\(\*Slab\[.*\]\)\.(MoveToFront|PushFront|Unlink|Len|Cap|Overdue|Hash)$|` + - `cslab\.\(\*(Readers|Slot)\)\.(Pin|Unpin)$|/s3fifo\.(tagOf|ghostOf)$`) + `cslab\.\(\*(Readers|Slot)\)\.(Pin|Unpin)$|/s3fifo\.(tagOf|ghostOf)$|` + + `/tinylfu\.(tagOf|kept|keep|printOf|printIn)$|/tinylfu\.\(\*climber\)\.(hit|miss)$`) func TestHotPathsAreInlined(t *testing.T) { t.Parallel() @@ -100,7 +101,7 @@ func TestHotPathsAreInlined(t *testing.T) { } // The test is only worth something if it has looked at the right code. - for _, pkg := range []string{"lru", "twoq", "sieve", "s3fifo"} { + for _, pkg := range []string{"lru", "twoq", "sieve", "s3fifo", "tinylfu"} { for _, method := range []string{"Get", "Set", "Peek", "Delete"} { found := false for name := range seen { diff --git a/internal/inlinecheck/probe/tinylfu/probe.go b/internal/inlinecheck/probe/tinylfu/probe.go new file mode 100644 index 0000000..c0cabac --- /dev/null +++ b/internal/inlinecheck/probe/tinylfu/probe.go @@ -0,0 +1,21 @@ +// Package probe uses a cache the way a program does: with key and value +// types of its own, and without importing anything else, not even another +// cache of this module. The test two directories up compiles it and looks at +// what the compiler made of it. +package probe + +import "github.com/floatdrop/evict/tinylfu" + +// Key is a key type that only this package instantiates the cache with. +type Key struct{ Tenant, ID string } + +// Value is a value type that only this package instantiates the cache with. +type Value struct{ Size, Hits int } + +// Use keeps the compiler from discarding the methods of the cache. +func Use(c *tinylfu.Cache[Key, Value], k Key, v Value) (Value, bool) { + c.Set(k, v) + c.Peek(k) + c.Delete(k) + return c.Get(k) +} diff --git a/tinylfu/bench_test.go b/tinylfu/bench_test.go new file mode 100644 index 0000000..03dbab9 --- /dev/null +++ b/tinylfu/bench_test.go @@ -0,0 +1,139 @@ +package tinylfu_test + +import ( + "math/rand/v2" + "strconv" + "testing" + + "github.com/floatdrop/evict/tinylfu" +) + +// These benchmarks measure single operations on a cache that fits in the CPU +// caches. For comparisons with other libraries and for larger workloads see +// the bench module. +// +// The cache is in one piece wherever a benchmark depends on what is cached: +// a cache with several shards evicts before it is full, and a hit would not +// always be one. + +const benchCapacity = 1 << 14 + +// benchKeys returns a power-of-two number of keys in random order, drawn from +// [0, benchCapacity). +func benchKeys() []int { + rng := rand.New(rand.NewPCG(1, 2)) + keys := make([]int, 1<<16) + for i := range keys { + keys[i] = rng.IntN(benchCapacity) + } + return keys +} + +func BenchmarkGet(b *testing.B) { + b.Run("hit", func(b *testing.B) { + c := tinylfu.New[int, int](benchCapacity, tinylfu.WithShards(1)) + for key := range benchCapacity { + c.Set(key, key) + } + keys := benchKeys() + + b.ReportAllocs() + i := 0 + for b.Loop() { + c.Get(keys[i&(len(keys)-1)]) + i++ + } + }) + + b.Run("miss", func(b *testing.B) { + c := tinylfu.New[int, int](benchCapacity, tinylfu.WithShards(1)) + for key := range benchCapacity { + c.Set(-key-1, key) + } + keys := benchKeys() + + b.ReportAllocs() + i := 0 + for b.Loop() { + c.Get(keys[i&(len(keys)-1)]) + i++ + } + }) + + b.Run("hit/string", func(b *testing.B) { + c := tinylfu.New[string, int](benchCapacity, tinylfu.WithShards(1)) + names := make([]string, benchCapacity) + for key := range benchCapacity { + names[key] = "a key of ordinary length " + strconv.Itoa(key) + c.Set(names[key], key) + } + keys := benchKeys() + + b.ReportAllocs() + i := 0 + for b.Loop() { + c.Get(names[keys[i&(len(keys)-1)]]) + i++ + } + }) +} + +func BenchmarkSet(b *testing.B) { + b.Run("update", func(b *testing.B) { + c := tinylfu.New[int, int](benchCapacity, tinylfu.WithShards(1)) + for key := range benchCapacity { + c.Set(key, key) + } + keys := benchKeys() + + b.ReportAllocs() + i := 0 + for b.Loop() { + c.Set(keys[i&(len(keys)-1)], i) + i++ + } + }) + + b.Run("evict", func(b *testing.B) { + c := tinylfu.New[int, int](benchCapacity, tinylfu.WithShards(1)) + for key := range benchCapacity { + c.Set(-key-1, key) + } + + b.ReportAllocs() + i := 0 + for b.Loop() { + c.Set(i, i) // Always a new key, so always an eviction. + i++ + } + }) +} + +// BenchmarkParallel has every CPU working on the same cache: three reads for +// every write, on keys of which a few are much more popular than the rest and +// a quarter does not fit. +func BenchmarkParallel(b *testing.B) { + const keySpace = 4 * benchCapacity + + c := tinylfu.New[int, int](benchCapacity) + zipf := rand.NewZipf(rand.New(rand.NewPCG(1, 2)), 1.01, 1, keySpace-1) + keys := make([]int, 1<<16) + for i := range keys { + keys[i] = int(zipf.Uint64()) + c.Set(keys[i], i) + } + + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + i := rand.Int() + for pb.Next() { + key := keys[i&(len(keys)-1)] + if i&3 == 0 { + c.Set(key, i) + } else { + c.Get(key) + } + i++ + } + }) +} diff --git a/tinylfu/climber.go b/tinylfu/climber.go new file mode 100644 index 0000000..a352536 --- /dev/null +++ b/tinylfu/climber.go @@ -0,0 +1,226 @@ +package tinylfu + +import ( + "math" + "sync" + "sync/atomic" + + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/internal/shards" +) + +// A climber finds the size that the small queue should have, by trying: it +// moves the size a step, sees whether the cache answers a larger share of its +// requests than before, and goes on in the same direction if so and back if +// not. The steps get shorter as it goes, so that it slows down near the top, +// and long again when the share of requests answered changes by much: the +// workload is another one then, and the search starts over. +// +// This is how Caffeine finds the size of its window, and the numbers are +// Caffeine's but for how often the size moves. It cannot do better than the +// share of requests answered lets it: where a larger small queue makes little +// difference, the noise in that share does, and the size wanders. +type climber struct { + // target is the size of the small queue, as a number of entries of a + // shard of the capacity that the climber was made for, before it is + // rounded down. step is how far it moves next, and by its sign which + // way, if things have got better by then; rate is what they have to be + // better than. + target, step, rate float64 + + // measured says that rate has been measured. The first measurement + // after the climber was made or told to start over sets it, and the + // size moves as if nothing had changed. + measured bool + + // The bounds of the target, and the step that the search starts with. + least, most, first float64 + + // period is the number of requests that the size is to stay put for. + // What the climber waits for is misses, though; see due. + period float64 +} + +const ( + // The small queue has this share of a shard to begin with, and never + // less, and never more than maxShare of it. + minShare = 0.01 + maxShare = 0.8 + + // The first step is this share of the capacity. Every step after it is + // shorter by the factor decay, until the share of requests answered + // changes by restart or more from one step to the next. + firstStep = 0.0625 + decay = 0.98 + restart = 0.05 + + // The size moves after about this many requests for every entry that + // the cache has room for. + periodPerEntry = 40 +) + +// init readies the climber for shards of the given capacity, of which there +// are as many as make up total. +func (c *climber) init(capacity, total int) { + n := float64(capacity) + *c = climber{ + least: max(minShare*n, 1), + most: max(maxShare*n, 1), + first: firstStep * n, + period: periodPerEntry * float64(total), + } + c.target, c.step = c.least, c.first +} + +// size returns the number of entries that the small queue of a shard of the +// capacity that the climber was made for is to have. +func (c *climber) size() int { return int(c.target) } + +// move takes a step, given how many of the requests since the last one the +// cache answered and how many it did not: on in the direction of the last +// step if the share of requests answered has grown since, or stayed the +// same, and back if it has shrunk. It returns the number of misses after +// which the next step is due. +func (c *climber) move(hits, misses uint64) (due uint64) { + rate := float64(hits) / float64(max(hits+misses, 1)) + if !c.measured { + c.rate, c.measured = rate, true + } + gain := rate - c.rate + c.rate = rate + + by := c.step + if gain < 0 { + by = -by + } + if math.Abs(gain) >= restart { + c.step = math.Copysign(c.first, by) + } else { + c.step = decay * by + } + c.target = min(max(c.target+by, c.least), c.most) + return c.due() +} + +// due returns the number of misses that come with a period's worth of +// requests if the cache answers the share of them that was measured last, or +// as many as there are requests if nothing has been measured yet. It is one at +// least. +func (c *climber) due() uint64 { + return uint64(max(c.period*(1-c.rate), 1)) +} + +// forget has the next measurement taken for what the ones after it are +// compared with, rather than compared with the last one. The size, the step +// and what due goes by stay as they are. +func (c *climber) forget() { c.measured = false } + +// A window is what the shards of a cache share of their small queues: the +// size that they are to have, and the climber that looks for it. +// +// The climber goes by the share of requests that the cache answers, of all +// shards together. A miss is a key that the cache is asked to put in and does +// not hold, as Caffeine has it: for a cache that is read through, that is +// the Set after a Get that found nothing, and the Get itself does not count. +// A shard counts its misses and the hits that it sees under its lock, and +// hands its counts on to the window every so often. The other hits are those +// of readers that take no lock and count in the slots of the cache's +// [cslab.Readers], which the window adds up when it is time to move. That is +// not often: adding up the slots takes a while. +type window struct { + // target holds the bits of climber.target, for the shards to read + // without taking mu. + target atomic.Uint64 + + _ shards.Pad + + // What the shards have handed on since the cache was made: the hits that + // they counted themselves and their misses. due is the number of misses + // at which the size moves next. + hits, misses, due atomic.Uint64 + + _ shards.Pad + + // mu is taken by a writer that moves the size, while it holds the lock + // of its own shard, and by Clear, which takes the locks of the shards + // while it holds mu. Writers only try for it, and give up if it is + // taken, so nobody waits for anybody in a circle. + mu sync.Mutex + climber climber + readers *cslab.Readers + + // The hits and misses that had been counted when the size last moved. + hitsThen, missesThen uint64 + + // base is the capacity of the largest shard, the one that the climber + // was made for. + base int +} + +// init readies the window for a cache whose shards have the given capacities +// at most, and which reads with the given readers. +func (w *window) init(base, total int, readers *cslab.Readers) { + w.base, w.readers = base, readers + w.climber.init(base, total) + w.target.Store(math.Float64bits(w.climber.target)) + w.due.Store(w.climber.due()) +} + +// sizeFor returns the number of entries that the small queue of a shard of +// the given capacity is to have: the climber's size, shrunk in proportion +// for a shard that is smaller than the largest one. It is one at least. +func (w *window) sizeFor(capacity int) int { + target := math.Float64frombits(w.target.Load()) + if capacity == w.base { + return int(target) + } + return max(int(target*float64(capacity)/float64(w.base)), 1) +} + +// note hands on what a shard has counted, and moves the size if that is due. +// The caller holds the lock of its shard. +func (w *window) note(hits, misses uint64) { + if hits != 0 { + w.hits.Add(hits) + } + // If somebody else has the window, another writer that is moving the + // size or Clear, the next batch tries again. + if w.misses.Add(misses) >= w.due.Load() && w.mu.TryLock() { + w.move() + w.mu.Unlock() + } +} + +// move moves the size if that is due. The caller holds mu, and has seen that +// a move is due; but another writer may have seen that too, and moved it in +// between. +func (w *window) move() { + misses := w.misses.Load() + if misses < w.due.Load() { + return + } + hits := w.readers.Hits() + w.hits.Load() + due := w.climber.move(hits-w.hitsThen, misses-w.missesThen) + w.hitsThen, w.missesThen = hits, misses + w.target.Store(math.Float64bits(w.climber.target)) + w.due.Store(misses + due) +} + +// forget makes the window measure afresh from here, and take the first +// measurement for what the ones after it are compared with. The size stays. +// The caller holds mu. +func (w *window) forget() { + w.hitsThen, w.missesThen = w.readers.Hits()+w.hits.Load(), w.misses.Load() + w.climber.forget() + w.due.Store(w.missesThen + w.climber.due()) +} + +// resize sets the size, as the climber might have; the search goes on from +// there. It is for tests. +func (w *window) resize(size int) { + w.mu.Lock() + defer w.mu.Unlock() + + w.climber.target = float64(size) + w.target.Store(math.Float64bits(w.climber.target)) +} diff --git a/tinylfu/climber_test.go b/tinylfu/climber_test.go new file mode 100644 index 0000000..b7c25c5 --- /dev/null +++ b/tinylfu/climber_test.go @@ -0,0 +1,187 @@ +package tinylfu + +import ( + "math" + "testing" +) + +// near reports whether two numbers are the same but for rounding. +func near(a, b float64) bool { return math.Abs(a-b) <= 1e-9*max(1, math.Abs(b)) } + +func TestClimberInit(t *testing.T) { + t.Parallel() + + // The small queue starts with a hundredth of the room, or with one + // entry, and stays between that and four fifths. The period is that of + // the whole cache, which may be in several shards. + for _, tt := range []struct { + capacity, total int + size int + least, most, first float64 + period float64 + }{ + {1, 1, 1, 1, 1, 0.0625, 40}, + {2, 2, 1, 1, 1.6, 0.125, 80}, + {100, 100, 1, 1, 80, 6.25, 4000}, + {250, 250, 2, 2.5, 200, 15.625, 10_000}, + {1000, 1000, 10, 10, 800, 62.5, 40_000}, + {1000, 7999, 10, 10, 800, 62.5, 319_960}, + } { + var c climber + c.rate, c.measured = 0.5, true // Whatever was there before is gone. + c.init(tt.capacity, tt.total) + if c.size() != tt.size || !near(c.target, tt.least) || !near(c.least, tt.least) || !near(c.most, tt.most) || + !near(c.first, tt.first) || !near(c.step, tt.first) || c.period != tt.period || c.measured || c.rate != 0 { + t.Errorf("capacity %d of %d: %+v, want %+v", tt.capacity, tt.total, c, tt) + } + // Until something has been measured, a period's worth of requests + // is a period's worth of misses. + if got := c.due(); got != uint64(tt.period) { + t.Errorf("capacity %d of %d: the first move is due after %d misses, want %v", tt.capacity, tt.total, got, tt.period) + } + } +} + +// feed has the climber move after a period of requests, of which the given +// share were answered, and returns the misses after which it moves next. +func feed(c *climber, rate float64) uint64 { + hits := uint64(math.Round(rate * c.period)) + return c.move(hits, uint64(c.period)-hits) +} + +// TestClimberMoves follows the size of the small queue through the answers +// that a cache might give. +func TestClimberMoves(t *testing.T) { + t.Parallel() + + var c climber + c.init(1000, 1000) + + for i, tt := range []struct { + rate float64 + target, step float64 // after the move + }{ + // The first measurement is what the next one is compared with. + // The size moves as if nothing had changed: up, the way it was + // set to go, with a step that is shorter next time. + {0.50, 72.5, 61.25}, + // Better: on, in steps that get shorter. + {0.52, 133.75, 60.025}, + // Worse: back. + {0.51, 73.725, -58.8245}, + // Better, so on in the direction of the last step, which is down. + {0.53, 14.9005, -57.64801}, + // No less than a hundredth of the room. + {0.54, 10, -56.4950498}, + // Much worse: back, and the steps are long again. + {0.40, 66.4950498, 62.5}, + // Much better: on, with long steps. + {0.50, 128.9950498, 62.5}, + // The same counts as better. + {0.50, 191.4950498, 61.25}, + } { + due := feed(&c, tt.rate) + if !near(c.target, tt.target) || !near(c.step, tt.step) || c.size() != int(tt.target) || !near(c.rate, tt.rate) || !c.measured { + t.Fatalf("move %d, to a rate of %v: target %v, size %d, next step %v, rate %v; want target %v, size %d, next step %v", + i, tt.rate, c.target, c.size(), c.step, c.rate, tt.target, int(tt.target), tt.step) + } + // The next move is due after the misses that a period brings at + // the rate that was just measured. + if want := 40_000 * (1 - tt.rate); math.Abs(float64(due)-want) > 1 { + t.Fatalf("move %d, to a rate of %v: the next move is due after %d misses, want %v", i, tt.rate, due, want) + } + } + + // As long as things get better it goes on, up to four fifths of the + // room and no further. + rate := 0.5 + for moves := 0; c.size() != 800; moves++ { + if moves == 100 { + t.Fatalf("after 100 steps up the size is %d, want 800", c.size()) + } + last := c.size() + rate += 0.001 + feed(&c, rate) + if c.size() <= last || c.size() > 800 { + t.Fatalf("a step up took the size from %d to %d", last, c.size()) + } + } + feed(&c, rate+0.001) + if c.size() != 800 || c.target != 800 { + t.Errorf("at the upper bound a step up leads to %d (%v), want 800", c.size(), c.target) + } +} + +// TestClimberRestart checks where long steps begin again: at a change of five +// points in the share of requests answered, and not below. +func TestClimberRestart(t *testing.T) { + t.Parallel() + + var c climber + c.init(1000, 1000) + feed(&c, 0.50) // What the next one is compared with: the step shrinks. + if !near(c.step, 61.25) { + t.Fatalf("after the first measurement the next step is %v, want 61.25", c.step) + } + feed(&c, 0.55) // Five points better: long again. + if !near(c.step, 62.5) { + t.Fatalf("after a gain of five points the next step is %v, want 62.5", c.step) + } + feed(&c, 0.5999) // A hair less than five points: shorter. + if !near(c.step, 61.25) { + t.Fatalf("after a gain of 4.99 points the next step is %v, want 61.25", c.step) + } + feed(&c, 0.5499) // Five points worse, all but a hair: back, and shorter. + if !near(c.step, -60.025) { + t.Fatalf("after a loss of 5.00 points less a hair the next step is %v, want -60.025", c.step) + } + feed(&c, 0.4999) // Five points worse: back, and long again. + if !near(c.step, 62.5) { + t.Fatalf("after a loss of five points the next step is %v, want 62.5", c.step) + } +} + +// TestClimberForget checks that a climber that is told to start over takes +// the next measurement for what to compare with, and keeps its size, its +// step, and what it knows of the share of requests that misses make up. +func TestClimberForget(t *testing.T) { + t.Parallel() + + var c climber + c.init(1000, 1000) + feed(&c, 0.5) + feed(&c, 0.9) // Much better: on, with a long step. + target, step := c.target, c.step + c.forget() + if c.measured || c.target != target || c.step != step { + t.Fatalf("after forget: %+v, want the target %v and the step %v, and nothing measured", c, target, step) + } + if got := c.due(); math.Abs(float64(got)-4000) > 1 { + t.Errorf("after forget the next move is due after %d misses, want 4000 as before", got) + } + // A much worse share of requests is not taken for a change for the + // worse: it is the first measurement since. + feed(&c, 0.3) + if !near(c.target, target+step) || !near(c.step, decay*step) { + t.Errorf("after forget and a share of 0.3 the target is %v and the step %v, want %v and %v", + c.target, c.step, target+step, decay*step) + } +} + +// TestClimberDue checks the bounds of the time between moves. +func TestClimberDue(t *testing.T) { + t.Parallel() + + var c climber + c.init(1000, 1000) + // A cache that answers nearly everything moves at the next miss. + c.move(1_000_000_000, 1) + if got := c.due(); got != 1 { + t.Errorf("at a rate of %v the next move is due after %d misses, want 1", c.rate, got) + } + // One that answers nothing waits for a whole period of misses. + c.move(0, 5) + if got := c.due(); got != 40_000 { + t.Errorf("at a rate of 0 the next move is due after %d misses, want 40,000", got) + } +} diff --git a/tinylfu/example_test.go b/tinylfu/example_test.go new file mode 100644 index 0000000..364f6f1 --- /dev/null +++ b/tinylfu/example_test.go @@ -0,0 +1,47 @@ +package tinylfu_test + +import ( + "fmt" + + "github.com/floatdrop/evict/tinylfu" +) + +func Example() { + cache := tinylfu.New[string, int](100) + + // "home" is asked for again and again, while pages that nobody comes + // back to pass through the cache and leave it alone. + cache.Set("home", 1) + for i := range 1000 { + cache.Get("home") + cache.Set(fmt.Sprint("page ", i), i) + } + + _, ok := cache.Get("home") + fmt.Println("home is cached:", ok) + _, ok = cache.Get("page 0") + fmt.Println("page 0 is cached:", ok) + + // Output: + // home is cached: true + // page 0 is cached: false +} + +// Set returns the entry that it pushed out of the cache, which is the place +// to release whatever that entry was holding on to. +func ExampleCache_Set() { + cache := tinylfu.New[string, string](1) + + cache.Set("a", "first") + + if key, value, displaced := cache.Set("a", "second"); displaced { + fmt.Printf("replaced %s=%s\n", key, value) + } + if key, value, displaced := cache.Set("b", "third"); displaced { + fmt.Printf("evicted %s=%s\n", key, value) + } + + // Output: + // replaced a=first + // evicted a=second +} diff --git a/tinylfu/export_test.go b/tinylfu/export_test.go new file mode 100644 index 0000000..0bfe56d --- /dev/null +++ b/tinylfu/export_test.go @@ -0,0 +1,144 @@ +package tinylfu + +import ( + "hash/maphash" + "slices" +) + +// OccupyReaders takes the slots in which the readers of the cache pin what +// they look at, save for one in every leave of them if leave is not zero, and +// returns a function that gives them back. A reader that finds no slot has to +// take the lock of the shard. +func (c *Cache[K, V]) OccupyReaders(leave int) (release func()) { + return c.readers.Occupy(leave) +} + +// Hold finds the entry for key like a reader does and keeps looking at it +// until the function that it returns is called: a reader that is held up at +// the worst moment. It reports whether there was an entry to look at. +func (c *Cache[K, V]) Hold(key K) (release func(), ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + + // A reader finds the entry and pins it without the lock. With the + // lock, there is no doubt about what it has found. + s.mu.Lock() + defer s.mu.Unlock() + + idx := s.slab.Find(uint32(h), key) + if idx == 0 { + return func() {}, false + } + pin := s.slab.Pin(idx) + if pin == nil { + return func() {}, false + } + return pin.Unpin, true +} + +// Waits returns how often a writer to the cache has had to wait for a reader +// to let go of an entry. +func (c *Cache[K, V]) Waits() (waits uint64) { + for i := range c.shards { + s := &c.shards[i] + s.mu.Lock() + waits += s.slab.Waits() + s.mu.Unlock() + } + return waits +} + +// Queues returns the keys in the main queue and in the small queue of the +// first shard, each from the newest to the oldest. +func (c *Cache[K, V]) Queues() (inMain, inSmall []K) { + s := &c.shards[0] + s.mu.Lock() + defer s.mu.Unlock() + + nodes, links := s.slab.Nodes, s.slab.Links + for idx := links[main].Next; idx != main; idx = links[idx].Next { + inMain = append(inMain, nodes[idx].Key) + } + for idx := links[small].Next; idx != small; idx = links[idx].Next { + inSmall = append(inSmall, nodes[idx].Key) + } + return inMain, inSmall +} + +// Print returns what the sketch of a shard knows key by. +func (c *Cache[K, V]) Print(key K) uint64 { + return printOf(maphash.Comparable(c.seed, key)) +} + +// Asked returns how often key has been asked for of late, by the sketch of +// its shard. +func (c *Cache[K, V]) Asked(key K) int { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + return s.seen.Estimate(printOf(h)) +} + +// Reads returns the reads that the entry for key has counted since a write +// last looked at it. +func (c *Cache[K, V]) Reads(key K) (reads uint32, ok bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + s.mu.Lock() + defer s.mu.Unlock() + + idx := s.slab.Find(uint32(h), key) + if idx == 0 { + return 0, false + } + return s.slab.Nodes[idx].Meta.Load(), true +} + +// Window returns the number of entries that the small queue of the first +// shard is to have. +func (c *Cache[K, V]) Window() int { return c.window.sizeFor(c.shards[0].capacity) } + +// Resize gives the small queue of the first shard a share of n entries, and +// the others theirs in proportion, as the search for the right size might. +// The search goes on from there. +func (c *Cache[K, V]) Resize(n int) { c.window.resize(n) } + +// Twins returns two of the keys 0, 1, 2 and so on that the cache indexes +// under the same hash. A reader that looks for one of them comes across the +// other and can only tell them apart by the keys themselves. +func Twins[V any](c *Cache[int, V]) (first, second int) { + seen := make(map[uint32]int) + for key := 0; ; key++ { + hash := uint32(maphash.Comparable(c.seed, key)) + if other, ok := seen[hash]; ok { + return other, key + } + seen[hash] = key + } +} + +// Apart returns n of the keys 0, 1, 2 and so on of which no two share a +// counter in the sketch of the first shard, and none shares one with any of +// the keys given. Of such keys the sketch knows exactly how often each has +// been asked for, as long as no others are in use. The cache must be in one +// piece. +func Apart[V any](c *Cache[int, V], n int, others ...int) []int { + seen := &c.shards[0].seen + var prints []uint64 + for _, key := range others { + prints = append(prints, c.Print(key)) + } + + keys := make([]int, 0, n) + for key := 0; len(keys) < n; key++ { + if key == 1<<24 { + panic("tinylfu: the sketch has no room for that many keys with counters of their own") + } + p := c.Print(key) + if !slices.ContainsFunc(prints, func(q uint64) bool { return seen.Shared(p, q) }) { + keys, prints = append(keys, key), append(prints, p) + } + } + return keys +} diff --git a/tinylfu/shards_test.go b/tinylfu/shards_test.go new file mode 100644 index 0000000..1849741 --- /dev/null +++ b/tinylfu/shards_test.go @@ -0,0 +1,560 @@ +package tinylfu + +import ( + "hash/maphash" + "math" + "math/rand/v2" + "reflect" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/sketch" +) + +func TestShards(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + capacity int + opts []Option + want int + }{ + {"small caches stay in one piece", 511, nil, 1}, + {"no capacity", 0, []Option{WithShards(8)}, 1}, + {"as requested", 1000, []Option{WithShards(4)}, 4}, + {"rounded up to a power of two", 1000, []Option{WithShards(5)}, 8}, + {"limited to the capacity", 6, []Option{WithShards(64)}, 4}, + {"zero means the default", 100, []Option{WithShards(0)}, 1}, + {"the last option wins", 1000, []Option{WithShards(2), WithShards(16)}, 16}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + c := New[int, int](tt.capacity, tt.opts...) + + if got := len(c.shards); got != tt.want { + t.Errorf("%d shards, want %d", got, tt.want) + } + if got := int(c.mask) + 1; got != len(c.shards) { + t.Errorf("mask selects among %d shards, there are %d", got, len(c.shards)) + } + + total := 0 + for i := range c.shards { + total += c.shards[i].capacity + if got, want := c.shards[i].slab.Cap(), c.shards[i].capacity; got != want { + t.Errorf("shard %d has a slab for %d entries, want %d", i, got, want) + } + } + if want := max(tt.capacity, 0); total != want || c.Cap() != want { + t.Errorf("shards hold %d entries, Cap() = %d, want %d", total, c.Cap(), want) + } + }) + } +} + +func TestDefaultShards(t *testing.T) { + t.Parallel() + + c := New[int, int](1 << 20) + + if runtime.GOMAXPROCS(0) > 1 && len(c.shards) == 1 { + t.Error("a large cache has a single shard by default") + } + + // Every shard gets its share of the keys. With twice as many keys as + // there is room for, each of them is as good as certain to fill up. + for key := range 2 * c.Cap() { + c.Set(key, key) + } + for i := range c.shards { + s := &c.shards[i] + if got, want := s.slab.Len(), s.capacity; got < want*9/10 { + t.Errorf("shard %d holds %d entries of %d after %d insertions", i, got, want, 2*c.Cap()) + } + } +} + +// TestShardsShareTheWindow checks that the shards of a cache have one size +// for their small queues, in proportion to their capacities, and that what +// one of them is asked for moves it for all. +func TestShardsShareTheWindow(t *testing.T) { + t.Parallel() + + // Eight shards: four of 129 entries and four of 128. They hand on their + // counts every 16 misses. + c := New[int, int](1028, WithShards(8)) + // The sketch of a shard: for its entries, with the counters halved + // after ten requests for each. + for i := range c.shards { + s := &c.shards[i] + want := 129 - i/4 + var seen sketch.Sketch + seen.Init(want, 10*want) + if s.capacity != want || s.batch != 16 || s.win != &c.window || !reflect.DeepEqual(&s.seen, &seen) { + t.Errorf("shard %d holds %d entries and hands on its counts every %d misses; want %d and 16, and a sketch for as many entries", + i, s.capacity, s.batch, want) + } + } + if c.window.base != 129 || c.window.climber.period != 40*1028 { + t.Fatalf("the window is for shards of %d entries and moves every %v requests, want 129 and %d", + c.window.base, c.window.climber.period, 40*1028) + } + + sizes := func() (all [8]int) { + for i := range c.shards { + all[i] = c.window.sizeFor(c.shards[i].capacity) + } + return all + } + // A hundredth of 129 is less than one, and one is the least. + if got := sizes(); got != [8]int{1, 1, 1, 1, 1, 1, 1, 1} { + t.Fatalf("the small queues are to have %v entries, want one each", got) + } + c.Resize(100) + if got, want := sizes(), [8]int{100, 100, 100, 100, 99, 99, 99, 99}; got != want { + t.Fatalf("after Resize(100) the small queues are to have %v entries, want %v", got, want) + } + c.Resize(1) + + // Keys that all belong to the first shard move the size for all, once + // the misses that they bring reach a period's worth. The shard hands + // them on in batches, so that is up to a batch later. + sets := 0 + for key := 0; c.Window() == 1; key++ { + if h := maphash.Comparable(c.seed, key); uint32(h>>32)&c.mask == 0 { + c.Set(key, key) + sets++ + } + if sets > 40*1028+16 { + t.Fatalf("the size has not moved after %d misses, want it to after %d", sets, 40*1028) + } + } + if sets < 40*1028 || sets%16 != 0 { + t.Errorf("the size moved after %d misses, want %d or the next multiple of 16", sets, 40*1028) + } + // Nothing was read: a share of 0 answered, measured for the first time. + // The size has gone up by the first step, and moves next after a + // period's worth of misses. + // The first step is 129/16 entries: 9.0625 for the large shards, and + // in proportion 8.99 for the others. + if got := sizes(); got[0] != 9 || got[7] != 8 { + t.Errorf("after the first move the small queues are to have %v entries", got) + } + if got, want := c.window.due.Load(), uint64(sets)+40*1028; got != want { + t.Errorf("the next move is due at %d misses, want %d", got, want) + } + for i := 1; i < len(c.shards); i++ { + if s := &c.shards[i]; s.slab.Len() != 0 { + t.Errorf("shard %d holds %d entries, want none", i, s.slab.Len()) + } + } +} + +// TestBatches checks how many misses a shard keeps to itself before it hands +// them on: none in a cache in one piece, and in one in several shards one in +// eight of its entries, up to 32. +func TestBatches(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + capacity, shards int + batch uint64 + }{ + {100, 1, 1}, + {100_000, 1, 1}, + {4096, 8, 32}, + {1024, 4, 32}, + {1028, 8, 16}, + {1024, 64, 2}, + {64, 64, 1}, + } { + c := New[int, int](tt.capacity, WithShards(tt.shards)) + for i := range c.shards { + if got := c.shards[i].batch; got != tt.batch { + t.Errorf("capacity %d in %d shards: shard %d hands on its counts every %d misses, want %d", + tt.capacity, tt.shards, i, got, tt.batch) + } + } + } +} + +// TestClearKeepsBacklog checks that Clear leaves a shard that hands on its +// misses in batches with the ones that it has yet to hand on. The window's +// next measurement starts without them, as all of them start without some, +// and ends without some of its own. +func TestClearKeepsBacklog(t *testing.T) { + t.Parallel() + + c := New[int, int](1024, WithShards(4)) // Batches of 32. + s := &c.shards[0] + sets := 0 + for key := 0; sets < 5; key++ { + if h := maphash.Comparable(c.seed, key); uint32(h>>32)&c.mask == 0 { + c.Set(key, key) + sets++ + } + } + if s.misses != 5 || c.window.misses.Load() != 0 { + t.Fatalf("after 5 misses the shard holds back %d and the window has %d, want 5 and 0", s.misses, c.window.misses.Load()) + } + c.Clear() + if s.misses != 5 || c.window.missesThen != 0 { + t.Errorf("after Clear the shard holds back %d misses and the window measures from %d, want 5 and 0", s.misses, c.window.missesThen) + } +} + +// check fails the test if the window is not as it should be between moves: +// counted from no later than now, with the next move due a period after the +// last one, and the size within its bounds and as the shards see it. +func (w *window) check(t *testing.T) { + t.Helper() + + w.mu.Lock() + defer w.mu.Unlock() + + hits, misses, due := w.readers.Hits()+w.hits.Load(), w.misses.Load(), w.due.Load() + if w.hitsThen > hits || w.missesThen > misses { + t.Errorf("the window measures from %d hits and %d misses, of which there have been %d and %d", w.hitsThen, w.missesThen, hits, misses) + } + if due-w.missesThen != w.climber.due() { + t.Errorf("the next move is due at %d misses, %d after the last one, want %d after", due, due-w.missesThen, w.climber.due()) + } + if c := &w.climber; c.target < c.least || c.target > c.most { + t.Errorf("the target %v is outside [%v, %v]", c.target, c.least, c.most) + } + if got := math.Float64frombits(w.target.Load()); got != w.climber.target { + t.Errorf("the shards see a target of %v, the climber has %v", got, w.climber.target) + } +} + +// TestMovesOnce checks that a move that is due happens once, however many +// writers find it due before it has happened. +func TestMovesOnce(t *testing.T) { + t.Parallel() + + c := New[int, int](100, WithShards(1)) + w := &c.window + w.misses.Store(w.due.Load()) // Due, as two writers find it. + + w.mu.Lock() + defer w.mu.Unlock() + w.move() // The first of them moves it, + target, step, due := w.climber.target, w.climber.step, w.due.Load() + if !w.climber.measured || target == 1 { + t.Fatalf("the size has not moved: %+v", w.climber) + } + w.move() // and the second finds it moved. + if w.climber.target != target || w.climber.step != step || w.due.Load() != due { + t.Errorf("the size moved twice: to %v and then %v, with the next move due at %d and then %d", + target, w.climber.target, due, w.due.Load()) + } +} + +// TestWindowUnderLoad has goroutines read and write a cache, and clear it +// every so often, while the window is checked. Then a move that is due has +// to happen at the next batch of misses. +func TestWindowUnderLoad(t *testing.T) { + t.Parallel() + + for _, n := range []int{1, 8, 64} { + c := New[int, int](256, WithShards(n)) + w := &c.window + var ( + stop atomic.Bool + wg sync.WaitGroup + ) + for g := range 8 { + wg.Go(func() { + rng := rand.New(rand.NewPCG(uint64(g), 9)) + for !stop.Load() { + // Sometimes a few hot keys, sometimes many cold ones. + key := rng.IntN(300 + rng.IntN(3)*1000) + if _, ok := c.Get(key); !ok { + c.Set(key, key) + } + if rng.IntN(50) == 0 { + c.Set(key, 1) + } + } + }) + } + wg.Go(func() { + for !stop.Load() { + c.Clear() + time.Sleep(3 * time.Millisecond) + } + }) + for start := time.Now(); time.Since(start) < 300*time.Millisecond; time.Sleep(50 * time.Microsecond) { + w.check(t) + } + stop.Store(true) + wg.Wait() + w.check(t) + + // Nobody else is about now, so the fields are this goroutine's to + // read without the lock. + last, due := w.missesThen, w.due.Load() + for key := -1; w.missesThen == last; key-- { + c.Set(key, key) + if w.misses.Load() > due+uint64(n)*batchSize { + t.Fatalf("%d shards: no move at %d misses, though one was due at %d", n, w.misses.Load(), due) + } + } + // The move took everything that had been handed on into account. + w.check(t) + if w.missesThen != w.misses.Load() || w.missesThen < due { + t.Errorf("%d shards: the move was at %d misses, with %d handed on and the move due at %d", n, w.missesThen, w.misses.Load(), due) + } + } +} + +// TestWindowCounts checks what the window goes by: the hits that readers +// count in their slots, those counted under the lock of a shard, and the +// misses, all since the size last moved. +func TestWindowCounts(t *testing.T) { + t.Parallel() + + c := New[int, int](100, WithShards(1)) + w := &c.window + misses := func(n int) { + for range n { + c.Set(-1-int(w.misses.Load()), 0) + } + } + // Keys -1, -2 and so on. The newest of them is sure to be in the cache, + // in the small queue; others may be or not. + misses(3999) + if w.climber.measured { + t.Fatal("the size moved after 3999 misses, want 4000") + } + for range 7 { + c.Get(-3999) // Hits that readers count in their slots. + } + c.Set(-3999, 1) // A write to an entry that is there counts as a hit. + release := c.OccupyReaders(0) + c.Get(-3999) // As does a read that takes the lock. + c.Get(-10000) // Never there: no hit. + c.Peek(-3999) + release() + c.Peek(-3999) // Peeking counts for nothing. + misses(1) + if !w.climber.measured || !near(w.climber.rate, 9.0/4009) { + t.Fatalf("after 9 hits and 4000 misses the rate is %v (measured: %t), want %v", w.climber.rate, w.climber.measured, 9.0/4009) + } + + // The next period counts from there. + due := w.due.Load() + if want := uint64(4000 + 3991); due != want { // 4000 times 4000/4009, rounded down. + t.Fatalf("the next move is due at %d misses, want %d", due, want) + } + for range 5 { + c.Get(-4000) + } + misses(int(due - w.misses.Load())) + if !near(w.climber.rate, 5.0/float64(5+due-4000)) { + t.Errorf("after the next period the rate is %v, want %v", w.climber.rate, 5.0/float64(5+due-4000)) + } + + // Clear starts the measurements over, and keeps the size and what the + // last measurement says of the misses to wait for. + size, rate := c.Window(), w.climber.rate + for range 100 { + c.Get(-int(due)) // Hits before Clear, which do not count after it, + } + c.Set(-int(due), 2) // nor does one that the shard counted itself. + c.Clear() + if w.climber.measured || c.Window() != size { + t.Fatalf("after Clear the size is %d and something is measured (%t), want %d and nothing", c.Window(), w.climber.measured, size) + } + from, next := w.misses.Load(), w.due.Load() + if want := from + uint64(4000*(1-rate)); next != want { + t.Fatalf("after Clear at %d misses the next move is due at %d, want %d", from, next, want) + } + misses(1) + for range 3 { + c.Get(-int(from + 1)) + } + misses(int(next - w.misses.Load())) + if want := 3.0 / float64(3+next-from); !w.climber.measured || !near(w.climber.rate, want) { + t.Errorf("the first period after Clear: rate %v (measured: %t), want %v", w.climber.rate, w.climber.measured, want) + } +} + +// TestBookkeeping checks through a long run of operations that the counters +// of a shard agree with its lists, that every entry is tagged with the queue +// it is in and with what the sketch knows its key by, and that the queues +// and the counts stay within their bounds. +func TestBookkeeping(t *testing.T) { + t.Parallel() + + c := New[int, int](40, WithShards(1)) + s := &c.shards[0] + + count := func(head uint32) int { + n := 0 + for idx := s.slab.Links[head].Next; idx != head; idx = s.slab.Links[idx].Next { + tag := s.slab.Links[idx].Tag + if got := tag & 1; got != head { + t.Fatalf("node %d is in queue %d but tagged for queue %d", idx, head, got) + } + if reads := s.slab.Nodes[idx].Meta.Load(); reads > maxCount { + t.Fatalf("node %d has counted %d reads, want at most %d", idx, reads, maxCount) + } + // What the main queue leaves an entry with is one less than + // what it came by, and the small queue leaves nothing. + if left := kept(tag); left >= maxCount || head == small && left != 0 { + t.Fatalf("node %d in queue %d was left with a count of %d", idx, head, left) + } + if got, want := printIn(s.slab.Hash(idx), tag), printOf(maphash.Comparable(c.seed, s.slab.Nodes[idx].Key)); got != want { + t.Fatalf("node %d is tagged for the sketch as %#x, want %#x", idx, got, want) + } + n++ + } + return n + } + + rng := rand.New(rand.NewPCG(5, 6)) + deleted, promoted, over := [queues]int{}, 0, 0 + sizes := map[int]bool{} + for step := range 200_000 { + key := rng.IntN(120) + mains, evicted := s.mains, false + switch op := rng.IntN(100); { + case step%10_000 == 9_999: + c.Clear() + case op < 30: + old, _, displaced := c.Set(key, step) + evicted = displaced && old != key + case op < 90: + c.Get(rng.IntN(60)) // Some keys are read much more than others. + default: + if idx := s.slab.Find(uint32(maphash.Comparable(c.seed, key)), key); idx != 0 { + deleted[s.slab.Links[idx].Tag&1]++ + } + c.Delete(key) + } + if s.mains > mains { + promoted++ + } + + smalls, mains := count(small), count(main) + if smalls != s.smalls || mains != s.mains { + t.Fatalf("step %d: the queues hold %d and %d entries, the shard counts %d and %d", step, smalls, mains, s.smalls, s.mains) + } + if smalls+mains > s.capacity { + t.Fatalf("step %d: %d entries, want at most %d", step, smalls+mains, s.capacity) + } + if got := c.Len(); got != smalls+mains { + t.Fatalf("step %d: Len() = %d, want %d", step, got, smalls+mains) + } + size := s.win.sizeFor(s.capacity) + if size < 1 || size > 32 { + t.Fatalf("step %d: the small queue is to have %d entries, want 1 to 32", step, size) + } + sizes[size] = true + // An eviction leaves the small queue within its share. Without + // one it may be over: after deletions there is room that is + // nobody's. + if smalls > size { + if evicted { + t.Fatalf("step %d: the small queue holds %d entries after an eviction, want at most %d", step, smalls, size) + } + over++ + } + } + + // The run is only worth something if it went down every road. + if deleted[small] == 0 || deleted[main] == 0 || promoted == 0 || over == 0 || len(sizes) < 5 { + t.Errorf("%d and %d entries were deleted from the queues, %d moved to the main queue, the small queue was over its share %d times and had %d different shares; want some of each", + deleted[small], deleted[main], promoted, over, len(sizes)) + } +} + +// TestHashSplit checks that a shard does not index its keys by the same bits +// of the hash that put them into the shard. If it did, the keys of a shard +// would all ask for the same few slots of its index, and every lookup would +// have to wade through the others. +func TestHashSplit(t *testing.T) { + t.Parallel() + + // Many small shards: the index of a shard has fewer slots than there + // are shards, so bits that were shared would leave no choice at all. + const shardCount, perShard = 1024, 256 + + c := New[uint64, struct{}](shardCount*perShard, WithShards(shardCount)) + rng := rand.New(rand.NewPCG(1, 2)) + for range 2 * shardCount * perShard { + c.Set(rng.Uint64(), struct{}{}) + } + + worst := 0 + for i := range c.shards { + worst = max(worst, c.shards[i].slab.MaxDisplacement()) + } + // With hashes that are independent, an entry is rarely more than a + // dozen slots from where it belongs. With shared bits it is hundreds. + if worst > 64 { + t.Errorf("an entry is %d slots away from the slot its hash asks for", worst) + } +} + +// TestTags checks what a tag holds: the queue, a count, and enough of the +// hash to tell the sketch which key an entry has, whatever the other two +// are. +func TestTags(t *testing.T) { + t.Parallel() + + rng := rand.New(rand.NewPCG(3, 4)) + for range 10_000 { + h := rng.Uint64() + + // The sketch knows a key by its hash, less the three bits that a + // tag has no room for. + known := printOf(h) + if want := h>>35<<32 | h&(1<<32-1); known != want { + t.Fatalf("the sketch knows %#x by %#x, want %#x", h, known, want) + } + for bit := range 64 { + if other := printOf(h ^ 1<= 32 && bit < 35) { + t.Fatalf("with bit %d the other way, the sketch knows %#x by %#x and by %#x", bit, h, known, other) + } + } + + for _, queue := range []uint32{small, main} { + tag := tagOf(h, queue) + if tag&1 != queue || kept(tag) != 0 { + t.Fatalf("the tag of %#x for queue %d says queue %d and a count of %d", h, queue, tag&1, kept(tag)) + } + for count := range uint32(maxCount) { + // Whatever the count was before. + for before := range uint32(maxCount) { + with := keep(keep(tag, before), count) + if with&1 != queue || kept(with) != count || printIn(uint32(h), with) != known { + t.Fatalf("the tag of %#x for queue %d with a count of %d says queue %d, a count of %d and %#x for the sketch, want %#x", + h, queue, count, with&1, kept(with), printIn(uint32(h), with), known) + } + } + } + } + // Moving up is a matter of one bit. + if tagOf(h, small)|main != tagOf(h, main) { + t.Fatalf("the tags of %#x for the two queues differ in more than the queue", h) + } + } +} + +// TestShardPadding checks that the shards of a cache keep out of each other's +// cache lines. +func TestShardPadding(t *testing.T) { + t.Parallel() + + cachetest.Padded(t, shard[int, int]{}) + cachetest.Padded(t, shard[string, []byte]{}) +} diff --git a/tinylfu/tinylfu.go b/tinylfu/tinylfu.go new file mode 100644 index 0000000..8d4799c --- /dev/null +++ b/tinylfu/tinylfu.go @@ -0,0 +1,653 @@ +// Package tinylfu provides a fixed-capacity cache that admits with TinyLFU and +// whose reads take no lock. +// +// The cache runs two queues, each in the order of arrival, as S3-FIFO does. A +// new entry goes into a small queue. When it has got to the end of that, it +// is up against the entry that the main queue would evict next, and the +// cache keeps the one that has been asked for more often of late; the other +// one goes. The main queue gives an entry that has been read since it was +// last looked at another round rather than putting it up for eviction, up to +// three times in a row for an entry that is read a lot. +// +// How often a key has been asked for is a matter of record, whether the key +// is in the cache or not: the cache keeps a frequency sketch, a table of small +// counters that keys share and that are halved every so often. So an entry +// that keeps coming back at long intervals gets in, where a queue of ghosts +// would long have forgotten it, and a flood of keys that are asked for once +// does not push out what is known to be in demand. This is the admission +// policy of TinyLFU, and the arrangement of a small queue, or window, in +// front of it is W-TinyLFU. +// +// How much of the room the small queue should have depends on the workload: +// a large one suits keys that are asked for again soon or never, a small one +// leaves more for what has proven itself. The cache finds out by trying. It +// starts with a hundredth, moves the share a step, and goes on in the same +// direction for as long as it answers more requests than before, and back +// when it answers fewer. The shards of a cache share what it finds. +// +// The search needs a slope to climb, and a clear one. Where a somewhat larger +// small queue answers about as many requests as the one the cache has, the +// share that it answers moves more by chance than by the size, and the size +// wanders, at some cost. That is the case when every key comes back a time or +// two after much the same while and then never again, and the while is longer +// than a key lasts in the small queue the cache has found so far: only a far +// larger one would do, and the sketch, which counts how often, has nothing to +// say about how recently. On a workload like that, the cache can answer far +// fewer requests than an LRU cache of the same size, and the lru package is +// the better choice. And when the workload changes from one phase to the +// next, the search starts over with long steps each time. +// +// Reading an entry does no more than count on it, up to three and no further, +// and add one to a tally of the requests answered that the reader keeps in a +// place it writes to anyway, which few other readers use at the same time. +// The sketch learns of the reads when a write next looks at the entry. So any +// number of goroutines can read at the same time without getting in each +// other's way, even when they read the same entry. +// +// TinyLFU and W-TinyLFU are described in "TinyLFU: A Highly Efficient Cache +// Admission Policy" by Einziger, Friedman and Manes (ACM Transactions on +// Storage, 2017), and the search for the size of the window in "Adaptive +// Software Cache Management" by Einziger, Eytan, Friedman and Manes +// (Middleware '18). What is described there keeps its entries in the order in +// which they were last used and counts every read in the sketch at once. This +// cache keeps them in the order of arrival and counts later, which is what +// lets it read without a lock; the queues are those of S3-FIFO ("FIFO queues +// are all you need for cache eviction" by Yang, Zhang, Qiu, Yue and Rashmi, +// SOSP '23). +// +// # Memory +// +// So that readers need no lock, an entry that leaves the cache is not taken +// apart right away: a reader may be looking at it. Its key and value stay +// referenced until the cache has made sure that nobody is, which it does for +// many entries at a time. Until then they are not garbage. A shard may hold +// on to as many entries that have left it as an eighth of its capacity, +// though at least one and never more than 1024, and it does so for as long +// as no more of them leave. [Cache.Clear] is the exception; it lets go of +// everything before it returns, and waits for a reader if it has to. +// +// The sketch holds no keys, only counters: eight to sixteen bytes for every +// entry that the cache has room for, and 64 at least for every shard. +// +// A cache also has a table in which its readers say what they are looking +// at. It takes between 2 and 4 KiB for every CPU that Go may use when the +// cache is created, whatever the capacity. For a great many small caches, +// the lru package is the better choice. +package tinylfu + +import ( + "hash/maphash" + "iter" + "math/rand/v2" + "runtime" + "slices" + "sync" + + "github.com/floatdrop/evict/internal/cslab" + "github.com/floatdrop/evict/internal/shards" + "github.com/floatdrop/evict/internal/sketch" +) + +// Cache is a fixed-capacity cache that admits with TinyLFU. It is safe for +// concurrent use and does not allocate once it has been created, except in +// [Cache.All]. A Cache must be created with [New]; the zero value is of no +// use. +// +// [Cache.Get] and [Cache.Peek] take no lock, with rare exceptions: when a key +// is not found while an entry is on its way out of the same shard, and when +// more goroutines read at the same moment than the cache was made for, the +// read takes the lock of the shard to be sure. Writes are spread over shards +// that are locked independently, and each shard evicts on its own. See +// [WithShards]. +type Cache[K comparable, V any] struct { + seed maphash.Seed + mask uint32 // number of shards minus one + shards []shard[K, V] + readers *cslab.Readers + capacity int + + // window is the size of the small queues, and the search for it. + window window +} + +// A shard is a cache of its own, holding a share of the keys. +type shard[K comparable, V any] struct { + // The entries of both queues, each in a list of its own with the newest + // entry first. Node.Meta counts the reads of an entry since a write + // last looked at it, up to maxCount. Link.Tag says which list a node + // is in and what the entry's count was left at then, and holds bits of + // the hash of the key; see tagOf. + slab cslab.Slab[K, V] + + // mu serializes the writers. Readers do not take it, unless they find + // the index in motion. + mu sync.Mutex + + // seen is how often keys have been asked for of late: once for every + // time that one was put in, and the reads that entries had counted + // when a write looked at them. + seen sketch.Sketch + + // The number of entries in each queue. + smalls, mains int + + capacity int // entries in both queues together + + // full says that the shard has been full since it was made or last + // cleared. Until then, new entries are not all sent through the small + // queue; see set. + full bool + + // win is the window of the cache. The shard tells it of the hits that it + // counted under its lock and of its misses, whenever it has batch of + // the latter. + win *window + hits, misses uint64 + batch uint64 + + _ shards.Pad +} + +// The queues of a shard. The numbers are the list heads in the slab as well +// as the lowest bit of Link.Tag. +const ( + small = 0 + main = 1 + + queues = 2 +) + +const ( + // maxCount is as far as the reads of an entry count for its standing in + // the main queue, and with the sketch each time a write looks at it. It + // is as far as they are counted at all. + maxCount = 3 + + // The counters of the sketch are halved when it has counted this many + // requests for every entry that the shard has room for, and after about + // half as many from then on; see sketch.Sketch.Init. + samplePerEntry = 10 + + // A shard of a cache that has several tells the window of its misses + // in batches, so that the writers to different shards do not all write + // to the same place at every miss: of this many at the most, and of one + // for every batchShare entries of the shard. The smaller the batches, + // the closer the window's measurements are to what happened. + batchSize = 32 + batchShare = 8 +) + +// The tag of a node has the queue of the entry in its lowest bit. The two +// bits above that hold the count that the entry was left with when the main +// queue last passed it. The rest is the upper 29 bits of the hash of the key. +// With the lower half of the hash, which the slab keeps for every entry, that +// makes 61 bits to tell the sketch which key an entry has, without hashing +// the key again. + +// tagOf returns the tag of an entry with the given hash in the given queue, +// with a count of zero. +func tagOf(h uint64, queue uint32) uint32 { return uint32(h>>35)<<3 | queue } + +// kept returns the count that a tag holds. +func kept(tag uint32) uint32 { return tag >> 1 & 3 } + +// keep returns the tag with the given count in it. +func keep(tag, count uint32) uint32 { return tag&^(3<<1) | count<<1 } + +// The sketch knows a key by the 61 bits of its hash that a node holds. + +// printOf returns what the sketch knows the key with the given hash by. +func printOf(h uint64) uint64 { return h>>35<<32 | uint64(uint32(h)) } + +// printIn returns the same for an entry, from the hash it is indexed under +// and its tag. +func printIn(hash, tag uint32) uint64 { return uint64(tag>>3)<<32 | uint64(hash) } + +// Option configures a [Cache]. +type Option func(*config) + +type config struct { + shards int +} + +// WithShards sets the number of shards, which is rounded up to a power of two +// and limited to the capacity. Zero or less stands for the default. +// +// Reads do not depend on it. A write waits for the other writes to the same +// shard, so more shards let more goroutines write at the same time. In +// exchange, each shard runs its queues and its sketch on its own share of the +// keys and the capacity: a shard that is full evicts even while another one +// has room. The size of the small queue is the same share for all of them. +// +// By default the cache has a single shard if its capacity is less than 512 +// and otherwise several per CPU, as long as that leaves each shard with at +// least 256 entries. On the traces that the benchmarks replay, shards of that +// size cost the hit ratio half a percentage point at the most, but for about +// a point on one trace and 2.6 points in one case; part of that is how much +// the cache varies from one run to the next anyway. See the report on shards +// there, which was made as on a machine with eight CPUs. +func WithShards(n int) Option { + return func(c *config) { c.shards = n } +} + +// New returns a cache that holds up to capacity entries. A cache with a +// capacity of zero or less holds nothing. +func New[K comparable, V any](capacity int, opts ...Option) *Cache[K, V] { + var cfg config + for _, opt := range opts { + opt(&cfg) + } + + capacity = max(capacity, 0) + n := shards.Count(capacity, cfg.shards, cslab.MaxCapacity) + + c := &Cache[K, V]{ + seed: maphash.MakeSeed(), + mask: uint32(n - 1), + shards: make([]shard[K, V], n), + readers: cslab.NewReaders(), + capacity: capacity, + } + // The first shard is the largest. + c.window.init(shards.Capacity(capacity, n, 0), capacity, c.readers) + for i := range c.shards { + s := &c.shards[i] + s.capacity = shards.Capacity(capacity, n, i) + s.slab.Init(s.capacity, queues, c.readers) + s.seen.Init(s.capacity, max(samplePerEntry*s.capacity, 1)) + s.win, s.batch = &c.window, 1 + if n > 1 { + s.batch = uint64(min(batchSize, max(s.capacity/batchShare, 1))) + } + } + return c +} + +// Each method starts by hashing the key. The upper half of the hash picks the +// shard and the lower half is what the shard indexes the key under, so that +// keys sharing a shard do not share a prefix of their probe sequences. + +// Get returns the value stored under key and counts the read, which speaks +// for the entry when the cache has to choose what to keep. +func (c *Cache[K, V]) Get(key K) (value V, ok bool) { + h := maphash.Comparable(c.seed, key) + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + + // The lookup counts the read on the entry, and as a request answered + // in the reader's slot. Readers do not wait for each other, so two of + // them may count as one on the entry. That is close enough. An entry + // that is read all the time has reached the limit, and is not written + // to again: its cache line stays shared between the CPUs. + value, ok, certain := s.slab.Lookup(hash, key, maxCount, true) + if s.slab.Overdue() { + s.help() + } + if certain { + return value, ok + } + + // The index was changing under the lookup, or too many goroutines are + // reading at once. Do it the plain way. + s.mu.Lock() + if idx := s.slab.Find(hash, key); idx != 0 { + n := &s.slab.Nodes[idx] + n.Meta.Store(min(n.Meta.Load()+1, maxCount)) + value, ok = n.Value, true + s.hits++ + } + s.mu.Unlock() + + return value, ok +} + +// Peek returns the value stored under key without counting the read. +func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { + h := maphash.Comparable(c.seed, key) + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + + value, ok, certain := s.slab.Lookup(hash, key, 0, false) + if s.slab.Overdue() { + s.help() + } + if certain { + return value, ok + } + + s.mu.Lock() + if idx := s.slab.Find(hash, key); idx != 0 { + value, ok = s.slab.Nodes[idx].Value, true + } + s.mu.Unlock() + + return value, ok +} + +// help is what a reader does for a shard that holds on to entries which are +// gone, because another reader was looking at them when the shard last tried +// to let go of them. A shard tries again whenever it is written to, but if +// nobody writes, it is up to the readers. Looking for the other readers takes +// a while, so only one read in about a thousand does it; that is soon enough. +// +//go:noinline +func (s *shard[K, V]) help() { + if rand.Uint32()&1023 != 0 || !s.mu.TryLock() { + return + } + s.slab.Reclaim() + s.mu.Unlock() +} + +// Set stores value under key. A key that the cache does not hold goes into +// the small queue, and the sketch counts a request for it. For an entry that +// is cached already, the write counts as a read. +// +// A cache that has not been full yet, or not since it was cleared, puts no +// more new entries into the small queue than that has room for by its share; +// the others go into the main queue. +// +// Set reports the entry that left the cache as a result, if there is one: +// either the previous entry for key, in which case oldKey == key, or the +// entry that the key's shard evicted to make room. A cache without capacity +// hands the new entry straight back. +func (c *Cache[K, V]) Set(key K, value V) (oldKey K, oldValue V, displaced bool) { + h := maphash.Comparable(c.seed, key) + s := &c.shards[uint32(h>>32)&c.mask] + + s.mu.Lock() + oldKey, oldValue, displaced = s.set(h, key, value) + s.mu.Unlock() + + return oldKey, oldValue, displaced +} + +func (s *shard[K, V]) set(h uint64, key K, value V) (oldKey K, oldValue V, displaced bool) { + sl := &s.slab + hash := uint32(h) + + if idx := sl.Find(hash, key); idx != 0 { + n := &sl.Nodes[idx] + oldKey, oldValue = n.Key, n.Value + + // Readers may be looking at the entry, so it gets its new value + // in another node, which takes over its place in its queue. + n = &sl.Nodes[sl.Replace(idx, key, value)] + n.Meta.Store(min(n.Meta.Load()+1, maxCount)) + s.hits++ + return oldKey, oldValue, true + } + + if s.capacity == 0 { + return key, value, true + } + // The key is asked for, whatever becomes of it, and the shard did not + // have it. That may be what makes the size of the small queue move. + s.seen.Add(printOf(h), 1) + s.misses++ + if s.misses >= s.batch { + s.win.note(s.hits, s.misses) + s.hits, s.misses = 0, 0 + } + if s.smalls+s.mains == s.capacity { + oldKey, oldValue = s.evict() + displaced = true + } + + // A shard that has yet to fill up for the first time has nothing to + // evict, and so no use for the small queue beyond its share: the + // entries that follow go straight to where the room is. Otherwise the + // entries that happen to come first would have to pass through the + // small queue one after the other later on, for no reason but their + // having been first. + warming := !s.full && s.smalls >= s.win.sizeFor(s.capacity) + + idx := sl.Insert(hash, key, value) + if warming { + sl.Links[idx].Tag = tagOf(h, main) + sl.PushFront(main, idx) + s.mains++ + } else { + sl.Links[idx].Tag = tagOf(h, small) + sl.PushFront(small, idx) + s.smalls++ + } + return oldKey, oldValue, displaced +} + +// look takes stock of the entry in node idx: it tells the sketch of the reads +// that the entry has counted since a write last looked at it, and returns +// what the entry's count comes to with them. +func (s *shard[K, V]) look(idx uint32) (count uint32) { + n := &s.slab.Nodes[idx] + link := &s.slab.Links[idx] + before := kept(link.Tag) + reads := n.Meta.Load() + if reads == 0 { + // Nothing to tell, and nothing to write to the entry either. + return before + } + // Readers do not wait for this. One that counts between the line + // above and the next is not counted, which is close enough. + n.Meta.Store(0) + + count = min(before+reads, maxCount) + s.seen.Add(printIn(s.slab.Hash(idx), link.Tag), int(count-before)) + return count +} + +// asked returns how often the key of the entry in node idx has been asked for +// of late, as far as the sketch knows. +func (s *shard[K, V]) asked(idx uint32) int { + return s.seen.Estimate(printIn(s.slab.Hash(idx), s.slab.Links[idx].Tag)) +} + +// promote moves the entry in node idx from the small queue to the front of +// the main queue. It arrives there without a count: the small queue keeps +// none, and a write has just looked at the entry. +func (s *shard[K, V]) promote(idx uint32) { + sl := &s.slab + sl.Unlink(idx) + sl.Links[idx].Tag |= main + sl.PushFront(main, idx) + s.smalls-- + s.mains++ +} + +// victim returns the entry that the main queue would evict: its oldest entry +// that has not been read since the queue last came by it, or zero if the +// queue is empty. The entries that it comes by on the way keep their places +// in the cache for another round, with one off their count. +func (s *shard[K, V]) victim() uint32 { + sl := &s.slab + links := sl.Links + // An entry is let off maxCount times in a row at the most, so after as + // many rounds the oldest one has no reads left, unless readers keep + // counting while the queue turns. Then it has to do all the same. + for range maxCount * s.mains { + idx := links[main].Prev + count := s.look(idx) + if count == 0 { + break + } + links[idx].Tag = keep(links[idx].Tag, count-1) + sl.Unlink(idx) + sl.PushFront(main, idx) + } + if s.mains == 0 { + return 0 + } + return links[main].Prev +} + +// evict makes room for one entry and returns the one that had to go. The +// shard is full. +func (s *shard[K, V]) evict() (key K, value V) { + sl := &s.slab + links := sl.Links + s.full = true + + // A small queue whose share has shrunk hands its oldest entries on to + // the main queue, which is then the one to evict from. + size := s.win.sizeFor(s.capacity) + for s.smalls > size { + idx := links[small].Prev + s.look(idx) + s.promote(idx) + } + + // The main queue has entries if the small one is short of its share, + // since the two of them fill the shard. + var gone uint32 + if s.smalls < size { + gone = s.victim() + } else { + // The small queue has its share, so its oldest entry is up + // against the one that the main queue would evict. It gets in + // only if it has been asked for more often of late: a tie goes + // to the entry that has been here for longer. + gone = links[small].Prev + s.look(gone) + if against := s.victim(); against != 0 && s.asked(gone) > s.asked(against) { + s.promote(gone) + gone = against + } + } + + if links[gone].Tag&1 == main { + s.mains-- + } else { + s.smalls-- + } + n := &sl.Nodes[gone] + key, value = n.Key, n.Value + sl.Unlink(gone) + sl.Remove(gone) + return key, value +} + +// Delete removes key and returns the value it had. The sketch keeps what it +// knows of the key, and learns of the reads that the entry has counted: a key +// that is deleted because its value is out of date is often back soon. +func (c *Cache[K, V]) Delete(key K) (value V, ok bool) { + h := maphash.Comparable(c.seed, key) + s, hash := &c.shards[uint32(h>>32)&c.mask], uint32(h) + + s.mu.Lock() + if idx := s.slab.Find(hash, key); idx != 0 { + value, ok = s.slab.Nodes[idx].Value, true + s.look(idx) + if s.slab.Links[idx].Tag&1 == main { + s.mains-- + } else { + s.smalls-- + } + s.slab.Unlink(idx) + s.slab.Remove(idx) + } + s.mu.Unlock() + + return value, ok +} + +// Len returns the number of entries in the cache. +func (c *Cache[K, V]) Len() int { + n := 0 + for i := range c.shards { + n += c.shards[i].slab.Len() + } + return n +} + +// Cap returns the number of entries the cache can hold. +func (c *Cache[K, V]) Cap() int { return c.capacity } + +// Clear removes every entry and empties the sketch. When it returns, the +// cache no longer refers to any of the keys and values it held. The share of +// the small queue stays as the cache has found it to be right, and the +// search for it goes on from there, with the share of requests that the +// cache answers from then on to compare with. +func (c *Cache[K, V]) Clear() { + // No move of the size comes between the shards, nor between them and + // the start of the new measurements. + c.window.mu.Lock() + defer c.window.mu.Unlock() + + for i := range c.shards { + s := &c.shards[i] + s.mu.Lock() + mark := s.slab.Reset() + s.seen.Reset() + s.smalls, s.mains, s.full = 0, 0, false + // A shard that tells the window of every miss has nothing to + // tell but hits since the last one, which belong to the time + // before. One that tells it in batches keeps what it has yet to + // tell: when the next measurement ends, it will be holding back + // about as much again, and a measurement that began without that + // would count every hit but too few misses. + if s.batch == 1 { + s.hits, s.misses = 0, 0 + } + // A reader may still be looking at one of the entries. It is done + // in no time, unless it is held up; either way the wait for it is + // not spent with the shard locked. + for !s.slab.Released(mark) { + s.mu.Unlock() + runtime.Gosched() + s.mu.Lock() + } + s.mu.Unlock() + } + c.window.forget() +} + +// All iterates over the entries without counting them as read. A shard yields +// the entries of its main queue first and then those of its small queue, each +// from the newest to the oldest. A cache with a single shard yields its +// entries in exactly that order. +// +// A shard is locked only while its entries are collected, not while they are +// yielded: the loop body may use the cache, and a change it makes to a shard +// that has not been reached yet shows. +func (c *Cache[K, V]) All() iter.Seq2[K, V] { + type entry struct { + key K + value V + } + + return func(yield func(K, V) bool) { + var entries []entry + for i := range c.shards { + s := &c.shards[i] + + s.mu.Lock() + nodes, links := s.slab.Nodes, s.slab.Links + entries = slices.Grow(entries[:0], s.smalls+s.mains) + for _, head := range [...]uint32{main, small} { + for idx := links[head].Next; idx != head; idx = links[idx].Next { + entries = append(entries, entry{nodes[idx].Key, nodes[idx].Value}) + } + } + s.mu.Unlock() + + for _, e := range entries { + if !yield(e.key, e.value) { + return + } + } + clear(entries) // Do not hold on to them while the next shard is at it. + } + } +} + +// The methods of a generic type are compiled in the package that uses them, +// and the calls they make are inlined there only if this package has inlined +// them itself; see the note in package lru. Instantiating the methods here +// makes sure it has. +var ( + _ = (*Cache[int, int]).Get + _ = (*Cache[int, int]).Peek + _ = (*Cache[int, int]).Set + _ = (*Cache[int, int]).Delete + _ = (*Cache[int, int]).Len + _ = (*Cache[int, int]).Clear + _ = (*Cache[int, int]).All +) diff --git a/tinylfu/tinylfu_test.go b/tinylfu/tinylfu_test.go new file mode 100644 index 0000000..de955d5 --- /dev/null +++ b/tinylfu/tinylfu_test.go @@ -0,0 +1,1122 @@ +package tinylfu_test + +import ( + "fmt" + "iter" + "math/rand/v2" + "runtime" + "slices" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + "weak" + + "github.com/floatdrop/evict" + "github.com/floatdrop/evict/internal/cachetest" + "github.com/floatdrop/evict/internal/sketch" + "github.com/floatdrop/evict/lru" + "github.com/floatdrop/evict/tinylfu" +) + +var factories = cachetest.Factories{ + Int: func(capacity, shards int) evict.Cache[int, int] { + return tinylfu.New[int, int](capacity, tinylfu.WithShards(shards)) + }, + String: func(capacity, shards int) evict.Cache[string, string] { + return tinylfu.New[string, string](capacity, tinylfu.WithShards(shards)) + }, + Float: func(capacity, shards int) evict.Cache[float64, int] { + return tinylfu.New[float64, int](capacity, tinylfu.WithShards(shards)) + }, + Blob: func(capacity, shards int) evict.Cache[int, *cachetest.Blob] { + return tinylfu.New[int, *cachetest.Blob](capacity, tinylfu.WithShards(shards)) + }, + BlobKey: func(capacity, shards int) evict.Cache[*cachetest.Blob, int] { + return tinylfu.New[*cachetest.Blob, int](capacity, tinylfu.WithShards(shards)) + }, + Wide: func(capacity, shards int) evict.Cache[string, cachetest.Wide] { + return tinylfu.New[string, cachetest.Wide](capacity, tinylfu.WithShards(shards)) + }, + // The sketch holds counters, not keys: it keeps no key alive. +} + +func TestCache(t *testing.T) { + t.Parallel() + + cachetest.Run(t, factories) +} + +// TestNoAllocs is not parallel: counting allocations needs the process to +// itself. +func TestNoAllocs(t *testing.T) { + cachetest.NoAllocs(t, factories) +} + +// A scene is a cache in one piece in which single entries can be followed +// around. Its keys go by the names 1, 2, 3 and so on, and no two of them +// share a counter of the sketch: what the sketch says of a key is how often +// it was asked for, to the letter, as long as the counters have not been +// halved. With ten additions for every entry until then, the scenes are over +// before that. +type scene struct { + t *testing.T + c *tinylfu.Cache[int, string] + keys []int // by name + names map[int]int // by key +} + +func newScene(t *testing.T, capacity, names int) *scene { + t.Helper() + + c := tinylfu.New[int, string](capacity, tinylfu.WithShards(1)) + s := &scene{t: t, c: c, names: map[int]int{}} + s.keys = append([]int{-1}, tinylfu.Apart(c, names)...) // Nothing goes by the name of 0. + for name, key := range s.keys { + s.names[key] = name + } + return s +} + +// set stores entries that need no room made for them. +func (s *scene) set(names ...int) { + s.t.Helper() + for _, name := range names { + if k, _, displaced := s.c.Set(s.keys[name], ""); displaced { + s.t.Fatalf("Set(%d) displaced %d", name, s.names[k]) + } + } +} + +// evicts stores an entry and checks which one made room for it. +func (s *scene) evicts(name, want int) { + s.t.Helper() + if k, _, displaced := s.c.Set(s.keys[name], ""); !displaced || s.names[k] != want { + s.t.Fatalf("Set(%d) evicted %d, %t, want %d", name, s.names[k], displaced, want) + } +} + +func (s *scene) get(name, times int) { + s.t.Helper() + for range times { + if _, ok := s.c.Get(s.keys[name]); !ok { + s.t.Fatalf("Get(%d) found nothing", name) + } + } +} + +// expect checks the main queue and the small queue, each from the newest to +// the oldest entry. +func (s *scene) expect(inMain, inSmall []int) { + s.t.Helper() + + named := func(keys []int) []int { + names := make([]int, 0, len(keys)) + for _, key := range keys { + names = append(names, s.names[key]) + } + return names + } + gotMain, gotSmall := s.c.Queues() + if !slices.Equal(named(gotMain), inMain) || !slices.Equal(named(gotSmall), inSmall) { + s.t.Fatalf("the main queue holds %v and the small queue %v, want %v and %v", named(gotMain), named(gotSmall), inMain, inSmall) + } + // All goes through them in the same order. + var all []int + for key := range s.c.All() { + all = append(all, key) + } + if !slices.Equal(named(all), slices.Concat(inMain, inSmall)) { + s.t.Fatalf("All yields %v, want %v followed by %v", named(all), inMain, inSmall) + } +} + +// asked checks how often the sketch has a key down as asked for. +func (s *scene) asked(name, want int) { + s.t.Helper() + if got := s.c.Asked(s.keys[name]); got != want { + s.t.Fatalf("the sketch has %d down as asked for %d times, want %d", name, got, want) + } +} + +type in = []int + +// TestAdmission follows entries through the small queue: out of the cache, or +// on into the main queue in place of the entry that it would evict next. +func TestAdmission(t *testing.T) { + t.Parallel() + + // Of ten entries, one is the share of the small queue to begin with. + s := newScene(t, 10, 30) + if got := s.c.Window(); got != 1 { + t.Fatalf("the small queue is to have %d entries, want 1", got) + } + + // While the cache fills up for the first time, the small queue takes + // its share, which is the first entry, and the rest goes straight to + // the main queue. + s.set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + s.expect(in{10, 9, 8, 7, 6, 5, 4, 3, 2}, in{1}) + + // From here on, what is new goes through the small queue. When its + // turn to leave comes, it is up against the oldest entry of the main + // queue, which is 2. Both have been asked for once, and a tie goes to + // the one that was there first. + s.evicts(11, 1) + s.expect(in{10, 9, 8, 7, 6, 5, 4, 3, 2}, in{11}) + s.evicts(12, 11) + s.expect(in{10, 9, 8, 7, 6, 5, 4, 3, 2}, in{12}) + + // 11 is asked for again. That makes twice, which the sketch knows + // although the cache had let go of it: this time it gets in, and 2 goes. + s.evicts(11, 12) + s.asked(11, 2) + s.evicts(13, 2) + s.expect(in{11, 10, 9, 8, 7, 6, 5, 4, 3}, in{13}) + + // A read is as good as that. The sketch learns of it when a write next + // looks at the entry. + s.get(13, 1) + s.asked(13, 1) + s.evicts(14, 3) + s.asked(13, 2) + s.expect(in{13, 11, 10, 9, 8, 7, 6, 5, 4}, in{14}) + + // Peeking at an entry does not count. + s.c.Peek(s.keys[14]) + s.c.Peek(s.keys[14]) + s.evicts(15, 14) + s.asked(14, 1) + s.expect(in{13, 11, 10, 9, 8, 7, 6, 5, 4}, in{15}) + + // Writing to one counts like reading it. + if k, _, displaced := s.c.Set(s.keys[15], "again"); !displaced || k != s.keys[15] { + t.Fatalf("Set of 15, which is cached, displaced %d, %t", s.names[k], displaced) + } + s.evicts(16, 4) + s.asked(15, 2) + s.expect(in{15, 13, 11, 10, 9, 8, 7, 6, 5}, in{16}) + + // Reads count up to three with the sketch between one look and the + // next, however many there were. + s.get(16, 9) + s.evicts(17, 5) + s.asked(16, 4) + s.expect(in{16, 15, 13, 11, 10, 9, 8, 7, 6}, in{17}) + + // Five more entries that were read once take the places of 6 to 10. + for name := 17; name <= 21; name++ { + s.get(name, 1) + s.evicts(name+1, name-11) + } + s.expect(in{21, 20, 19, 18, 17, 16, 15, 13, 11}, in{22}) + + // Now the oldest entry of the main queue is 11, which was asked for + // twice. One that was asked for as often does not get in, + s.get(22, 1) + s.evicts(23, 22) + s.expect(in{21, 20, 19, 18, 17, 16, 15, 13, 11}, in{23}) + // and one that was asked for more often does. + s.get(23, 2) + s.evicts(24, 11) + s.expect(in{23, 21, 20, 19, 18, 17, 16, 15, 13}, in{24}) + + // Deleting an entry does not make the sketch forget its key. It learns + // of the reads that the entry had counted, even. + s.get(24, 2) + if _, ok := s.c.Delete(s.keys[24]); !ok { + t.Fatal("Delete(24) found nothing") + } + s.asked(24, 3) + s.expect(in{23, 21, 20, 19, 18, 17, 16, 15, 13}, nil) + // So when 24 is back, it is one that has been asked for four times, + // and gets in ahead of 13 with its two. + s.set(24) + s.evicts(25, 13) + s.expect(in{24, 23, 21, 20, 19, 18, 17, 16, 15}, in{25}) + + // A cache that was cleared has forgotten all of it, and fills up like + // a new one. + s.c.Clear() + for name := 1; name <= 25; name++ { + s.asked(name, 0) + } + s.set(16, 24, 3, 4, 5, 6, 7, 8, 9, 10) + s.expect(in{10, 9, 8, 7, 6, 5, 4, 3, 24}, in{16}) + s.evicts(11, 16) + s.evicts(12, 11) + s.expect(in{10, 9, 8, 7, 6, 5, 4, 3, 24}, in{12}) +} + +// TestMainQueue follows entries around the main queue. +func TestMainQueue(t *testing.T) { + t.Parallel() + + // A main queue of three entries is quick to go round: the small queue + // has seven of the ten. + s := newScene(t, 10, 30) + s.c.Resize(7) + s.set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + s.expect(in{10, 9, 8}, in{7, 6, 5, 4, 3, 2, 1}) + + // 8 and 9 have been read, so they get another round, and the one that + // 1 is up against is 10. An entry gets a round for every read, up to + // three, however often it was read: 8 has three coming and 9 this one. + s.get(8, 5) + s.get(9, 1) + s.evicts(11, 1) + s.expect(in{9, 8, 10}, in{11, 7, 6, 5, 4, 3, 2}) + s.asked(8, 4) + s.asked(9, 2) + + // 10 stays where it is for as long as it holds its own. + s.evicts(12, 2) + s.expect(in{9, 8, 10}, in{12, 11, 7, 6, 5, 4, 3}) + s.get(3, 1) + s.evicts(13, 10) + s.expect(in{3, 9, 8}, in{13, 12, 11, 7, 6, 5, 4}) + + // The second round of 8. For 9 there is none: it is the one that 4 is + // up against, and holds its own with its two requests. + s.evicts(14, 4) + s.expect(in{8, 3, 9}, in{14, 13, 12, 11, 7, 6, 5}) + // The sketch was told of the reads of 8 once and is not told again. + s.asked(8, 4) + + s.get(5, 2) + s.evicts(15, 9) + s.expect(in{5, 8, 3}, in{15, 14, 13, 12, 11, 7, 6}) + s.evicts(16, 6) + s.expect(in{5, 8, 3}, in{16, 15, 14, 13, 12, 11, 7}) + s.get(7, 2) + s.evicts(17, 3) + s.expect(in{7, 5, 8}, in{17, 16, 15, 14, 13, 12, 11}) + + // The third round of 8, and its last. + s.evicts(18, 11) + s.expect(in{8, 7, 5}, in{18, 17, 16, 15, 14, 13, 12}) + s.asked(8, 4) + + s.get(12, 3) + s.evicts(19, 5) + s.expect(in{12, 8, 7}, in{19, 18, 17, 16, 15, 14, 13}) + s.get(13, 3) + s.evicts(20, 7) + s.expect(in{13, 12, 8}, in{20, 19, 18, 17, 16, 15, 14}) + + // After that it is an entry like any other. It is the one that 14 is + // up against, and stays because it was asked for four times. + s.evicts(21, 14) + s.expect(in{13, 12, 8}, in{21, 20, 19, 18, 17, 16, 15}) + // Until it is read again. + s.get(8, 1) + s.evicts(22, 15) + s.expect(in{8, 13, 12}, in{22, 21, 20, 19, 18, 17, 16}) + s.asked(8, 5) +} + +// TestWindow checks what the queues do when the share of the small one +// changes. +func TestWindow(t *testing.T) { + t.Parallel() + + s := newScene(t, 10, 30) + s.c.Resize(5) + s.set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + s.expect(in{10, 9, 8, 7, 6}, in{5, 4, 3, 2, 1}) + + // The share of the small queue shrinks to two entries. When room is + // next needed, it hands its oldest entries on to the main queue until + // it is within its share, read or not; what they were read goes on + // record. After that it is 4 that is up against 6. + s.get(1, 2) + s.c.Resize(2) + s.evicts(11, 4) + s.expect(in{3, 2, 1, 10, 9, 8, 7, 6}, in{11, 5}) + s.asked(1, 3) + + // The share grows to four. Until the small queue has that many, the + // main queue is the one to make room: its oldest entry that has not + // been read goes, and has nothing to be up against. + s.c.Resize(4) + s.evicts(12, 6) + s.expect(in{3, 2, 1, 10, 9, 8, 7}, in{12, 11, 5}) + s.get(7, 1) + s.evicts(13, 8) + s.expect(in{7, 3, 2, 1, 10, 9}, in{13, 12, 11, 5}) + // Now it has. + s.evicts(14, 5) + s.expect(in{7, 3, 2, 1, 10, 9}, in{14, 13, 12, 11}) + + // After deletions there is room, and what is new joins the small + // queue all the same once the cache has been full: it may be over its + // share for a while. + for _, name := range []int{10, 9} { + if _, ok := s.c.Delete(s.keys[name]); !ok { + t.Fatalf("Delete(%d) found nothing", name) + } + } + s.set(15, 16) + s.expect(in{7, 3, 2, 1}, in{16, 15, 14, 13, 12, 11}) + // 11 and 12 are handed on, and 13 is up against 1, which was asked + // for three times. + s.evicts(17, 13) + s.expect(in{12, 11, 7, 3, 2, 1}, in{17, 16, 15, 14}) + + // A cache that is cleared keeps the share that it has found: of the + // next ten entries, four fill the small queue. + s.c.Clear() + if got := s.c.Window(); got != 4 { + t.Fatalf("after Clear the small queue is to have %d entries, want 4", got) + } + s.set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + s.expect(in{10, 9, 8, 7, 6, 5}, in{4, 3, 2, 1}) +} + +// TestSmallCaches checks the caches that have no room for two queues to speak +// of. +func TestSmallCaches(t *testing.T) { + t.Parallel() + + // With room for one entry there is the small queue only, and what is + // new takes the place of what is there. + s := newScene(t, 1, 5) + s.set(1) + s.expect(nil, in{1}) + s.get(1, 3) + s.evicts(2, 1) + s.evicts(1, 2) + s.expect(nil, in{1}) + s.asked(1, 5) + + // With room for two there is one of each. + s = newScene(t, 2, 6) + s.set(1, 2) + s.expect(in{2}, in{1}) + s.evicts(3, 1) + s.get(3, 1) + s.evicts(4, 2) + s.expect(in{3}, in{4}) +} + +// A workload is the keys that a program asks its cache for, one after the +// other. +type workload = iter.Seq[int] + +// replay runs the workload against a cache the way a program uses one, and +// returns the share of the requests that the cache answered. +func replay(c evict.Cache[int, int], w workload) float64 { + hits, requests := 0, 0 + for key := range w { + requests++ + if _, ok := c.Get(key); ok { + hits++ + } else { + c.Set(key, key) + } + } + return float64(hits) / float64(requests) +} + +// TestFindsItsWindow checks that the cache finds a size for its small queue +// that suits the workload, by what comes of it: on a workload that an LRU +// cache is bad at, it does much better than one, and on a workload that an +// LRU cache is as good at as a cache can be, it does not do much worse. +func TestFindsItsWindow(t *testing.T) { + t.Parallel() + + const capacity, requests = 1000, 1_500_000 + + // Popular keys among a flood of keys that are asked for once: the + // newest entries are the ones least worth keeping. + t.Run("frequency", func(t *testing.T) { + t.Parallel() + + w := func(yield func(int) bool) { + rng := rand.New(rand.NewPCG(1, 2)) + zipf := rand.NewZipf(rng, 1.01, 1, 20_000) + for i := range requests { + key := int(zipf.Uint64()) + if i%2 == 0 { + key = -i - 1 + } + if !yield(key) { + return + } + } + } + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + got, plain := replay(c, w), replay(lru.New[int, int](capacity), w) + t.Logf("answered %.1f%%, an LRU cache %.1f%%; the small queue has ended up with %d entries", 100*got, 100*plain, c.Window()) + if got < plain+0.03 { + t.Errorf("answered %.1f%% of the requests, want at least three points more than the %.1f%% of an LRU cache", 100*got, 100*plain) + } + }) + + // The keys that are asked for are the newest six hundred, whichever + // they are at the time, and every third request is for one that is + // new: how long ago a key was asked for says everything, and how often + // says nothing. An LRU cache answers all there is to answer. The small + // queue has to be large for that; with the hundredth of the room that + // it starts with, the cache answers 15% of the requests. + t.Run("recency", func(t *testing.T) { + t.Parallel() + + w := func(yield func(int) bool) { + rng := rand.New(rand.NewPCG(1, 2)) + newest := 0 + for range requests { + key := newest - rng.IntN(600) + if rng.IntN(10) < 3 { + newest++ + key = newest + } + if !yield(key) { + return + } + } + } + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + got, plain := replay(c, w), replay(lru.New[int, int](capacity), w) + t.Logf("answered %.1f%%, an LRU cache %.1f%%; the small queue has ended up with %d entries", 100*got, 100*plain, c.Window()) + if got < plain-0.15 { + t.Errorf("answered %.1f%% of the requests, want no more than fifteen points fewer than the %.1f%% of an LRU cache", 100*got, 100*plain) + } + if c.Window() < capacity/4 { + t.Errorf("the small queue has ended up with %d entries of %d, want a quarter at least", c.Window(), capacity) + } + }) +} + +// model is the policy at its simplest: a slice for each queue, with the +// newest entry first, and the search for the size of the small queue as +// Caffeine has it, every so many misses. +type model struct { + capacity int + // known is what the sketch of the cache knows a key by. Keys that + // share counters there share them here. + known func(key int) uint64 + asked sketch.Sketch + + small, main []*entry + // full says that the cache has been full since it was made or cleared. + full bool + + window int + target, step, previous float64 + // measured says that previous has been measured since the model was + // made or cleared. + measured bool + // The requests answered and not since the size last moved, and the + // misses after which it moves next. + hits, misses, due int + moves int +} + +type entry struct { + key, value int + reads int // since a write last looked, up to three + count int // what the main queue left it with +} + +func newModel(capacity int, known func(key int) uint64) *model { + m := &model{capacity: capacity, known: known} + m.asked.Init(capacity, max(10*capacity, 1)) + m.target = max(0.01*float64(capacity), 1) + m.window = int(m.target) + m.step = 0.0625 * float64(capacity) + m.due = 40 * capacity + return m +} + +func (m *model) find(key int) *entry { + for _, queue := range [][]*entry{m.main, m.small} { + if i := slices.IndexFunc(queue, func(e *entry) bool { return e.key == key }); i >= 0 { + return queue[i] + } + } + return nil +} + +func (m *model) Get(key int) (int, bool) { + e := m.find(key) + if e == nil { + return 0, false + } + e.reads = min(e.reads+1, 3) + m.hits++ + return e.value, true +} + +func (m *model) Peek(key int) (int, bool) { + e := m.find(key) + if e == nil { + return 0, false + } + return e.value, true +} + +func (m *model) Set(key, value int) (oldKey, oldValue int, displaced bool) { + if e := m.find(key); e != nil { + oldValue, e.value = e.value, value + e.reads = min(e.reads+1, 3) + m.hits++ + return key, oldValue, true + } + if m.capacity == 0 { + return key, value, true + } + + m.asked.Add(m.known(key), 1) + m.misses++ + if m.misses >= m.due { + m.climb() + } + if len(m.small)+len(m.main) == m.capacity { + oldKey, oldValue = m.evict() + displaced = true + } + + // A cache that has never been full does not put more into the small + // queue than its share. + e := &entry{key: key, value: value} + if !m.full && len(m.small) >= m.window { + m.main = slices.Insert(m.main, 0, e) + } else { + m.small = slices.Insert(m.small, 0, e) + } + return oldKey, oldValue, displaced +} + +// climb is the hill climber of Caffeine. The first time, there is nothing to +// compare with but the share of requests answered that it has just measured. +// It moves again after the misses that forty requests per entry would bring +// at that share, or after one at least. +func (m *model) climb() { + rate := float64(m.hits) / float64(m.hits+m.misses) + if !m.measured { + m.previous, m.measured = rate, true + } + change := rate - m.previous + amount := m.step + if change < 0 { + amount = -m.step + } + if change >= 0.05 || change <= -0.05 { + m.step = 0.0625 * float64(m.capacity) + if amount < 0 { + m.step = -m.step + } + } else { + m.step = 0.98 * amount + } + m.previous, m.hits, m.misses = rate, 0, 0 + m.due = max(int(40*float64(m.capacity)*(1-rate)), 1) + m.moves++ + + least, most := max(0.01*float64(m.capacity), 1), max(0.8*float64(m.capacity), 1) + m.target = min(max(m.target+amount, least), most) + m.window = int(m.target) +} + +// look is what a write does when it comes by an entry: the sketch learns of +// the reads that count and that it has not been told of. It returns the +// count of the entry. +func (m *model) look(e *entry) int { + count := min(e.count+e.reads, 3) + m.asked.Add(m.known(e.key), count-e.count) + e.reads = 0 + return count +} + +// victim turns the main queue until its oldest entry is one that has no +// count left, and returns that. +func (m *model) victim() *entry { + for len(m.main) > 0 { + last := len(m.main) - 1 + e := m.main[last] + count := m.look(e) + if count == 0 { + return e + } + e.count = count - 1 + m.main = slices.Insert(m.main[:last], 0, e) + } + return nil +} + +func (m *model) evict() (key, value int) { + m.full = true + + moveUp := func() { + last := len(m.small) - 1 + e := m.small[last] + m.small = m.small[:last] + m.main = slices.Insert(m.main, 0, e) + } + for len(m.small) > m.window { + m.look(m.small[len(m.small)-1]) + moveUp() + } + + if len(m.small) < m.window { + gone := m.victim() + m.main = m.main[:len(m.main)-1] + return gone.key, gone.value + } + + candidate := m.small[len(m.small)-1] + m.look(candidate) + against := m.victim() + if against == nil || m.asked.Estimate(m.known(candidate.key)) <= m.asked.Estimate(m.known(against.key)) { + m.small = m.small[:len(m.small)-1] + return candidate.key, candidate.value + } + m.main = m.main[:len(m.main)-1] + moveUp() + return against.key, against.value +} + +func (m *model) Delete(key int) (int, bool) { + e := m.find(key) + if e == nil { + return 0, false + } + m.look(e) + same := func(other *entry) bool { return other == e } + m.small = slices.DeleteFunc(m.small, same) + m.main = slices.DeleteFunc(m.main, same) + return e.value, true +} + +func (m *model) Len() int { return len(m.small) + len(m.main) } +func (m *model) Cap() int { return m.capacity } + +// Clear keeps the size of the small queue, and what the last measurement says +// of how many misses to wait for, but measures afresh. +func (m *model) Clear() { + m.small, m.main, m.full = nil, nil, false + m.asked.Reset() + m.measured, m.hits, m.misses = false, 0, 0 + m.due = max(int(40*float64(m.capacity)*(1-m.previous)), 1) +} + +func (m *model) All() iter.Seq2[int, int] { + return func(yield func(int, int) bool) { + for _, e := range slices.Concat(m.main, m.small) { + if !yield(e.key, e.value) { + return + } + } + } +} + +func TestAgainstModel(t *testing.T) { + t.Parallel() + + for i, tt := range []struct{ capacity, keys, steps int }{ + {0, 3, 100}, + {1, 3, 2_000}, + {2, 4, 5_000}, + {3, 6, 5_000}, + {8, 12, 20_000}, + {8, 100, 20_000}, + {10, 15, 20_000}, + {20, 30, 20_000}, + {50, 40, 20_000}, // Never full. + {50, 80, 50_000}, + {50, 500, 50_000}, + {300, 1000, 100_000}, + } { + // Once the way it is meant to be, and once with readers that have + // to take the lock. And once with the odd Clear among the + // operations, and once without: Clear starts the measurements for + // the size of the small queue over, and the cache is cleared too + // often here for one of them ever to be complete. + for _, crowded := range []bool{false, true} { + for _, clears := range []bool{true, false} { + t.Run(fmt.Sprintf("capacity=%d/keys=%d/crowded=%t/clears=%t", tt.capacity, tt.keys, crowded, clears), func(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](tt.capacity, tinylfu.WithShards(1)) + if crowded { + c.OccupyReaders(0) + } + m := newModel(tt.capacity, c.Print) + rng := rand.New(rand.NewPCG(uint64(i), 1)) + ops := cachetest.RandomOps(rng, tt.keys, tt.steps) + if !clears { + ops = withoutClear(ops) + } + cachetest.Compare(t, c, m, ops) + + if got := c.Window(); got != m.window { + t.Errorf("the small queue is to have %d entries, want %d", got, m.window) + } + // The comparison is only worth something if the size of + // the small queue has moved, where there is room for it + // to. + if !clears && tt.capacity >= 8 && tt.keys > tt.capacity && m.moves < 2 { + t.Errorf("the size of the small queue has moved %d times in %d steps, want 2 at least", m.moves, tt.steps) + } + }) + } + } + } +} + +// withoutClear returns the operations with a Peek of no consequence in place of +// every Clear. +func withoutClear(ops iter.Seq[cachetest.Op]) iter.Seq[cachetest.Op] { + return func(yield func(cachetest.Op) bool) { + for op := range ops { + if op.Kind == cachetest.Clear { + op.Kind = cachetest.Peek + } + if !yield(op) { + return + } + } + } +} + +// TestKeysWithOneHash checks that a read of one key does not count for another +// key that happens to have the same hash. A reader finds its entry by the +// hash and has to look at the other one to see that it is not the one it +// wants; what the cache keeps must not depend on that, or it would depend on +// the seed of the hash. +func TestKeysWithOneHash(t *testing.T) { + t.Parallel() + + c := tinylfu.New[int, int](20, tinylfu.WithShards(1)) + stored, other := tinylfu.Twins(c) + + c.Set(stored, 0) + for range 3 { + if _, ok := c.Get(other); ok { + t.Fatalf("Get found key %d, which was never set", other) + } + if _, ok := c.Peek(other); ok { + t.Fatalf("Peek found key %d, which was never set", other) + } + } + if reads, ok := c.Reads(stored); !ok || reads != 0 { + t.Errorf("the entry for key %d has counted %d reads, %t; want none: those of %d, which has the same hash, were counted for it", + stored, reads, ok, other) + } + + // Its own are counted, up to three. + for want := uint32(1); want <= 5; want++ { + c.Get(stored) + c.Peek(stored) + if reads, _ := c.Reads(stored); reads != min(want, 3) { + t.Fatalf("the entry has counted %d reads after %d, want %d", reads, want, min(want, 3)) + } + } +} + +// TestHeldUpReader checks what a reader costs the others that stops in the +// middle of reading an entry, as one does whose goroutine has to make way: the +// value it is looking at stays alive, and that is all. Nobody waits for the +// reader, except for Clear, which promises to leave nothing behind. +func TestHeldUpReader(t *testing.T) { + t.Parallel() + + for _, shards := range []int{1, 4} { + c := tinylfu.New[int, *cachetest.Blob](64, tinylfu.WithShards(shards)) + + ref := func() weak.Pointer[cachetest.Blob] { + blob := new(cachetest.Blob) + c.Set(0, blob) + return weak.Make(blob) + }() + + release, ok := c.Hold(0) + if !ok { + t.Fatal("Hold found no entry for a key that was just set") + } + + // The entry is replaced and deleted, and so is everything else, many + // times over. None of it waits. + for i := range 10_000 { + c.Set(i%200, new(cachetest.Blob)) + if i%3 == 0 { + c.Delete((i + 1) % 200) + } + } + runtime.GC() + runtime.GC() + if ref.Value() == nil { + t.Fatalf("%d shards: the value that a reader is looking at was let go of", shards) + } + + // Clear does wait, and keeps the cache usable while it does. + done := make(chan struct{}) + go func() { + defer close(done) + c.Clear() + }() + select { + case <-done: + t.Fatalf("%d shards: Clear returned while a reader was looking at an entry", shards) + case <-time.After(20 * time.Millisecond): + } + for i := range 1_000 { + c.Set(i%200, nil) + c.Get(i % 200) + c.Delete(i % 200) + } + + release() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("%d shards: Clear is still waiting although the reader has left", shards) + } + if cachetest.Alive([]weak.Pointer[cachetest.Blob]{ref}, 0) != 0 { + t.Errorf("%d shards: the value is alive after Clear", shards) + } + + // The same for an entry that is in the cache when it is cleared, + // rather than one that has left it before. And Clear waits for + // that reader only: one that looks at an entry which came and + // went after Clear had begun is none of its business. + c.Set(1, new(cachetest.Blob)) + release, ok = c.Hold(1) + if !ok { + t.Fatal("Hold found no entry for a key that was just set") + } + done = make(chan struct{}) + go func() { + defer close(done) + c.Clear() + }() + select { + case <-done: + t.Fatalf("%d shards: Clear returned while a reader was looking at an entry it removed", shards) + case <-time.After(20 * time.Millisecond): + } + // Clear has begun once the cache is empty. An entry that comes and + // goes before that is one that Clear is right to wait for, and a + // goroutine may be that late to start when the machine is busy. + for deadline := time.Now().Add(10 * time.Second); c.Len() != 0; runtime.Gosched() { + if time.Now().After(deadline) { + t.Fatalf("%d shards: Clear has not removed the entry yet", shards) + } + } + c.Set(1, new(cachetest.Blob)) + later, ok := c.Hold(1) + if !ok { + t.Fatal("Hold found no entry for a key that was set while Clear was waiting") + } + c.Delete(1) + + release() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("%d shards: Clear is waiting for a reader of an entry that came after it", shards) + } + later() + if waits := c.Waits(); waits != 0 { + t.Errorf("%d shards: writers had to wait for a reader %d times", shards, waits) + } + runtime.KeepAlive(c) + } +} + +// TestKeysThatTakeACallToCompare checks that readers do not hold up writers +// when comparing two keys is not a matter of a few instructions. +// +// Go takes the CPU away from a goroutine that has had it for a while at the +// next function it calls. Were a reader to call one while it has an entry +// pinned, and comparing keys of some types is a call, readers would keep +// getting held up at just that point whenever there are more of them than +// CPUs. Enough of them, and a writer finds every spare node of its shard +// pinned and has to wait. +func TestKeysThatTakeACallToCompare(t *testing.T) { + t.Parallel() + + type wide struct { + a, b, c, d, e string + n int + } + keys := make([]any, 64) + for i := range keys { + if i%2 == 0 { + keys[i] = i + } else { + keys[i] = wide{a: "a", b: "b", c: "c", d: "d", e: strconv.Itoa(i), n: i} + } + } + + c := tinylfu.New[any, int](len(keys), tinylfu.WithShards(1)) + for i, key := range keys { + c.Set(key, i) + } + + var ( + wg sync.WaitGroup + stop atomic.Bool + ) + for r := range 16 * runtime.GOMAXPROCS(0) { + wg.Go(func() { + for i := r; !stop.Load(); i++ { + if v, ok := c.Get(keys[i%len(keys)]); ok && v < 0 { + t.Errorf("Get(%v) = %d", keys[i%len(keys)], v) + return + } + } + }) + } + for start, i := time.Now(), 0; time.Since(start) < 300*time.Millisecond; i++ { + c.Set(keys[i%len(keys)], i) + } + stop.Store(true) + wg.Wait() + + // The race detector turns what a reader does while it has an entry + // pinned into a series of calls, so with it readers are held up there + // after all. That says nothing about a program. + if waits := c.Waits(); waits != 0 && !cachetest.Race { + t.Errorf("the writer had to wait for readers %d times", waits) + } +} + +// TestReadersHelp checks that what a shard could not let go of, because +// readers were looking at it, does not stay around for want of a write: once +// those readers are gone, the ones that come after them see to it. +func TestReadersHelp(t *testing.T) { + t.Parallel() + + // Whichever way the readers read, they help. + t.Run("Get", func(t *testing.T) { + t.Parallel() + testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { c.Get(key) }) + }) + t.Run("Peek", func(t *testing.T) { + t.Parallel() + testReadersHelp(t, func(c *tinylfu.Cache[int, *cachetest.Blob], key int) { c.Peek(key) }) + }) +} + +func testReadersHelp(t *testing.T, read func(c *tinylfu.Cache[int, *cachetest.Blob], key int)) { + t.Helper() + + const ( + capacity = 16 // The shard has patience for two entries that are gone. + held = 4 + ) + c := tinylfu.New[int, *cachetest.Blob](capacity, tinylfu.WithShards(1)) + + refs := make([]weak.Pointer[cachetest.Blob], capacity) + for key := range capacity { + blob := new(cachetest.Blob) + refs[key] = weak.Make(blob) + c.Set(key, blob) + } + + // Four readers stop in the middle of reading an entry each. Every one + // of them is a goroutine of its own, as readers are, and stays one for + // as long as it looks: a goroutine can only look at so many entries at + // once, and four of them find a place to say so however few there are. + var ( + looking, readers sync.WaitGroup + moveOn = make(chan struct{}) + ) + for key := range held { + looking.Add(1) + readers.Go(func() { + release, ok := c.Hold(key) + if !ok { + t.Errorf("Hold(%d) found no entry", key) + } + looking.Done() + <-moveOn + release() + }) + } + looking.Wait() + + alive := func(limit int) int { return cachetest.Alive(refs, limit) } + + // Everything is deleted. The shard lets go of what it can. + for key := range capacity { + c.Delete(key) + } + if got := alive(held + 2); got < held || got > held+2 { + t.Fatalf("%d values are alive after all were deleted, want the %d that readers look at and up to 2 more", got, held) + } + + // The readers that are not held up try to help every so often, to no + // avail for now. They must not get in the way of a writer when they do. + var wg sync.WaitGroup + wg.Go(func() { + for i := range 20_000 { + c.Set(capacity+i%capacity, nil) + c.Delete(capacity + i%capacity) + } + }) + for range 4 { + wg.Go(func() { + for i := range 100_000 { + read(c, i%(2*capacity)) + } + }) + } + wg.Wait() + if got := alive(len(refs)); got < held { + t.Fatalf("%d values are alive, fewer than the %d that readers are looking at", got, held) + } + + // The readers move on, and nobody writes to the cache again. + close(moveOn) + readers.Wait() + for i := range 400_000 { + read(c, i%capacity) + } + if got := alive(0); got != 0 { + t.Errorf("%d values are alive after plenty of reads, want none: the readers did not help", got) + } + runtime.KeepAlive(c) +} + +// TestCrowdedReaders runs the tests of reading and writing at the same time +// once more, with caches whose readers find few of the slots free in which +// they say what they are looking at, or none. They then read under the lock +// of the shard, as they do when the index changes under them, and that must +// not make a difference to anyone. +func TestCrowdedReaders(t *testing.T) { + t.Parallel() + + for _, leave := range []int{0, 3} { + t.Run(fmt.Sprintf("leave=%d", leave), func(t *testing.T) { + t.Parallel() + + cachetest.Versions(t, func(capacity, shards int) evict.Cache[string, cachetest.Wide] { + c := tinylfu.New[string, cachetest.Wide](capacity, tinylfu.WithShards(shards)) + c.OccupyReaders(leave) + return c + }) + cachetest.Present(t, func(capacity, shards int) evict.Cache[int, int] { + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(shards)) + c.OccupyReaders(leave) + return c + }) + }) + } +} + +func FuzzAgainstModel(f *testing.F) { + f.Add(3, []byte{}) + f.Add(1, []byte{128, 129, 0, 130, 1}) + f.Add(3, []byte{128, 129, 130, 131, 128, 129, 130, 0, 131}) + f.Add(10, []byte("an arbitrary sequence of operations \xff on a small cache")) + + f.Fuzz(func(t *testing.T, capacity int, ops []byte) { + capacity = max(capacity%40, 0) + c := tinylfu.New[int, int](capacity, tinylfu.WithShards(1)) + cachetest.Compare(t, c, newModel(capacity, c.Print), cachetest.BytesOps(ops)) + }) +} From 644a16ea77c4e03d7ad60414f42206ef56e373f1 Mon Sep 17 00:00:00 2001 From: Vsevolod Strukchinsky Date: Wed, 11 Nov 2026 21:27:18 +0500 Subject: [PATCH 4/4] Benchmark tinylfu, and replay and measure everything again tinylfu joins the caches of the benchmarks, as one whose hit ratio varies from one replay to the next like otter's: its sketch has keys share counters by a hash that is seeded at random. The reports are made again for all caches, and the README and the docs that quote them follow the new numbers. On the eleven traces tinylfu answers 47.9% of the requests on average, otter 47.5% and s3fifo 44.8%, and tinylfu is never more than 5.3 points behind the best cache. The README now suggests it if in doubt, with what speaks against it: it was designed on these traces, it varies between runs, it is up to four points behind lru on Mixed at its larger capacities, and with eight CPUs and 1% writes it was 22% slower than s3fifo, which has not been explained. Co-Authored-By: Claude Opus 5.5 --- README.md | 228 +- bench/README.md | 16 +- bench/caches/caches.go | 21 + bench/cmd/hitratio/main.go | 4 +- bench/cmd/hitratio/main_test.go | 9 +- bench/cmd/hitratio/preamble.go | 22 +- bench/report/chart.go | 1 + bench/results/hitratio-cloudphysics.svg | 34 +- bench/results/hitratio-ds1.svg | 38 +- bench/results/hitratio-glimpse.svg | 34 +- bench/results/hitratio-mixed.svg | 36 +- bench/results/hitratio-msr.svg | 30 +- bench/results/hitratio-oltp.svg | 38 +- bench/results/hitratio-p8.svg | 38 +- bench/results/hitratio-s3.svg | 38 +- bench/results/hitratio-scarab.svg | 36 +- bench/results/hitratio-twitter.svg | 32 +- bench/results/hitratio-zipf.svg | 38 +- bench/results/hitratio.csv | 260 +- bench/results/hitratio.md | 263 +- bench/results/memory.md | 38 +- bench/results/sharding.csv | 782 +++--- bench/results/sharding.md | 225 +- bench/results/throughput-evict.svg | 84 +- bench/results/throughput-get.svg | 84 +- bench/results/throughput-mixed-writes-1.svg | 86 +- bench/results/throughput-mixed-writes-10.svg | 86 +- bench/results/throughput-mixed-writes-25.svg | 86 +- .../throughput-readthrough-capacity-16k.svg | 86 +- .../throughput-readthrough-capacity-1m.svg | 84 +- bench/results/throughput-set.svg | 93 +- bench/results/throughput.md | 120 +- bench/results/throughput.txt | 2410 +++++++++-------- lru/lru.go | 9 +- 34 files changed, 3095 insertions(+), 2394 deletions(-) diff --git a/README.md b/README.md index 31e425c..1c4c36d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/floatdrop/evict.svg)](https://pkg.go.dev/github.com/floatdrop/evict) [![CI](https://github.com/floatdrop/evict/actions/workflows/ci.yml/badge.svg)](https://github.com/floatdrop/evict/actions/workflows/ci.yml) -Fixed-capacity in-memory caches for Go: one API, four eviction policies. +Fixed-capacity in-memory caches for Go: one API, five eviction policies. | Package | Evicts | Reads | |:---|:---|:---| @@ -11,6 +11,7 @@ Fixed-capacity in-memory caches for Go: one API, four eviction policies. | [`twoq`](https://pkg.go.dev/github.com/floatdrop/evict/twoq) | with [2Q](http://www.vldb.org/conf/1994/P439.PDF), which keeps new entries apart from proven ones | take the lock of a shard | | [`sieve`](https://pkg.go.dev/github.com/floatdrop/evict/sieve) | with [SIEVE](https://www.usenix.org/conference/nsdi24/presentation/zhang-yazhuo), a queue with lazy promotion | take no lock | | [`s3fifo`](https://pkg.go.dev/github.com/floatdrop/evict/s3fifo) | with [S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147), which makes new entries prove themselves in a small queue first | take no lock | +| [`tinylfu`](https://pkg.go.dev/github.com/floatdrop/evict/tinylfu) | with S3-FIFO's queues and the admission of [TinyLFU](https://arxiv.org/abs/1512.00727): once the cache is full, a new entry gets into its main queue only if its key has been asked for more often of late than the entry it would push out | take no lock | All of them @@ -21,7 +22,7 @@ All of them - **tell you what they dropped**: `Set` returns the entry it replaced or evicted, so that whatever the entry held on to can be released; - **are safe for concurrent use**: entries are spread over shards that are locked one by one, - and `sieve` and `s3fifo` read without a lock at all; + and `sieve`, `s3fifo` and `tinylfu` read without a lock at all; - depend on nothing but the standard library. ``` @@ -38,11 +39,11 @@ package main import ( "fmt" - "github.com/floatdrop/evict/s3fifo" + "github.com/floatdrop/evict/tinylfu" ) func main() { - cache := s3fifo.New[string, int](10_000) + cache := tinylfu.New[string, int](10_000) cache.Set("answer", 42) @@ -62,7 +63,7 @@ func main() { } ``` -The four packages have the same methods, which are also the interface +The five packages have the same methods, which are also the interface [`evict.Cache`](https://pkg.go.dev/github.com/floatdrop/evict#Cache): | Method | | @@ -75,36 +76,41 @@ The four packages have the same methods, which are also the interface | `Clear()` | Removes every entry. | | `All()` | Iterates over the entries. | -`New` takes options. All four packages have `WithShards`. `twoq` has `WithRecentRatio` and +`New` takes options. All five packages have `WithShards`. `twoq` has `WithRecentRatio` and `WithGhostRatio` for the sizes of its queues, and `s3fifo` has `WithSmallRatio`, `WithGhostRatio`, `WithPromotionReads` and `WithGhostReturns` for tuning it to a workload -that is known; see [what that is worth](bench/results/tuning.md). +that is known; see [what that is worth](bench/results/tuning.md). `tinylfu` has nothing else +to set: it finds the size of its small queue by itself. ## Choosing a policy -**If in doubt, use `s3fifo`.** Of the four it answers the most requests on nine of the eleven -traces that the benchmarks replay, its reads take no lock, and it never does much worse -than `lru`: about two points at the most, where `sieve` can lose everything. It has a weak spot -of its own, though. On the trace called Mixed, at some capacities, it answers what `lru` -answers while `twoq` and otter answer four to six times as much. +**If in doubt, use `tinylfu`.** On the eleven traces that the benchmarks replay it answers +more requests on average than any other cache here, otter included: 47.9% of them, against +47.5% for otter and 44.8% for `s3fifo`. At no capacity is it more than 5.3 points behind +the best cache, and its reads take no lock. Two things temper that. It was designed with +these very traces at hand; on fourteen others, in a simulation of the policy that is not +part of this repository, it came out level with S3-FIFO and W-TinyLFU. And it has weak +spots of its own: on the trace called Mixed, at its larger capacities, it answers up to four +points less than `lru`, and where every key comes back once or twice after much the same +while and never again, it can answer far less than `lru` does; see +[its documentation](https://pkg.go.dev/github.com/floatdrop/evict/tinylfu). | | Use it when | Mind | |:---|:---|:---| -| `s3fifo` | you do not know the workload, or it is read far more than it is written | Writes cost about twice what they do in `lru`, and at worst three times. A cache of fewer than ten entries per shard is too small for it. | -| `sieve` | several CPUs read through the cache and it has to be as fast as can be, in less memory than `s3fifo` takes, and you have measured that your workload is one it copes with | It can fail outright. On one of the traces a SIEVE cache in one piece answers less than 1% of the requests where LRU answers 44%; see [when not to use it](https://pkg.go.dev/github.com/floatdrop/evict/sieve#hdr-When_not_to_use_it). On the traces here its hit ratio is ahead of that of `s3fifo` on one only. | +| `tinylfu` | you do not know the workload, or it has loops or scans in it, and it is read far more than it is written | Writes that evict get 5 to 10% less done than in `s3fifo`, and with eight CPUs and 1% writes among the reads it was 22% slower than `s3fifo` here. Its hit ratio varies from one run to the next, by up to 5.6 points on these traces: keys share counters in its sketch by a hash that is seeded at random. | +| `s3fifo` | you want the same answers on every run, or the workload is read far more than it is written and has no loops in it | It falls far behind on Mixed, a trace with loops, and a few points behind on Glimpse. On Mixed, at some capacities, it answers what `lru` answers while `tinylfu`, `twoq` and otter answer four to six times as much. Writes cost about twice what they do in `lru`, and at worst three times. A cache of fewer than ten entries per shard is too small for it. | +| `sieve` | several CPUs read through the cache and it has to be as fast as can be, in less memory than `s3fifo` and `tinylfu` take, and you have measured that your workload is one it copes with | It can fail outright. On one of the traces a SIEVE cache in one piece answers less than 1% of the requests where LRU answers 44%; see [when not to use it](https://pkg.go.dev/github.com/floatdrop/evict/sieve#hdr-When_not_to_use_it). On the traces here its hit ratio is ahead of that of `s3fifo` on one only. | | `twoq` | a quarter or more of the operations are writes, or one goroutine at a time uses the cache | It takes more memory than `lru` and `sieve`, because it keeps a node for every key it remembers. Reads take a lock. | -| `lru` | memory is what counts, there are a great many small caches, or you want the behavior that everyone knows | The lowest hit ratio of the four on nine traces of eleven. Reads take a lock. | +| `lru` | memory is what counts, there are a great many small caches, or you want the behavior that everyone knows | The lowest hit ratio of the five on nine traces of eleven. Reads take a lock. | And when to use [otter](https://github.com/maypok86/otter) instead: when you need what -these caches do not have (expiry, weights, loading, statistics); when the worst case matters -more than the usual one, since otter is never more than six points behind the best cache -on these traces and each of the four here is thirty or more behind somewhere; or when eight -CPUs read a cache while a tenth to a quarter of their operations overwrite entries. (For -writes alone, `twoq` and `lru` stay ahead.) The numbers are below. +these caches do not have (expiry, weights, loading, statistics), or when eight CPUs read a +cache while a tenth to a quarter of their operations overwrite entries. (For writes alone, +`twoq` and `lru` stay ahead.) The numbers are below. ## Benchmarks -The benchmarks are in [`bench`](bench), a module of its own, and compare the four caches +The benchmarks are in [`bench`](bench), a module of its own, and compare the five caches with otter v2, the state of the art among Go caches, and with hashicorp's `golang-lru`, a cache behind a single lock. [bench/README.md](bench/README.md) says what is measured and how; `make bench` runs it all again. What follows is a summary of @@ -115,56 +121,63 @@ how; `make bench` runs it all again. What follows is a summary of The share of requests, in percent, that a cache answers when a trace is replayed against it, averaged over the eight capacities at which each trace is replayed. Ten of the traces are public recordings of real systems, checked against a SHA-256 before they are used; -`optimal` is the bound that no cache can beat. Otter's hit ratio differs from one replay to -the next, by up to 3.8 points, so it is replayed eleven times: its number is the median, -and a number is in bold only if the cache leads whichever of those replays it is compared -with. - -| trace | evict/lru | evict/twoq | evict/sieve | evict/s3fifo | otter | golang-lru/2q | optimal | -|:---|---:|---:|---:|---:|---:|---:|---:| -| Zipf | 29.97 | 36.53 | 37.55 | **38.25** | 37.86 | 37.61 | 44.07 | -| S3 | 23.33 | 32.78 | 35.54 | 38.90 | **44.57** | 33.15 | 59.37 | -| DS1 | 23.70 | 28.88 | 38.87 | 37.05 | **46.52** | 34.47 | 50.21 | -| OLTP | 32.40 | 39.07 | 30.60 | 39.11 | 38.92 | 37.60 | 52.77 | -| P8 | 24.10 | 32.11 | 27.66 | 35.31 | 36.22 | 31.45 | 53.53 | -| Glimpse | 21.81 | 27.40 | 30.14 | 43.24 | **45.21** | 33.20 | 47.81 | -| Mixed | 44.11 | 56.80 | 0.43 | 43.84 | **61.62** | 35.97 | 64.22 | -| Scarab | 59.10 | 61.00 | 60.53 | **61.60** | 58.97 | 61.14 | 70.30 | -| Twitter | 58.37 | 59.72 | 58.99 | **60.32** | 59.28 | 58.80 | 65.46 | -| MSR | 64.40 | 66.31 | 66.16 | **67.22** | 66.56 | 66.57 | 73.60 | -| CloudPhysics | 20.90 | 24.41 | 24.70 | **27.59** | 26.73 | 25.59 | 34.23 | - -- **`s3fifo` and otter share the wins.** Of the 88 combinations of trace and capacity, - `s3fifo` wins 37 outright and otter 31. Six go to one of the 2Q caches, and 14 have no - clear winner. -- **`s3fifo` is ahead** on Zipf and on the traces of a recommendation service, a key-value - cache and block storage (Scarab, Twitter, MSR, CloudPhysics), by 0.4 to 2.6 points on - average and by no more than 4.8 at any capacity. -- **Otter is ahead by more where it is ahead**: on two of the disk traces of the ARC paper - and on the two traces with loops in them (S3, DS1, Glimpse, Mixed), by 2 to 18 points - over `s3fifo` on average, and by 61 at the capacity where `s3fifo` does worst. Against the - best of the four on each of those traces, which is `sieve` on DS1 and `twoq` on Mixed, it - is 2 to 8 points. On OLTP and P8 the two are within a point of each other. +`optimal` is the bound that no cache can beat. The hit ratios of otter and of `tinylfu` +differ from one replay to the next, by up to 4.1 and 5.6 points, so they are replayed eleven +times: their numbers are medians, and a number is in bold only if the cache leads whichever +of those replays it is compared with. + +| trace | evict/lru | evict/twoq | evict/sieve | evict/s3fifo | evict/tinylfu | otter | golang-lru/2q | optimal | +|:---|---:|---:|---:|---:|---:|---:|---:|---:| +| Zipf | 29.97 | 36.53 | 37.55 | **38.25** | 37.66 | 37.90 | 37.61 | 44.07 | +| S3 | 23.33 | 32.78 | 35.54 | 38.90 | **44.83** | 44.57 | 33.15 | 59.37 | +| DS1 | 23.70 | 28.88 | 38.87 | 37.05 | **47.07** | 46.54 | 34.47 | 50.21 | +| OLTP | 32.40 | 39.07 | 30.60 | 39.11 | 38.76 | 38.97 | 37.60 | 52.77 | +| P8 | 24.10 | 32.11 | 27.66 | 35.31 | **37.86** | 36.20 | 31.45 | 53.53 | +| Glimpse | 21.81 | 27.40 | 30.14 | 43.24 | 45.20 | 45.28 | 33.20 | 47.81 | +| Mixed | 44.11 | 56.80 | 0.43 | 43.84 | 60.32 | **61.61** | 35.97 | 64.22 | +| Scarab | 59.10 | 61.00 | 60.53 | **61.60** | 60.31 | 58.99 | 61.14 | 70.30 | +| Twitter | 58.37 | 59.72 | 58.99 | **60.32** | 59.59 | 59.29 | 58.80 | 65.46 | +| MSR | 64.40 | 66.31 | 66.16 | **67.22** | 67.06 | 66.54 | 66.57 | 73.60 | +| CloudPhysics | 20.90 | 24.41 | 24.70 | 27.59 | 27.80 | 26.77 | 25.59 | 34.23 | + +- **`s3fifo` and `tinylfu` share the wins.** Of the 88 combinations of trace and capacity, + `s3fifo` wins 32 outright and `tinylfu` 25. Six go to one of the 2Q caches, one to otter, + and 24 have no clear winner. +- **`s3fifo` is ahead** on Zipf and OLTP and on the traces of a recommendation service, a + key-value cache and block storage (Scarab, Twitter, MSR), by 0.2 to 1.3 points over + `tinylfu` on average and by no more than 2.5 at any capacity. +- **`tinylfu` and otter are ahead by more where they are ahead**: on two of the disk traces + of the ARC paper and on the two traces with loops in them (S3, DS1, Glimpse, Mixed), by 2 + to 18 points over `s3fifo` on average, and by 58 and 61 at the capacity where `s3fifo` + does worst. On P8 and CloudPhysics `tinylfu` is ahead of every other cache. +- **`tinylfu` and otter are close.** `tinylfu` is ahead on average on seven of the traces, + by up to 1.7 points (P8), and otter on the other four, by up to 1.3 (Mixed). Of the 88 + combinations, `tinylfu` is ahead on every replay in 42 and behind in 14. These are the + traces that `tinylfu` was designed with, which favors it. - **`sieve` is close behind `s3fifo` where it works**: about a point on Zipf, Scarab, Twitter and MSR, about three on S3 and CloudPhysics, and ahead of it on DS1. Where it does not work it is far behind: 8 to 13 points on OLTP, P8 and Glimpse, and on Mixed it answers next to nothing. - **`twoq` is ahead of `lru` on the average of every trace**, though not at every capacity: - `lru` leads in 10 of the 88 combinations. It is the best of the four on Mixed. -- **Otter has no bad trace.** In none of the 88 combinations is it more than 5.9 points - behind the best cache. Each of the four here is somewhere: `twoq` by 32 points, `s3fifo` - and `lru` by 63, `sieve` by 76. + `lru` leads in 10 of the 88 combinations. Of the five here only `tinylfu` beats it on Mixed. +- **Neither otter nor `tinylfu` has a bad trace.** In none of the 88 combinations is otter + more than 5.9 points behind the best cache, or `tinylfu` more than 5.3. Each of the other + four here is somewhere: `twoq` by 32 points, `s3fifo` and `lru` by 63, `sieve` by 76. - Averages hide a lot. [hitratio.md](bench/results/hitratio.md) has every capacity, with a chart for each trace. These numbers are for caches in one piece, so that the policy is all that is compared. A program gets shards by default, and [sharding.md](bench/results/sharding.md) has what that -changes to the median of eleven runs: less than a tenth of a point for `lru`, about a -tenth at the most for `s3fifo`, the same for `twoq` but for two cases of half a point and -of a point. For `sieve` it changes a great deal on two of the traces it does badly on, for -the better: 3 to 4 points on OLTP and anything from 2 to 41 on Mixed, depending on the -capacity. On P8 it is a point at the most, and on Glimpse nothing. It was measured for a machine with eight CPUs; one with more gives a cache of -some size more and smaller shards. +changes to the median of eleven runs: about a tenth of a point at the most for `s3fifo`; +the same for `lru` but for one case of more than a point on Glimpse, a short trace; and the +same for `twoq` but for two cases of half a point and of a point. For `sieve` it changes a +great deal on two of the traces it does badly on, for the better: 3 to 4 points on OLTP and +anything from 2 to 41 on Mixed, depending on the capacity. On P8 it is a point or so at the +most, and on Glimpse nothing. `tinylfu` loses a little in shards: about a point on DS1 at a +million entries, half a point on S3 and P8 at their smaller capacities, and 2.6 points on +Mixed at 750 entries; some of what the report shows for it is not the shards but how much +it varies from one replay to the next anyway. It was measured for a machine with eight CPUs; +one with more gives a cache of some size more and smaller shards. ### Throughput @@ -174,35 +187,46 @@ numbers are medians of five runs on an Apple M3 Max with Go 1.27.3; [throughput.md](bench/results/throughput.md) has two and four CPUs as well, the range of the runs, and which differences are beyond noise. -| Benchmark | evict/lru | evict/twoq | evict/sieve | evict/s3fifo | otter | golang-lru | -|:---|---:|---:|---:|---:|---:|---:| -| `Get`: reads, all of them hits | 37.0 → 36.4 | 48.3 → 47.3 | 35.0 → 278.0 | 36.2 → 276.6 | 26.6 → 205.3 | 33.6 → 4.1 | -| `Mixed`: the same, 10% of them writes | 37.3 → 32.2 | 44.5 → 41.5 | 33.0 → 67.7 | 33.0 → 67.8 | 21.4 → 81.9 | 33.6 → 4.2 | -| `Set`: writes that replace | 35.9 → 25.3 | 43.4 → 29.3 | 21.9 → 12.9 | 21.7 → 13.1 | 8.6 → 15.7 | 31.7 → 4.2 | -| `Evict`: writes that evict | 16.5 → 31.4 | 16.8 → 31.4 | 13.3 → 12.0 | 8.5 → 10.5 | 3.5 → 5.8 | 7.4 → 3.2 | -| `ReadThrough`: read, store on a miss | 30.1 → 34.5 | 31.6 → 34.8 | 28.5 → 52.1 | 21.9 → 43.2 | 12.4 → 23.6 | 16.5 → 3.5 | -| `ReadThrough`, a million entries | 8.9 → 29.4 | 7.8 → 28.6 | 8.4 → 41.6 | 7.3 → 37.0 | 4.1 → 17.7 | 6.2 → 2.8 | +| Benchmark | evict/lru | evict/twoq | evict/sieve | evict/s3fifo | evict/tinylfu | otter | golang-lru | +|:---|---:|---:|---:|---:|---:|---:|---:| +| `Get`: reads, all of them hits | 39.7 → 35.9 | 46.7 → 44.4 | 35.6 → 265.8 | 36.1 → 274.7 | 35.0 → 258.1 | 27.2 → 202.3 | 34.2 → 4.3 | +| `Mixed`: the same, 10% of them writes | 37.6 → 34.0 | 42.5 → 42.8 | 32.9 → 68.6 | 33.4 → 69.9 | 33.1 → 62.6 | 21.7 → 80.8 | 33.5 → 4.4 | +| `Set`: writes that replace | 36.5 → 24.9 | 43.3 → 28.4 | 22.4 → 13.8 | 22.5 → 13.6 | 22.2 → 12.8 | 8.9 → 15.9 | 32.1 → 4.3 | +| `Evict`: writes that evict | 16.5 → 31.3 | 16.9 → 30.9 | 13.4 → 12.0 | 8.6 → 10.5 | 8.2 → 9.8 | 3.6 → 5.9 | 7.5 → 3.3 | +| `ReadThrough`: read, store on a miss | 30.3 → 34.1 | 31.9 → 33.7 | 28.6 → 53.3 | 22.1 → 43.1 | 21.7 → 46.0 | 12.7 → 23.6 | 16.7 → 3.6 | +| `ReadThrough`, a million entries | 8.8 → 30.1 | 8.4 → 28.7 | 8.9 → 41.5 | 7.2 → 36.8 | 7.5 → 37.8 | 4.3 → 17.9 | 6.3 → 2.9 | ![Throughput of ReadThrough with a million entries by number of CPUs](bench/results/throughput-readthrough-capacity-1m.svg) -- **Reads scale in `sieve` and `s3fifo`**, as they do in otter: with eight CPUs both read - about a third faster than otter does. `lru` and `twoq` read under a lock, and with eight - CPUs they are slower than with four. -- **With one CPU all four are faster than otter** at everything measured, by a factor of +- **Reads scale in `sieve`, `s3fifo` and `tinylfu`**, as they do in otter: with eight CPUs + they read 28 to 36% faster than otter does. `lru` and `twoq` read under a lock, + and with eight CPUs they are slower than with four. +- **With one CPU all five are faster than otter** at everything measured, by a factor of 1.3 to 5. -- **Reading through is what `sieve` and `s3fifo` are best at.** `ReadThrough` is the - benchmark that comes closest to a cache in front of a database. From two CPUs on `sieve` - is the fastest in it, and from four on `s3fifo` is the second; with one CPU it is `twoq` - or `lru`. With eight, `sieve` and `s3fifo` do about twice what otter does. -- **Writes are where `lru` and `twoq` are at home.** They change an entry in place. `sieve` - and `s3fifo` move a replaced entry to a fresh node so that readers need no lock. A write - that replaces costs them 1.6 to 2.5 times as much, and up to three times with four CPUs; - one that evicts costs 1.2 to 3 times as much. Their replacing writes are fastest with one - CPU, and every cache here writes slower with eight CPUs than with four. -- **Otter wins when eight CPUs keep overwriting the entries that they read**: by a fifth - over `sieve` and `s3fifo` with 10% writes, and by nearly a quarter over `twoq` with 25%. - With four CPUs the fastest of the caches here are still well ahead, though `lru` is - behind otter already with 10% writes. +- **Reading through is what the caches that read without a lock are best at.** + `ReadThrough` is the benchmark that comes closest to a cache in front of a database. From + two CPUs on `sieve` is the fastest in it, and from four on `s3fifo` or `tinylfu` is the + second; with one CPU it is `twoq`, or `sieve` and `lru` with a million entries. With + eight CPUs, those three do about twice what otter does. +- **`tinylfu` reads about as fast as `s3fifo`, and writes a little slower.** With one, two + and eight CPUs its `Get` is within 6% of that of `s3fifo`; with four, both vary by about a + third from run to run. With eight CPUs it reads through faster, because it misses less. + Its writes do more, for its sketch: those that evict get 5 to 10% less done, and those + that replace about as much but for 17% less with four CPUs. With 1% writes among the + reads and eight CPUs it falls further behind, by 22%; with 10% the medians differ by 10%, + but the runs overlap. In an experiment that is not part of this repository, taking out + what `tinylfu` does differently on that path did not close the gap, and it has not been + explained yet. +- **Writes are where `lru` and `twoq` are at home.** They change an entry in place. `sieve`, + `s3fifo` and `tinylfu` move a replaced entry to a fresh node so that readers need no lock. + A write that replaces costs them 1.6 to 2.6 times as much, and up to 3.4 times with four + CPUs; one that evicts costs 1.2 to 3.2 times as much. Their replacing writes are fastest + with one CPU, and every cache here writes slower with eight CPUs than with four. +- **Otter wins when eight CPUs keep overwriting the entries that they read**: by a sixth + over `sieve` and `s3fifo` and by more than a quarter over `tinylfu` with 10% writes, and + by nearly a quarter over `twoq` with 25%. With four CPUs the fastest of the caches here are + still well ahead, though `lru` is behind otter up to 10% writes and ahead of it only with + 25%. - **One machine is one machine.** In the numbers that otter publishes, from Linux on x86, its writes get five times as fast between one CPU and eight; here they do not get twice as fast. How the caches compare on such a machine may well differ, and the `Benchmarks` @@ -219,16 +243,17 @@ entries. [memory.md](bench/results/memory.md) has every size, and strings. | | Bytes per entry | of which the collector has to scan | CPU time of one collection, a million entries | ten million | |:---|---:|---:|---:|---:| | evict/lru | 46–58 | 0–1 | 0.2 ms | 0.2 ms | -| evict/sieve | 54–71 | 0–2 | 0.3 ms | 0.2 ms | -| evict/twoq | 74–82 | 0–1 | 0.3 ms | 0.2 ms | +| evict/sieve | 54–72 | 0–2 | 0.4 ms | 0.2 ms | +| evict/twoq | 74–82 | 0–1 | 0.4 ms | 0.2 ms | | evict/s3fifo | 68–90 | 0–2 | 0.3 ms | 0.2 ms | -| otter | 84–95 | 41–50 | 64 ms | 631 ms | -| golang-lru | 110–127 | 94–111 | 65 ms | 736 ms | -| golang-lru/2q | 156–183 | 136–163 | 99 ms | 1,229 ms | +| evict/tinylfu | 67–82 | 0–2 | 0.4 ms | 0.2 ms | +| otter | 83–94 | 40–50 | 64 ms | 634 ms | +| golang-lru | 110–127 | 93–111 | 65 ms | 740 ms | +| golang-lru/2q | 156–183 | 136–163 | 97 ms | 1,167 ms | The bytes per entry go up and down with the capacity, because the index of a cache has a power of two of slots, and the ranges overlap: at 100,000 entries `s3fifo` takes 90 bytes -and otter 84. What does not overlap is the work for the garbage collector. A cache that is +and otter 83. What does not overlap is the work for the garbage collector. A cache that is made of pointers gives it more to do the larger it is, and a cache of this module gives it next to nothing at any size. That is for keys and values that hold no pointers. If they do, as strings do, the collector has to go through the entries to follow them, which was not @@ -249,14 +274,15 @@ bound that it rarely reaches exactly. By default a cache of fewer than 512 entri shard, and a larger one has up to eight per CPU, of 256 entries or more. `WithShards(1)` gives the textbook behavior of a policy. -**Reads without a lock** (`sieve`, `s3fifo`). The index is made of atomic words. A reader +**Reads without a lock** (`sieve`, `s3fifo`, `tinylfu`). The index is made of atomic words. A reader that finds its key says which node it is about to read by writing the node's number into a slot of its own, checks that the index still points there, and reads. A writer never changes a node that readers can reach: replacing a value moves the entry to a fresh node, and a node that has left the index is reused only after a look at the slots shows that nobody has it. A read thus costs two writes to a cache line that the reading CPU has to itself, and nothing -that all CPUs share. Both policies were chosen for this: reading an entry sets a mark or -counts to three, and writes nothing once that is done. +that all CPUs share. The three policies were chosen for this: reading an entry sets a mark or +counts to three, and writes nothing once that is done. `tinylfu` also adds the read to a +count of hits in the reader's slot, a third write to the same line. Two consequences are worth knowing: @@ -267,6 +293,18 @@ Two consequences are worth knowing: - A cache has a table of those slots, of 2 to 4 KiB per CPU. For a great many small caches, use `lru`. +**Admission by frequency** (`tinylfu`). Each shard keeps a count-min sketch of how often +keys have been asked for of late, whether it holds them or not: four 4-bit counters per key +in one 64-byte block, halved with the tally of what they have counted whenever that tally +reaches ten requests per entry. Only writers touch it. A read counts on its entry, and the +next write that comes by the entry tells the sketch. A new entry waits in a small queue, +and once the shard is full and the small queue has its share, an entry whose turn comes +gets into the main queue only if the sketch has its key down as asked for more often than +the entry that the main queue would evict next. How much room the small queue gets, the +cache finds out as Caffeine does, by hill climbing on the share of requests it answers. It +takes that share from the hits that readers count in their slots and, as Caffeine does, +counts a write of a key that it does not hold as a miss. + **Tests.** Every cache passes the same conformance suite, is compared operation by operation against a simple model of its policy, and is fuzzed against that model. The lock-free path is tested for values that tear, keys that go missing while they are present, diff --git a/bench/README.md b/bench/README.md index fe5011b..1d6b1ca 100644 --- a/bench/README.md +++ b/bench/README.md @@ -22,7 +22,7 @@ libraries they are compared with. | Name | What it is | |:---|:---| -| `evict/lru`, `evict/twoq`, `evict/sieve`, `evict/s3fifo` | The caches of this module. | +| `evict/lru`, `evict/twoq`, `evict/sieve`, `evict/s3fifo`, `evict/tinylfu` | The caches of this module. | | `otter` | [otter](https://github.com/maypok86/otter) v2, the state of the art among Go caches. It evicts with adaptive W-TinyLFU, reads without locks, and writes through buffers that a background task drains. | | `golang-lru`, `golang-lru/2q` | [hashicorp/golang-lru](https://github.com/hashicorp/golang-lru) v2: an LRU and a 2Q cache behind a single lock, the way most caches used to be built. They are here for perspective, and `golang-lru` also as a check: the tests require `evict/lru` to agree with it on every single request of a trace. Its 2Q is not the same algorithm as `evict/twoq`: it promotes an entry that is read again while it is still recent, where the 2Q of the paper waits for it to come back after it has been evicted. | | `optimal` | Not a cache but the bound on hit ratios: what a cache would score that knew the requests to come ([Belady's algorithm](https://en.wikipedia.org/wiki/Cache_replacement_policies#Belady's_algorithm), with the option of not storing a key at all). | @@ -43,14 +43,16 @@ background, as it is in its own simulator. The traces are downloaded on first use, into the user's cache directory unless `-dir` says otherwise, and each file is checked against the SHA-256 that [dataset/dataset.go](dataset/dataset.go) has on record for it. A run on another machine -therefore replays the very same requests, and the numbers for the caches of this module -and for golang-lru come out the same to the last digit. +therefore replays the very same requests, and the numbers for golang-lru and for the caches +of this module but `evict/tinylfu` come out the same to the last digit. Otter's do not. It admits some entries at random, and its hit ratio moves from one replay to the next because of it: by several percentage points at some capacities, and hardly at -all at others. A cache like that is replayed eleven times (`-runs`), and the report has the -median and the range. A cache is only called the winner of a row if it beat the others on -every replay. +all at others. Nor do those of `evict/tinylfu`, whose sketch has keys share counters by a +hash that is seeded at random, so that which keys it confuses differs from one replay to the +next. A cache like that is replayed eleven times (`-runs`), and the report has the median and +the range. A cache is only called the winner of a row if it beat the others on every +replay. | Dataset | Workload | Requests | Keys | Download | |:---|:---|---:|---:|---:| @@ -149,7 +151,7 @@ measures comes after that. With a million entries it is not: the share of reads `ReadThrough` reports for those caches is that of a cache which is still learning, and a few tenths of a percentage point higher where a cache got through more requests in its second, as it does with more CPUs. Otter's share moves with the number of CPUs in the small -cache as well, from 87.0% to 87.6%. That is not the warm-up, and why it does was not looked +cache as well, between 86.9% with two CPUs and 87.6% with eight. That is not the warm-up, and why it does was not looked into. | Benchmark | What every goroutine does | diff --git a/bench/caches/caches.go b/bench/caches/caches.go index 02c0502..8f6205c 100644 --- a/bench/caches/caches.go +++ b/bench/caches/caches.go @@ -9,6 +9,7 @@ import ( "github.com/floatdrop/evict/lru" "github.com/floatdrop/evict/s3fifo" "github.com/floatdrop/evict/sieve" + "github.com/floatdrop/evict/tinylfu" "github.com/floatdrop/evict/twoq" ) @@ -104,6 +105,21 @@ func All[K comparable, V any]() []Kind[K, V] { return s3fifoCache[K, V]{s3fifo.New[K, V](capacity, s3fifo.WithShards(shards))} }, }, + { + Name: "evict/tinylfu", + New: func(capacity int) Cache[K, V] { + return tinylfuCache[K, V]{tinylfu.New[K, V](capacity)} + }, + Exact: func(capacity int) Cache[K, V] { + return tinylfuCache[K, V]{tinylfu.New[K, V](capacity, tinylfu.WithShards(1))} + }, + Shards: func(capacity, shards int) Cache[K, V] { + return tinylfuCache[K, V]{tinylfu.New[K, V](capacity, tinylfu.WithShards(shards))} + }, + // Which keys share counters in its sketch is up to a hash that + // is seeded at random. + Varies: true, + }, { Name: "otter", // Setting the initial capacity is not a default, but it is what @@ -170,6 +186,11 @@ type s3fifoCache[K comparable, V any] struct{ c *s3fifo.Cache[K, V] } func (c s3fifoCache[K, V]) Get(key K) (V, bool) { return c.c.Get(key) } func (c s3fifoCache[K, V]) Set(key K, value V) { c.c.Set(key, value) } +type tinylfuCache[K comparable, V any] struct{ c *tinylfu.Cache[K, V] } + +func (c tinylfuCache[K, V]) Get(key K) (V, bool) { return c.c.Get(key) } +func (c tinylfuCache[K, V]) Set(key K, value V) { c.c.Set(key, value) } + type otterCache[K comparable, V any] struct{ c *otter.Cache[K, V] } func (c otterCache[K, V]) Get(key K) (V, bool) { return c.c.GetIfPresent(key) } diff --git a/bench/cmd/hitratio/main.go b/bench/cmd/hitratio/main.go index 45b2115..8f4374a 100644 --- a/bench/cmd/hitratio/main.go +++ b/bench/cmd/hitratio/main.go @@ -166,7 +166,7 @@ type input struct { // defaultCaches are the columns of the report: the caches of this module, // the ones they are compared with, and the best that any cache could do. -var defaultCaches = []string{"evict/lru", "evict/twoq", "evict/sieve", "evict/s3fifo", "otter", "golang-lru/2q", report.Bound} +var defaultCaches = []string{"evict/lru", "evict/twoq", "evict/sieve", "evict/s3fifo", "evict/tinylfu", "otter", "golang-lru/2q", report.Bound} func allPlayers() []player { var players []player @@ -246,7 +246,7 @@ func shardingPlayers() []player { continue } players = append(players, - player{kind.Name, false, func(t *input, capacity int) int { + player{kind.Name, kind.Varies, func(t *input, capacity int) int { return replay.Hits(kind.Exact(capacity), t.keys) }}, player{kind.Name + " in shards", true, func(t *input, capacity int) int { diff --git a/bench/cmd/hitratio/main_test.go b/bench/cmd/hitratio/main_test.go index c7d01f0..37a0368 100644 --- a/bench/cmd/hitratio/main_test.go +++ b/bench/cmd/hitratio/main_test.go @@ -48,7 +48,7 @@ func TestRun(t *testing.T) { md := read(t, dir, "hitratio.md") for _, want := range []string{ "# Hit ratios\n", - "the median of 3 replays", + "the medians of 3 replays", "in this run by up to ", "Produced by `go run ./cmd/hitratio -caches evict/lru,evict/sieve@4,otter,optimal -runs 3 ", "## At a glance\n", @@ -118,9 +118,10 @@ func TestSharding(t *testing.T) { "as on a machine with 8 CPUs", "up to 64.", "### Small\n", - "| capacity | evict/lru | evict/twoq | evict/sieve | evict/s3fifo |\n", - // A cache of 100 entries is in one piece either way. - "| 100 | 0.00 | 0.00 | 0.00 | 0.00 |\n", + "| capacity | evict/lru | evict/twoq | evict/sieve | evict/s3fifo | evict/tinylfu |\n", + // A cache of 100 entries is in one piece either way. The last one + // is not the same on every replay even so. + "| 100 | 0.00 | 0.00 | 0.00 | 0.00 | ", } { if !strings.Contains(md, want) { t.Errorf("sharding.md lacks %q:\n%s", want, md) diff --git a/bench/cmd/hitratio/preamble.go b/bench/cmd/hitratio/preamble.go index 27ff989..3ff65e8 100644 --- a/bench/cmd/hitratio/preamble.go +++ b/bench/cmd/hitratio/preamble.go @@ -45,16 +45,17 @@ it is missing. The column `+"`optimal`"+` is not a cache but the bound that no c beat: what one would score that knew the requests to come. The caches of this module run in one piece here, with a single shard, and otter evicts on -the spot rather than in the background, as it does in its own simulator. The caches of -this module and those of golang-lru then evict the same way every time, and their numbers -are the same on every run. Otter's are not: it admits some entries at random, which moves -its hit ratio from one replay to the next, in this run %s. -Its number is the median of %d replays, which is also what the charts show, and the column -next to it has the lowest and the highest of them. That is not all the range there is: -of further replays, about %d in a hundred would come out below it or above it. +the spot rather than in the background, as it does in its own simulator. Most of the +caches then evict the same way every time, and their numbers are the same on every run. +Two do not. Otter admits some entries at random, and `+"`evict/tinylfu`"+` counts requests in a +sketch in which keys share counters, by a hash that is seeded at random. That moves their +hit ratios from one replay to the next, in this run %s. +Their numbers are the medians of %d replays, which is also what the charts show, and the +column next to each has the lowest and the highest of them. That is not all the range +there is: of further replays, about %d in a hundred would come out below it or above it. A number is in bold if the cache did better in that row than every other one, on each of -their replays. Where the range of otter reaches the best of the others, the row has no +their replays. Where the range of one cache reaches the best of the others, the row has no winner. A program does not get the caches of this module in one piece unless it asks for that: by @@ -101,6 +102,11 @@ so the hit ratio of a cache in shards differs from run to run. The number is the difference for the median of %d replays, and the brackets hold the differences for the worst and for the best of them. +`+"`evict/tinylfu`"+` differs from run to run in one piece as well: which keys share counters in +its sketch is up to the same hash. What its replays in shards are compared with is the +median of as many replays in one piece, so its brackets are never empty, not even where a +cache is too small to be split. + %s`, sharedCPUs, 8*sharedCPUs, runs, produced(command)) } diff --git a/bench/report/chart.go b/bench/report/chart.go index 248f5e3..e3318f3 100644 --- a/bench/report/chart.go +++ b/bench/report/chart.go @@ -51,6 +51,7 @@ var ( "otter": 4, "golang-lru": 5, "golang-lru/2q": 6, + "evict/tinylfu": 7, } light = [...]string{"#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948"} dark = [...]string{"#3987e5", "#d95926", "#199e70", "#c98500", "#d55181", "#008300", "#9085e9", "#e66767"} diff --git a/bench/results/hitratio-cloudphysics.svg b/bench/results/hitratio-cloudphysics.svg index e50ca7f..2958caa 100644 --- a/bench/results/hitratio-cloudphysics.svg +++ b/bench/results/hitratio-cloudphysics.svg @@ -1,6 +1,6 @@ CloudPhysics -Line chart of Hit ratio, % by Cache capacity, entries for evict/lru, evict/twoq, evict/sieve, evict/s3fifo, otter, golang-lru/2q, optimal. The numbers are in the table that goes with it. +Line chart of Hit ratio, % by Cache capacity, entries for evict/lru, evict/twoq, evict/sieve, evict/s3fifo, evict/tinylfu, otter, golang-lru/2q, optimal. The numbers are in the table that goes with it.