From 72c21e5548f9f25506d34fb55e63420bff6a19c4 Mon Sep 17 00:00:00 2001 From: Nick Brandt Date: Thu, 25 Jul 2024 12:03:54 -0400 Subject: [PATCH 1/2] Add support for CLOCK cache --- clock/clock.go | 138 ++++++++++++++++++++++++++++++++++++++++++++ clock/clock_test.go | 46 +++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 clock/clock.go create mode 100644 clock/clock_test.go diff --git a/clock/clock.go b/clock/clock.go new file mode 100644 index 00000000..cb06d0b7 --- /dev/null +++ b/clock/clock.go @@ -0,0 +1,138 @@ +//go:build go1.19 + +/* +Copyright 2024 Vimeo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clock + +import "sync/atomic" + +type bufferItem[K comparable, V any] struct { + key K + value V +} + +type buffer[K comparable, V any] []*bufferItem[K, V] + +func (b buffer[K, V]) insertAt(index int, key K, value V) { + b[index] = &bufferItem[K, V]{ + key: key, + value: value, + } +} + +// Cache is a cache based on the CLOCK cache policy. +// It stores elements in a ring buffer, and stores a +// touch count for each element in the cache which +// is uses to determine whether or not a given element +// should be evicted (lower touches means more likely +// to be evicted). +type Cache[K comparable, V any] struct { + MaxEntries int + OnEvicted func(key K, val V) + + indices map[K]int + buf buffer[K, V] + touches []atomic.Int64 + clockhand int +} + +func New[K comparable, V any](maxEntries int) *Cache[K, V] { + return &Cache[K, V]{ + MaxEntries: maxEntries, + indices: make(map[K]int, maxEntries), + buf: make(buffer[K, V], 0, maxEntries), + touches: make([]atomic.Int64, maxEntries), + clockhand: 0, + } +} + +// Add inserts a given key-value pair into the cache, +// evicting a previous entry if necessary +func (c *Cache[K, V]) Add(key K, val V) { + c.checkAndInit() + if index, hit := c.indices[key]; hit { + c.touches[index].Add(1) + return + } + // Not full yet, insert at clock hand + if !c.IsFull() { + c.buf = append(c.buf, &bufferItem[K, V]{ + key: key, + value: val, + }) + c.indices[key] = len(c.buf) - 1 + c.clockhand = (c.clockhand + 1) % len(c.buf) + return + } + // Full, evict by reference bit then replace + hand := c.clockhand + for c.touches[hand].Load() > 0 { + c.touches[hand].Add(-1) + hand = (hand + 1) % len(c.buf) + } + c.Evict(c.buf[hand].key) + c.buf.insertAt(hand, key, val) + c.indices[key] = hand + c.clockhand = hand +} + +// Get returns the value for a given key, if present. +// ok bool will be false if the key does not exist +func (c *Cache[K, V]) Get(key K) (value V, ok bool) { + c.checkAndInit() + if index, hit := c.indices[key]; hit { + c.touches[index].Add(1) + return c.buf[index].value, true + } + return +} + +// Evict will remove the given key, if present, from +// the cache +func (c *Cache[K, V]) Evict(key K) { + c.checkAndInit() + index, ok := c.indices[key] + if !ok { + return + } + delete(c.indices, key) + if c.OnEvicted != nil { + c.OnEvicted(key, c.buf[index].value) + } + c.buf[index] = nil + c.touches[index].Store(0) +} + +// IsFull returns whether or not the cache is at capacity, +// as defined by the cache's max entries +func (c *Cache[K, V]) IsFull() bool { + return len(c.buf) == c.MaxEntries +} + +// checkAndInit ensures that the backing map and slices +// of the cache are not nil +func (c *Cache[K, V]) checkAndInit() { + if c.indices == nil { + c.indices = make(map[K]int, c.MaxEntries) + } + if c.buf == nil { + c.buf = make(buffer[K, V], 0, c.MaxEntries) + } + if c.touches == nil { + c.buf = make(buffer[K, V], c.MaxEntries) + } +} diff --git a/clock/clock_test.go b/clock/clock_test.go new file mode 100644 index 00000000..9a6909b5 --- /dev/null +++ b/clock/clock_test.go @@ -0,0 +1,46 @@ +//go:build go1.19 + +/* +Copyright 2024 Vimeo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clock + +import "testing" + +func TestCacheGet(t *testing.T) { + tests := []struct { + name string + keyToAdd string + keyToGet string + valueToAdd int + expectedOk bool + }{ + {"string_hit", "myKey", "myKey", 1234, true}, + {"string_miss", "myKey", "nonsense", 1234, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cache := New[string, int](5) + cache.Add(test.keyToAdd, test.valueToAdd) + value, ok := cache.Get(test.keyToGet) + if ok != test.expectedOk { + t.Fatalf("cache hit = %v; want %v", ok, test.expectedOk) + } else if ok && value != test.valueToAdd { + t.Fatalf("got %v; want %v", value, test.valueToAdd) + } + }) + } +} From 0907313e754ca24797fe3f1fb9da9c6a29a99892 Mon Sep 17 00:00:00 2001 From: Nick Brandt Date: Thu, 25 Jul 2024 15:40:50 -0400 Subject: [PATCH 2/2] Add another test, clean up add logic a bit --- clock/clock.go | 23 ++++++----- clock/clock_test.go | 93 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/clock/clock.go b/clock/clock.go index cb06d0b7..64c8d170 100644 --- a/clock/clock.go +++ b/clock/clock.go @@ -75,19 +75,18 @@ func (c *Cache[K, V]) Add(key K, val V) { value: val, }) c.indices[key] = len(c.buf) - 1 - c.clockhand = (c.clockhand + 1) % len(c.buf) - return - } - // Full, evict by reference bit then replace - hand := c.clockhand - for c.touches[hand].Load() > 0 { - c.touches[hand].Add(-1) - hand = (hand + 1) % len(c.buf) + } else { + // Full, evict by reference bit then replace + for c.touches[c.clockhand].Load() > 0 { + c.touches[c.clockhand].Add(-1) + c.clockhand += 1 % len(c.buf) + } + c.Evict(c.buf[c.clockhand].key) + c.buf.insertAt(c.clockhand, key, val) + c.indices[key] = c.clockhand + c.touches[c.clockhand].Add(1) } - c.Evict(c.buf[hand].key) - c.buf.insertAt(hand, key, val) - c.indices[key] = hand - c.clockhand = hand + c.clockhand = (c.clockhand + 1) % len(c.buf) } // Get returns the value for a given key, if present. diff --git a/clock/clock_test.go b/clock/clock_test.go index 9a6909b5..f3fc4f72 100644 --- a/clock/clock_test.go +++ b/clock/clock_test.go @@ -44,3 +44,96 @@ func TestCacheGet(t *testing.T) { }) } } + +func TestCacheAdd(t *testing.T) { + tests := []struct { + name string + size int + keysToAdd []string + valueToAdd []string + expectedCache buffer[string, string] + }{ + { + "beyond_capacity_evicts_first_untouched", + 3, + []string{"key-a", "key-b", "key-c", "key-d", "key-e"}, + []string{"val-a", "val-b", "val-c", "val-d", "val-e"}, + buffer[string, string]{ + &bufferItem[string, string]{key: "key-e", value: "val-e"}, + &bufferItem[string, string]{key: "key-b", value: "val-b"}, + &bufferItem[string, string]{key: "key-d", value: "val-d"}, + }, + }, + { + "multiple_touches_decreases_eviction_chances", + 3, + []string{"key-a", "key-b", "key-c", "key-a", "key-a", "key-d", "key-e"}, + []string{"val-a", "val-b", "val-c", "val-a", "val-a", "val-d", "val-e"}, + buffer[string, string]{ + &bufferItem[string, string]{key: "key-a", value: "val-a"}, + &bufferItem[string, string]{key: "key-e", value: "val-e"}, + &bufferItem[string, string]{key: "key-d", value: "val-d"}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cache := New[string, string](test.size) + for i := 0; i < len(test.keysToAdd); i++ { + key := test.keysToAdd[i] + val := test.valueToAdd[i] + cache.Add(key, val) + } + for i := 0; i < len(cache.buf); i++ { + cachedKey := cache.buf[i].key + cachedValue := cache.buf[i].value + expectedKey := test.expectedCache[i].key + expectedValue := test.expectedCache[i].value + if cachedKey != expectedKey { + t.Fatalf("bad cache key; got %s, wanted %s at index %d", cachedKey, expectedKey, i) + } + if cachedValue != expectedValue { + t.Fatalf("bad cache value; got %s, wanted %s at index %d", cachedValue, expectedValue, i) + } + } + }) + } +} + +func BenchmarkGetAllHits(b *testing.B) { + b.ReportAllocs() + type complexStruct struct { + a, b, c, d, e, f int64 + k, l, m, n, o, p float64 + } + // Populate the cache + l := New[int, complexStruct](32) + for z := 0; z < 32; z++ { + l.Add(z, complexStruct{a: int64(z)}) + } + + b.ResetTimer() + for z := 0; z < b.N; z++ { + // take the lower 5 bits as mod 32 so we always hit + l.Get(z & 31) + } +} + +func BenchmarkGetHalfHits(b *testing.B) { + b.ReportAllocs() + type complexStruct struct { + a, b, c, d, e, f int64 + k, l, m, n, o, p float64 + } + // Populate the cache + l := New[int, complexStruct](32) + for z := 0; z < 32; z++ { + l.Add(z, complexStruct{a: int64(z)}) + } + + b.ResetTimer() + for z := 0; z < b.N; z++ { + // take the lower 4 bits as mod 16 shifted left by 1 to + l.Get((z&15)<<1 | z&16>>4 | z&1<<4) + } +}