From 02b9f18dde461891c1b6ecdd50d9a8e791ad473b Mon Sep 17 00:00:00 2001 From: Nitin Garg <113666283+gargnitingoogle@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:12:15 +0530 Subject: [PATCH] Switch statCache to internal.cache.Lru (#1514) * statCache uses internal lru.Cache This change switches internal.cache.metadata.statCache to use gcsfuse-internal lru.Cache from jacobsa/*lrucache.Cache . This is a step towards unification of metadata cache with the readonly cache interface and design. Currently statCache takes in capacity (count) for initialization, but is now well-placed to switch to size (in Mbytes) . * minor cleanup * Remove CheckInvariants() from StatCache As new lru cache doesn't expose its checkInvariants method, StatCache also cannot implement a CheckInvariants() method. * rename invariantCache->testHelperCache This change is there because there this helper class no more does any CheckInvariants() call because it is not supported anymore on statCache. --- internal/cache/metadata/stat_cache.go | 21 +++++++------ internal/cache/metadata/stat_cache_test.go | 30 ++++++------------- internal/storage/caching/integration_test.go | 5 ++-- .../mock_gcscaching/mock_stat_cache.go | 17 ----------- 4 files changed, 21 insertions(+), 52 deletions(-) diff --git a/internal/cache/metadata/stat_cache.go b/internal/cache/metadata/stat_cache.go index 90f22b8a40..712577b9f0 100644 --- a/internal/cache/metadata/stat_cache.go +++ b/internal/cache/metadata/stat_cache.go @@ -17,8 +17,10 @@ package metadata import ( "time" + unsafe "unsafe" + + "github.com/googlecloudplatform/gcsfuse/internal/cache/lru" "github.com/googlecloudplatform/gcsfuse/internal/storage/gcs" - "github.com/jacobsa/util/lrucache" ) // A cache mapping from name to most recent known record for the object of that @@ -47,24 +49,21 @@ type StatCache interface { // entry. Return hit == false when there is neither a positive nor a negative // entry, or the entry has expired according to the supplied current time. LookUp(name string, now time.Time) (hit bool, o *gcs.Object) - - // Panic if any internal invariants have been violated. The careful user can - // arrange to call this at crucial moments. - CheckInvariants() } // Create a new stat cache that holds the given number of entries, which must // be positive. func NewStatCache(capacity int) (sc StatCache) { + sizeOfEntry := entry{}.Size() sc = &statCache{ - c: lrucache.New(capacity), + c: lru.NewCache(uint64(capacity) * sizeOfEntry), } return } type statCache struct { - c lrucache.Cache + c *lru.Cache } // An entry in the cache, pairing an object with the expiration time for the @@ -74,6 +73,10 @@ type entry struct { expiration time.Time } +func (e entry) Size() uint64 { + return uint64(unsafe.Sizeof(gcs.Object{}) + unsafe.Sizeof(entry{})) +} + // Should the supplied object for a new positive entry replace the given // existing entry? func shouldReplace(o *gcs.Object, existing entry) bool { @@ -149,7 +152,3 @@ func (sc *statCache) LookUp( return } - -func (sc *statCache) CheckInvariants() { - sc.c.CheckInvariants() -} diff --git a/internal/cache/metadata/stat_cache_test.go b/internal/cache/metadata/stat_cache_test.go index 9a73b9ba29..e16d465fe1 100644 --- a/internal/cache/metadata/stat_cache_test.go +++ b/internal/cache/metadata/stat_cache_test.go @@ -29,60 +29,48 @@ func TestStatCache(t *testing.T) { RunTests(t) } // Invariant-checking cache //////////////////////////////////////////////////////////////////////// -type invariantsCache struct { +type testHelperCache struct { wrapped metadata.StatCache } -func (c *invariantsCache) Insert( +func (c *testHelperCache) Insert( o *gcs.Object, expiration time.Time) { - c.wrapped.CheckInvariants() - defer c.wrapped.CheckInvariants() - c.wrapped.Insert(o, expiration) } -func (c *invariantsCache) AddNegativeEntry( +func (c *testHelperCache) AddNegativeEntry( name string, expiration time.Time) { - c.wrapped.CheckInvariants() - defer c.wrapped.CheckInvariants() - c.wrapped.AddNegativeEntry(name, expiration) } -func (c *invariantsCache) Erase(name string) { - c.wrapped.CheckInvariants() - defer c.wrapped.CheckInvariants() - +func (c *testHelperCache) Erase(name string) { c.wrapped.Erase(name) } -func (c *invariantsCache) LookUp( +func (c *testHelperCache) LookUp( name string, now time.Time) (hit bool, o *gcs.Object) { - c.wrapped.CheckInvariants() - defer c.wrapped.CheckInvariants() - hit, o = c.wrapped.LookUp(name, now) return } -func (c *invariantsCache) LookUpOrNil( +func (c *testHelperCache) LookUpOrNil( name string, now time.Time) (o *gcs.Object) { _, o = c.LookUp(name, now) return } -func (c *invariantsCache) Hit( +func (c *testHelperCache) Hit( name string, now time.Time) (hit bool) { hit, _ = c.LookUp(name, now) return } -func (c *invariantsCache) NegativeEntry( +func (c *testHelperCache) NegativeEntry( name string, now time.Time) (negative bool) { hit, o := c.LookUp(name, now) @@ -100,7 +88,7 @@ var someTime = time.Date(2015, 4, 5, 2, 15, 0, 0, time.Local) var expiration = someTime.Add(time.Second) type StatCacheTest struct { - cache invariantsCache + cache testHelperCache } func init() { RegisterTestSuite(&StatCacheTest{}) } diff --git a/internal/storage/caching/integration_test.go b/internal/storage/caching/integration_test.go index 03920cd2e8..9079fd5787 100644 --- a/internal/storage/caching/integration_test.go +++ b/internal/storage/caching/integration_test.go @@ -38,7 +38,6 @@ func TestIntegration(t *testing.T) { RunTests(t) } type IntegrationTest struct { ctx context.Context - cache metadata.StatCache clock timeutil.SimulatedClock wrapped gcs.Bucket @@ -55,12 +54,12 @@ func (t *IntegrationTest) SetUp(ti *TestInfo) { // Set up dependencies. const cacheCapacity = 100 - t.cache = metadata.NewStatCache(cacheCapacity) + cache := metadata.NewStatCache(cacheCapacity) t.wrapped = fake.NewFakeBucket(&t.clock, "some_bucket") t.bucket = caching.NewFastStatBucket( ttl, - t.cache, + cache, &t.clock, t.wrapped) } diff --git a/internal/storage/caching/mock_gcscaching/mock_stat_cache.go b/internal/storage/caching/mock_gcscaching/mock_stat_cache.go index 96188b7c75..08d846f042 100644 --- a/internal/storage/caching/mock_gcscaching/mock_stat_cache.go +++ b/internal/storage/caching/mock_gcscaching/mock_stat_cache.go @@ -61,23 +61,6 @@ func (m *mockStatCache) AddNegativeEntry(p0 string, p1 time.Time) { } } -func (m *mockStatCache) CheckInvariants() { - // Get a file name and line number for the caller. - _, file, line, _ := runtime.Caller(1) - - // Hand the call off to the controller, which does most of the work. - retVals := m.controller.HandleMethodCall( - m, - "CheckInvariants", - file, - line, - []interface{}{}) - - if len(retVals) != 0 { - panic(fmt.Sprintf("mockStatCache.CheckInvariants: invalid return values: %v", retVals)) - } -} - func (m *mockStatCache) Erase(p0 string) { // Get a file name and line number for the caller. _, file, line, _ := runtime.Caller(1)