Skip to content

Commit

Permalink
feat: add cacher for gorm to reduce the load of db
Browse files Browse the repository at this point in the history
Signed-off-by: Gaius <[email protected]>
  • Loading branch information
gaius-qi committed Dec 30, 2024
1 parent 4d7bdc5 commit 828aff5
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 6 deletions.
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/gin-contrib/zap v1.1.3
github.com/gin-gonic/gin v1.10.0
github.com/go-echarts/statsview v0.3.4
github.com/go-gorm/caches/v4 v4.0.5
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
github.com/go-playground/validator/v10 v10.23.0
github.com/go-redis/cache/v9 v9.0.0
Expand All @@ -42,6 +43,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hashicorp/memberlist v0.5.1
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible
github.com/jarcoal/httpmock v1.3.1
Expand Down Expand Up @@ -94,7 +96,7 @@ require (
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/mysql v1.4.7
gorm.io/driver/postgres v1.4.8
gorm.io/gorm v1.24.6
gorm.io/gorm v1.25.0
gorm.io/plugin/soft_delete v1.2.1
k8s.io/component-base v0.31.2
moul.io/zapgorm2 v1.3.0
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,8 @@ github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gorm/caches/v4 v4.0.5 h1:Sdj9vxbEM0sCmv5+s5o6GzoVMuraWF0bjJJvUU+7c1U=
github.com/go-gorm/caches/v4 v4.0.5/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
Expand Down Expand Up @@ -861,6 +863,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
Expand Down Expand Up @@ -2272,8 +2276,8 @@ gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.24.6 h1:wy98aq9oFEetsc4CAbKD2SoBCdMzsbSIvSUUFJuHi5s=
gorm.io/gorm v1.24.6/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/plugin/dbresolver v1.1.0/go.mod h1:tpImigFAEejCALOttyhWqsy4vfa2Uh/vAUVnL5IRF7Y=
gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M=
gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk=
Expand Down
6 changes: 3 additions & 3 deletions manager/config/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ const (
DefaultLFUCacheTTL = 3 * time.Minute

// DefaultLFUCacheSize is default size for lfu cache.
DefaultLFUCacheSize = 50 * 1000
DefaultLFUCacheSize = 30 * 1000
)

const (
Expand All @@ -88,10 +88,10 @@ const (

const (
// DefaultJobGCInterval is the default interval for gc job.
DefaultJobGCInterval = 6 * time.Hour
DefaultJobGCInterval = 3 * time.Hour

// DefaultJobGCTTL is the default ttl for job.
DefaultJobGCTTL = 12 * time.Hour
DefaultJobGCTTL = 6 * time.Hour

// DefaultJobGCBatchSize is the default batch size for operating on the database in gc job.
DefaultJobGCBatchSize = 5000
Expand Down
82 changes: 82 additions & 0 deletions manager/database/cacher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* Copyright 2024 The Dragonfly Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package database

import (
"context"

caches "github.com/go-gorm/caches/v4"
lru "github.com/hashicorp/golang-lru/v2"
)

// defaultCacheSize is the default size of the cache.
const defaultCacheSize = 1024

// cacher is a cache implementation using LRU for gorm.
type cacher struct {
// store is the LRU cache.
store *lru.Cache[string, any]
}

// newCacher creates a new cacher.
func newCacher() (caches.Cacher, error) {
store, err := lru.New[string, any](defaultCacheSize)
if err != nil {
return nil, err
}

return &cacher{store}, nil
}

// Get impl should check if a specific key exists in the cache and return its value
// look at Query.Marshal.
func (c *cacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
val, ok := c.store.Get(key)
if !ok {
return nil, nil
}

if err := q.Unmarshal(val.([]byte)); err != nil {
return nil, err
}

return q, nil
}

// Store impl should store a cached representation of the val param
// look at Query.Unmarshal.
func (c *cacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
res, err := val.Marshal()
if err != nil {
return err
}

c.store.Add(key, res)
return nil
}

// Invalidate impl should invalidate all cached values. It will be called when
// INSERT / UPDATE / DELETE queries are sent to the DB.
func (c *cacher) Invalidate(ctx context.Context) error {
var err error
c.store, err = lru.New[string, any](defaultCacheSize)
if err != nil {
return err
}

return nil
}
16 changes: 16 additions & 0 deletions manager/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"errors"
"fmt"

caches "github.com/go-gorm/caches/v4"
"github.com/redis/go-redis/v9"
"gorm.io/gorm"

Expand Down Expand Up @@ -63,6 +64,21 @@ func New(cfg *config.Config) (*Database, error) {
return nil, fmt.Errorf("invalid database type %s", cfg.Database.Type)
}

// Add cache for reducing database load.
cacher, err := newCacher()
if err != nil {
logger.Errorf("cacher: %s", err.Error())
return nil, err
}

if err := db.Use(&caches.Caches{Conf: &caches.Config{
Easer: true,
Cacher: cacher,
}}); err != nil {
logger.Errorf("use cache: %s", err.Error())
return nil, err
}

rdb, err := pkgredis.NewRedis(&redis.UniversalOptions{
Addrs: cfg.Database.Redis.Addrs,
MasterName: cfg.Database.Redis.MasterName,
Expand Down

0 comments on commit 828aff5

Please sign in to comment.