Skip to content

Commit

Permalink
feat(keyvalue): add badger as keyvalue store
Browse files Browse the repository at this point in the history
KeyValue store will be used by Incidents, so that the functions
can decide based on history not just on the event that triggered them.
  • Loading branch information
mentos1386 committed Apr 28, 2024
1 parent e77764c commit bb1ba5e
Show file tree
Hide file tree
Showing 9 changed files with 142 additions and 17 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ node_modules/

# Database
zdravko.db*
zdravko_kv.db*
temporal.db*

# Keys
Expand Down
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module code.tjo.space/mentos1386/zdravko
go 1.21.6

require (
github.com/dgraph-io/badger/v4 v4.2.0
github.com/go-playground/validator/v10 v10.18.0
github.com/golang-jwt/jwt/v5 v5.2.0
github.com/gorilla/sessions v1.2.2
Expand All @@ -16,6 +17,7 @@ require (
github.com/spf13/viper v1.18.2
github.com/temporalio/ui-server/v2 v2.23.0
go.k6.io/k6 v0.49.0
go.temporal.io/api v1.27.0
go.temporal.io/sdk v1.26.0-rc.2
go.temporal.io/server v1.22.4
golang.org/x/exp v0.0.0-20231127185646-65229373498e
Expand Down Expand Up @@ -49,6 +51,7 @@ require (
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coreos/go-oidc/v3 v3.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.9.0 // indirect
Expand All @@ -71,10 +74,12 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.7.0-rc.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
Expand Down Expand Up @@ -165,7 +170,6 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.temporal.io/api v1.27.0 // indirect
go.temporal.io/version v0.3.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20140601200337-fc41e106ee0e/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
Expand All @@ -103,6 +108,7 @@ github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue7
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
Expand Down Expand Up @@ -172,6 +178,8 @@ github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand All @@ -198,6 +206,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -653,6 +663,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
12 changes: 7 additions & 5 deletions internal/config/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ import (
)

type ServerConfig struct {
Port string `validate:"required"`
RootUrl string `validate:"required,url"`
DatabasePath string `validate:"required"`
SessionSecret string `validate:"required"`
Port string `validate:"required"`
RootUrl string `validate:"required,url"`
SqliteDatabasePath string `validate:"required"`
KeyValueDatabasePath string `validate:"required"`
SessionSecret string `validate:"required"`

Jwt ServerJwt `validate:"required"`
OAuth2 ServerOAuth2 `validate:"required"`
Expand Down Expand Up @@ -47,7 +48,8 @@ func NewServerConfig() *ServerConfig {
// Set defaults
v.SetDefault("port", GetEnvOrDefault("PORT", "8000"))
v.SetDefault("rooturl", GetEnvOrDefault("ROOT_URL", "http://localhost:8000"))
v.SetDefault("databasepath", GetEnvOrDefault("DATABASE_PATH", "zdravko.db"))
v.SetDefault("sqlitedatabasepath", GetEnvOrDefault("SQLITE_DATABASE_PATH", "zdravko.db"))
v.SetDefault("keyvaluedatabasepath", GetEnvOrDefault("KEYVALUE_DATABASE_PATH", "zdravko_kv.db"))
v.SetDefault("sessionsecret", os.Getenv("SESSION_SECRET"))
v.SetDefault("temporal.uihost", GetEnvOrDefault("TEMPORAL_UI_HOST", "127.0.0.1:8223"))
v.SetDefault("temporal.serverhost", GetEnvOrDefault("TEMPORAL_SERVER_HOST", "127.0.0.1:7233"))
Expand Down
11 changes: 7 additions & 4 deletions internal/handlers/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"log/slog"

"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/web/templates/components"
"github.com/gorilla/sessions"
"github.com/jmoiron/sqlx"
Expand All @@ -26,20 +27,22 @@ func GetPageByTitle(pages []*components.Page, title string) *components.Page {
}

type BaseHandler struct {
db *sqlx.DB
config *config.ServerConfig
logger *slog.Logger
db *sqlx.DB
kvStore kv.KeyValueStore
config *config.ServerConfig
logger *slog.Logger

temporal client.Client

store *sessions.CookieStore
}

func NewBaseHandler(db *sqlx.DB, temporal client.Client, config *config.ServerConfig, logger *slog.Logger) *BaseHandler {
func NewBaseHandler(db *sqlx.DB, kvStore kv.KeyValueStore, temporal client.Client, config *config.ServerConfig, logger *slog.Logger) *BaseHandler {
store := sessions.NewCookieStore([]byte(config.SessionSecret))

return &BaseHandler{
db: db,
kvStore: kvStore,
config: config,
logger: logger,
temporal: temporal,
Expand Down
82 changes: 82 additions & 0 deletions internal/kv/badger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package kv

import (
"time"

badger "github.com/dgraph-io/badger/v4"
"github.com/pkg/errors"
)

type BadgerKeyValueStore struct {
db *badger.DB
}

func NewBadgerKeyValueStore(path string) (*BadgerKeyValueStore, error) {
db, err := badger.Open(badger.DefaultOptions(path))
if err != nil {
return nil, errors.Wrap(err, "failed to open badger db")
}
return &BadgerKeyValueStore{db: db}, nil
}

func (b *BadgerKeyValueStore) Close() error {
return b.db.Close()
}

func (b *BadgerKeyValueStore) Set(key string, value []byte, ttl time.Duration) error {
return b.db.Update(func(txn *badger.Txn) error {
e := badger.NewEntry([]byte(key), value).WithTTL(ttl)
return txn.SetEntry(e)
})
}

func (b *BadgerKeyValueStore) Increment(key string) (int, error) {
var value int
return value, b.db.Update(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))
if err != nil {
return err
}
valCopy, err := item.ValueCopy(nil)
if err != nil {
return err
}
value = int(valCopy[0]) + 1
return txn.Set([]byte(key), []byte{byte(value)})
})
}

func (b *BadgerKeyValueStore) Get(key string) ([]byte, error) {
var value []byte
return value, b.db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))
if err != nil {
return err
}
valCopy, err := item.ValueCopy(value)
if err != nil {
return err
}
value = valCopy
return nil
})
}

func (b *BadgerKeyValueStore) Delete(key string) error {
return b.db.Update(func(txn *badger.Txn) error {
return txn.Delete([]byte(key))
})
}

func (b *BadgerKeyValueStore) Keys(prefix string) ([]string, error) {
var keys []string
return keys, b.db.View(func(txn *badger.Txn) error {
itr := txn.NewIterator(badger.DefaultIteratorOptions)
defer itr.Close()
for itr.Seek([]byte(prefix)); itr.ValidForPrefix([]byte(prefix)); itr.Next() {
item := itr.Item()
keys = append(keys, string(item.Key()))
}
return nil
})
}
13 changes: 13 additions & 0 deletions internal/kv/kv.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package kv

import "time"

type KeyValueStore interface {
Close() error

Get(key string) ([]byte, error)
Set(key string, value []byte, ttl time.Duration) error
Increment(key string) (int, error)
Delete(key string) error
Keys(prefix string) ([]string, error)
}
8 changes: 5 additions & 3 deletions pkg/server/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/handlers"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/web/static"
"github.com/jmoiron/sqlx"
"github.com/labstack/echo/v4"
Expand All @@ -15,16 +16,17 @@ import (

func Routes(
e *echo.Echo,
db *sqlx.DB,
sqlDb *sqlx.DB,
kvStore kv.KeyValueStore,
temporalClient client.Client,
cfg *config.ServerConfig,
logger *slog.Logger,
) {
h := handlers.NewBaseHandler(db, temporalClient, cfg, logger)
h := handlers.NewBaseHandler(sqlDb, kvStore, temporalClient, cfg, logger)

// Health
e.GET("/health", func(c echo.Context) error {
err := db.Ping()
err := sqlDb.Ping()
if err != nil {
return err
}
Expand Down
15 changes: 11 additions & 4 deletions pkg/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import (

"code.tjo.space/mentos1386/zdravko/database"
"code.tjo.space/mentos1386/zdravko/internal/config"
"code.tjo.space/mentos1386/zdravko/internal/kv"
"code.tjo.space/mentos1386/zdravko/internal/temporal"
"code.tjo.space/mentos1386/zdravko/web/templates"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
"github.com/pkg/errors"
)

type Server struct {
Expand All @@ -33,14 +35,19 @@ func (s *Server) Name() string {
}

func (s *Server) Start() error {
db, err := database.ConnectToDatabase(s.logger, s.cfg.DatabasePath)
sqliteDb, err := database.ConnectToDatabase(s.logger, s.cfg.SqliteDatabasePath)
if err != nil {
return err
return errors.Wrap(err, "failed to connect to sqlite")
}

temporalClient, err := temporal.ConnectServerToTemporal(s.logger, s.cfg)
if err != nil {
return err
return errors.Wrap(err, "failed to connect to temporal")
}

kvStore, err := kv.NewBadgerKeyValueStore(s.cfg.KeyValueDatabasePath)
if err != nil {
return errors.Wrap(err, "failed to open kv store")
}

s.worker = NewWorker(temporalClient, s.cfg)
Expand All @@ -49,7 +56,7 @@ func (s *Server) Start() error {
s.echo.Use(middleware.Logger())
s.echo.Use(middleware.Recover())
s.echo.Use(middleware.Secure())
Routes(s.echo, db, temporalClient, s.cfg, s.logger)
Routes(s.echo, sqliteDb, kvStore, temporalClient, s.cfg, s.logger)

go func() {
if err := s.worker.Start(); err != nil {
Expand Down

0 comments on commit bb1ba5e

Please sign in to comment.