Skip to content

Commit

Permalink
sqlite database storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@reinkrul
reinkrul committed Nov 6, 2023
1 parent 5bf25d8 commit 6ddc667
Show file tree
Hide file tree
Showing 12 changed files with 176 additions and 24 deletions.
12 changes: 9 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/chromedp/chromedp v0.9.3
github.com/dlclark/regexp2 v1.10.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang-migrate/migrate/v4 v4.16.2
github.com/goodsign/monday v1.0.1
github.com/google/uuid v1.4.0
github.com/hashicorp/vault/api v1.10.0
Expand All @@ -33,6 +34,7 @@ require (
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_model v0.5.0
github.com/redis/go-redis/v9 v9.2.1
github.com/santhosh-tekuri/jsonschema v1.2.4
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
Expand All @@ -51,8 +53,6 @@ require (
schneider.vip/problem v1.8.1
)

require github.com/santhosh-tekuri/jsonschema v1.2.4

require (
github.com/PaesslerAG/gval v1.2.2 // indirect
github.com/alexandrevicenzi/go-sse v1.6.0 // indirect
Expand Down Expand Up @@ -90,7 +90,7 @@ require (
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.3.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
Expand All @@ -104,6 +104,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/gorm v1.9.16 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/kr/text v0.2.0 // indirect
Expand All @@ -117,6 +118,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
Expand Down Expand Up @@ -165,5 +167,9 @@ require (
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
gopkg.in/Regis24GmbH/go-diacritics.v2 v2.0.3 // indirect
gorm.io/driver/sqlite v1.5.4
gorm.io/gorm v1.25.5
rsc.io/qr v0.2.0 // indirect
)

require github.com/golang-migrate/migrate v3.5.4+incompatible
23 changes: 16 additions & 7 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,12 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA=
github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk=
github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA=
github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -218,8 +222,8 @@ github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUz
github.com/goodsign/monday v1.0.1 h1:yJogH0uQNn4blHjoC3ESbdV0P1OhDtGYdd6x0w7QZBo=
github.com/goodsign/monday v1.0.1/go.mod h1:r4T4breXpoFwspQNM+u2sLxJb2zyTaxVGqUfTBjWOu8=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -300,8 +304,9 @@ github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
Expand Down Expand Up @@ -612,8 +617,8 @@ go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
Expand Down Expand Up @@ -850,6 +855,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=
gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY=
Expand Down
11 changes: 10 additions & 1 deletion storage/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,18 @@ package storage
type Config struct {
BBolt BBoltConfig `koanf:"bbolt"`
Redis RedisConfig `koanf:"redis"`
SQL SQLConfig `koanf:"sql"`
}

// DefaultConfig returns the default configuration for the module.
func DefaultConfig() Config {
return Config{}
return Config{
SQL: SQLConfig{
ConnectionString: "file::memory:?cache=shared",
},
}
}

type SQLConfig struct {
ConnectionString string `koanf:"connection"`
}
72 changes: 68 additions & 4 deletions storage/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,29 @@ package storage

import (
"context"
"embed"
"errors"
"fmt"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/sqlite3"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/nuts-foundation/go-stoabs"
"github.com/nuts-foundation/nuts-node/core"
"github.com/nuts-foundation/nuts-node/storage/log"
"github.com/redis/go-redis/v9"
"github.com/sirupsen/logrus"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"strings"
"sync"
"time"
)

const storeShutdownTimeout = 5 * time.Second

//go:embed sql_migrations/*.sql
var sqlMigrationsFS embed.FS

// New creates a new instance of the storage engine.
func New() Engine {
return &engine{
Expand All @@ -48,6 +58,7 @@ type engine struct {
stores map[string]stoabs.Store
databases []database
sessionDatabase SessionDatabase
sqlDB *gorm.DB
config Config
}

Expand All @@ -61,13 +72,18 @@ func (e engine) Name() string {
}

func (e engine) Start() error {
err := e.migrateSQL()
if err != nil {
return fmt.Errorf("failed to migrate SQL database: %w", err)
}
return nil
}

func (e engine) Shutdown() error {
e.storesMux.Lock()
defer e.storesMux.Unlock()

// Close KV stores
shutdown := func(store stoabs.Store) error {
// Refactored to separate function, otherwise defer would be in for loop which leaks resources.
ctx, cancel := context.WithTimeout(context.Background(), storeShutdownTimeout)
Expand All @@ -91,9 +107,14 @@ func (e engine) Shutdown() error {
return errors.New("one or more stores failed to close")
}

// Close session database
e.sessionDatabase.close()

return nil
// Close SQL db
underlyingDB, err := e.sqlDB.DB()
if err != nil {
return err
}
return underlyingDB.Close()
}

func (e *engine) Configure(config core.ServerConfig) error {
Expand All @@ -114,8 +135,7 @@ func (e *engine) Configure(config core.ServerConfig) error {
return fmt.Errorf("unable to configure BBolt database: %w", err)
}
e.databases = append(e.databases, bboltDB)

return nil
return e.initSQLDatabase()
}

func (e *engine) GetProvider(moduleName string) Provider {
Expand All @@ -129,6 +149,50 @@ func (e *engine) GetSessionDatabase() SessionDatabase {
return e.sessionDatabase
}

func (e *engine) SQLDatabase() *gorm.DB {
return e.sqlDB
}

func (e *engine) initSQLDatabase() error {
// Note: only SQLite is supported for now
var err error
e.sqlDB, err = gorm.Open(sqlite.Open(e.config.SQL.ConnectionString), &gorm.Config{})
return err
}

func (e *engine) migrateSQL() error {
log.Logger().Debug("Running database migrations...")
underlyingDB, err := e.sqlDB.DB()
if err != nil {
return err
}
sourceDriver, err := iofs.New(sqlMigrationsFS, "sql_migrations")
if err != nil {
return err
}
databaseDriver, err := sqlite3.WithInstance(underlyingDB, &sqlite3.Config{})
if err != nil {
return err
}
migrations, err := migrate.NewWithInstance("iofs", sourceDriver, e.sqlDB.Name(), databaseDriver)
migrations.Log = migrationLogger{}
if err != nil {
return err
}
return migrations.Up()
}

type migrationLogger struct {
}

func (m migrationLogger) Printf(format string, v ...interface{}) {
log.Logger().Infof(format, v...)
}

func (m migrationLogger) Verbose() bool {
return log.Logger().Level >= logrus.DebugLevel
}

type provider struct {
moduleName string
engine *engine
Expand Down
33 changes: 33 additions & 0 deletions storage/engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"errors"
"github.com/nuts-foundation/go-stoabs"
"github.com/nuts-foundation/nuts-node/core"
"github.com/nuts-foundation/nuts-node/storage/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
Expand Down Expand Up @@ -96,3 +97,35 @@ func Test_engine_Shutdown(t *testing.T) {
assert.EqualError(t, err, "one or more stores failed to close")
})
}

func Test_engine_sqlDatabase(t *testing.T) {
t.Run("test migrations", func(t *testing.T) {
// override default migrations with test migrations
old := sqlMigrationsFS
t.Cleanup(func() {
sqlMigrationsFS = old
})
sqlMigrationsFS = test.SQLMigrations
e := New()
require.NoError(t, e.Configure(*core.NewServerConfig()))

// Start() runs migrations
require.NoError(t, e.Start())
// Verify the test table can be queried
underlyingDB, err := e.SQLDatabase().DB()
require.NoError(t, err)
row := underlyingDB.QueryRow("SELECT count(*) FROM testtable")
require.NoError(t, row.Err())
var count int
assert.NoError(t, row.Scan(&count))
assert.Equal(t, 1, count)

require.NoError(t, e.Shutdown())
})
t.Run("actual migrations", func(t *testing.T) {
e := New()
require.NoError(t, e.Configure(*core.NewServerConfig()))
require.NoError(t, e.Start())
require.NoError(t, e.Shutdown())
})
}
3 changes: 3 additions & 0 deletions storage/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"errors"
"github.com/nuts-foundation/go-stoabs"
"github.com/nuts-foundation/nuts-node/core"
"gorm.io/gorm"
"time"
)

Expand All @@ -37,6 +38,8 @@ type Engine interface {
GetProvider(moduleName string) Provider
// GetSessionDatabase returns the SessionDatabase
GetSessionDatabase() SessionDatabase

SQLDatabase() *gorm.DB
}

// Provider lets callers get access to stores.
Expand Down
Loading

0 comments on commit 6ddc667

Please sign in to comment.