From dd4f6c010713541b221be6e3649ead9a4bcc05e4 Mon Sep 17 00:00:00 2001 From: Yury Gargay Date: Thu, 12 Oct 2023 15:42:36 +0200 Subject: [PATCH] Implement SQLite Store using gorm and relational approach (#1065) Restructure data handling for improved performance and flexibility. Introduce 'G'-prefixed fields to represent Gorm relations, simplifying resource management. Eliminate complexity in lookup tables for enhanced query and write speed. Enable independent operations on data structures, requiring adjustments in the Store interface and Account Manager. --- .github/workflows/golang-test-darwin.yml | 5 +- .github/workflows/golang-test-linux.yml | 18 +- .github/workflows/golang-test-windows.yml | 5 + .gitignore | 3 +- client/cmd/testutil.go | 2 +- client/internal/engine_test.go | 5 +- dns/nameserver.go | 10 +- go.mod | 6 +- go.sum | 12 +- management/client/client_test.go | 2 +- management/cmd/management.go | 2 +- management/cmd/migration_down.go | 66 +++ management/cmd/migration_up.go | 66 +++ management/cmd/root.go | 14 + management/server/account.go | 35 +- management/server/account_test.go | 21 +- management/server/config.go | 2 + management/server/dns.go | 2 +- management/server/dns_test.go | 2 +- management/server/file_store.go | 24 + management/server/file_store_test.go | 2 +- management/server/group.go | 5 +- management/server/group_test.go | 5 + management/server/management_proto_test.go | 2 +- management/server/management_test.go | 3 +- management/server/metrics/selfhosted.go | 2 + management/server/metrics/selfhosted_test.go | 9 + management/server/nameserver_test.go | 2 +- management/server/network.go | 24 +- management/server/peer.go | 13 +- management/server/peer_test.go | 4 +- management/server/personal_access_token.go | 4 +- management/server/policy.go | 21 +- management/server/route_test.go | 2 +- management/server/rule.go | 7 +- management/server/setupkey.go | 9 +- management/server/sqlite_store.go | 457 +++++++++++++++++++ management/server/sqlite_store_test.go | 229 ++++++++++ management/server/store.go | 67 ++- management/server/store_test.go | 88 ++++ management/server/testdata/store.json | 63 ++- management/server/user.go | 10 +- management/server/user_test.go | 6 + route/route.go | 10 +- 44 files changed, 1239 insertions(+), 107 deletions(-) create mode 100644 management/cmd/migration_down.go create mode 100644 management/cmd/migration_up.go create mode 100644 management/server/sqlite_store.go create mode 100644 management/server/sqlite_store_test.go create mode 100644 management/server/store_test.go diff --git a/.github/workflows/golang-test-darwin.yml b/.github/workflows/golang-test-darwin.yml index 97fdeabe8e6..8cd28bcb4c6 100644 --- a/.github/workflows/golang-test-darwin.yml +++ b/.github/workflows/golang-test-darwin.yml @@ -12,6 +12,9 @@ concurrency: jobs: test: + strategy: + matrix: + store: ['JsonFile', 'Sqlite'] runs-on: macos-latest steps: - name: Install Go @@ -33,4 +36,4 @@ jobs: run: go mod tidy - name: Test - run: go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./... + run: NETBIRD_STORE_KIND=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./... diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 13061f6eb0e..026779885f9 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -15,6 +15,7 @@ jobs: strategy: matrix: arch: ['386','amd64'] + store: ['JsonFile', 'Sqlite'] runs-on: ubuntu-latest steps: - name: Install Go @@ -41,17 +42,16 @@ jobs: run: go mod tidy - name: Test - run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./... + run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_KIND=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./... test_client_on_docker: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Install Go uses: actions/setup-go@v4 with: go-version: "1.20.x" - - name: Cache Go modules uses: actions/cache@v3 with: @@ -64,7 +64,7 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev + run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib - name: Install modules run: go mod tidy @@ -82,7 +82,7 @@ jobs: run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/... - name: Generate Engine Test bin - run: CGO_ENABLED=0 go test -c -o engine-testing.bin ./client/internal + run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal - name: Generate Peer Test bin run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/... @@ -95,15 +95,17 @@ jobs: - name: Run Iface tests in docker run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1 - - name: Run RouteManager tests in docker run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin -test.timeout 5m -test.parallel 1 - name: Run nftables Manager tests in docker run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/firewall --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/nftablesmanager-testing.bin -test.timeout 5m -test.parallel 1 - - name: Run Engine tests in docker - run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin -test.timeout 5m -test.parallel 1 + - name: Run Engine tests in docker with file store + run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_KIND="JsonFile" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin -test.timeout 5m -test.parallel 1 + + - name: Run Engine tests in docker with sqlite store + run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_KIND="Sqlite" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin -test.timeout 5m -test.parallel 1 - name: Run Peer tests in docker run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/peer-testing.bin -test.timeout 5m -test.parallel 1 \ No newline at end of file diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index 6dd91666c11..1fc84ff2acb 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -14,6 +14,9 @@ concurrency: jobs: test: + strategy: + matrix: + store: ['JsonFile', 'Sqlite'] runs-on: windows-latest steps: - name: Checkout code @@ -40,6 +43,8 @@ jobs: - run: mv ${{ env.downloadPath }}/wintun/bin/amd64/wintun.dll 'C:\Windows\System32\' - run: choco install -y sysinternals + - run: choco install -y mingw + - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=C:\Users\runneradmin\go\pkg\mod - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=C:\Users\runneradmin\AppData\Local\go-build diff --git a/.gitignore b/.gitignore index dc62780ad6d..7edcc708716 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ client/.distfiles/ infrastructure_files/setup.env infrastructure_files/setup-*.env .vscode -.DS_Store \ No newline at end of file +.DS_Store +*.db diff --git a/client/cmd/testutil.go b/client/cmd/testutil.go index 6d47021dd09..47ae9ddb466 100644 --- a/client/cmd/testutil.go +++ b/client/cmd/testutil.go @@ -65,7 +65,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste t.Fatal(err) } s := grpc.NewServer() - store, err := mgmt.NewFileStore(config.Datadir, nil) + store, err := mgmt.NewStoreFromJson(config.Datadir, nil) if err != nil { t.Fatal(err) } diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index ea4a23a8de6..42012bd0a10 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -1039,10 +1039,11 @@ func startManagement(dataDir string) (*grpc.Server, string, error) { return nil, "", err } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - store, err := server.NewFileStore(config.Datadir, nil) + store, err := server.NewStoreFromJson(config.Datadir, nil) if err != nil { - log.Fatalf("failed creating a store: %s: %v", config.Datadir, err) + return nil, "", err } + peersUpdateManager := server.NewPeersUpdateManager() eventStore := &activity.InMemoryEventStore{} if err != nil { diff --git a/dns/nameserver.go b/dns/nameserver.go index 7751f8e1c6d..f3ae2569d39 100644 --- a/dns/nameserver.go +++ b/dns/nameserver.go @@ -50,19 +50,21 @@ func ToNameServerType(typeString string) NameServerType { // NameServerGroup group of nameservers and with group ids type NameServerGroup struct { // ID identifier of group - ID string + ID string `gorm:"primaryKey"` + // AccountID is a reference to Account that this object belongs + AccountID string `gorm:"index"` // Name group name Name string // Description group description Description string // NameServers list of nameservers - NameServers []NameServer + NameServers []NameServer `gorm:"serializer:json"` // Groups list of peer group IDs to distribute the nameservers information - Groups []string + Groups []string `gorm:"serializer:json"` // Primary indicates that the nameserver group is the primary resolver for any dns query Primary bool // Domains indicate the dns query domains to use with this nameserver group - Domains []string + Domains []string `gorm:"serializer:json"` // Enabled group status Enabled bool } diff --git a/go.mod b/go.mod index 8be1599970a..1f8eec24ed3 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/libp2p/go-netroute v0.2.0 github.com/magiconair/properties v1.8.5 - github.com/mattn/go-sqlite3 v1.14.16 + github.com/mattn/go-sqlite3 v1.14.17 github.com/mdlayher/socket v0.4.0 github.com/miekg/dns v1.1.43 github.com/mitchellh/hashstructure/v2 v2.0.2 @@ -74,6 +74,8 @@ require ( golang.org/x/term v0.8.0 google.golang.org/api v0.126.0 gopkg.in/yaml.v3 v3.0.1 + gorm.io/driver/sqlite v1.5.3 + gorm.io/gorm v1.25.4 ) require ( @@ -110,6 +112,8 @@ require ( github.com/googleapis/gax-go/v2 v2.10.0 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/native v1.0.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect diff --git a/go.sum b/go.sum index 25182ca85df..15e69283c2d 100644 --- a/go.sum +++ b/go.sum @@ -383,6 +383,10 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josephspurrier/goversioninfo v0.0.0-20200309025242-14b0ab84c6ca/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= @@ -441,8 +445,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1189,6 +1193,10 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.5.3 h1:7/0dUgX28KAcopdfbRWWl68Rflh6osa4rDh+m51KL2g= +gorm.io/driver/sqlite v1.5.3/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw= +gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= gvisor.dev/gvisor v0.0.0-20221203005347-703fd9b7fbc0 h1:Wobr37noukisGxpKo5jAsLREcpj61RxrWYzD8uwveOY= diff --git a/management/client/client_test.go b/management/client/client_test.go index 86c598adbd9..b66dacc7382 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -53,7 +53,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) { t.Fatal(err) } s := grpc.NewServer() - store, err := mgmt.NewFileStore(config.Datadir, nil) + store, err := mgmt.NewStoreFromJson(config.Datadir, nil) if err != nil { t.Fatal(err) } diff --git a/management/cmd/management.go b/management/cmd/management.go index f85cf225eaf..fda16566c15 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -126,7 +126,7 @@ var ( if err != nil { return err } - store, err := server.NewFileStore(config.Datadir, appMetrics) + store, err := server.NewStore(config.StoreKind, config.Datadir, appMetrics) if err != nil { return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err) } diff --git a/management/cmd/migration_down.go b/management/cmd/migration_down.go new file mode 100644 index 00000000000..6d136ec1acf --- /dev/null +++ b/management/cmd/migration_down.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "errors" + "flag" + "fmt" + "os" + "path" + + "github.com/netbirdio/netbird/management/server" + "github.com/netbirdio/netbird/util" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var shortDown = "Rollback SQLite store to JSON file store. Please make a backup of the SQLite file before running this command." + +var downCmd = &cobra.Command{ + Use: "downgrade [--datadir directory] [--log-file console]", + Aliases: []string{"down"}, + Short: shortDown, + Long: shortDown + + "\n\n" + + "This command reads the content of {datadir}/store.db and migrates it to {datadir}/store.json that can be used by File store driver.", + RunE: func(cmd *cobra.Command, args []string) error { + flag.Parse() + err := util.InitLog(logLevel, logFile) + if err != nil { + return fmt.Errorf("failed initializing log %v", err) + } + + sqliteStorePath := path.Join(mgmtDataDir, "store.db") + if _, err := os.Stat(sqliteStorePath); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s doesn't exist, couldn't continue the operation", sqliteStorePath) + } + + fileStorePath := path.Join(mgmtDataDir, "store.json") + if _, err := os.Stat(fileStorePath); err == nil { + return fmt.Errorf("%s already exists, couldn't continue the operation", fileStorePath) + } + + sqlstore, err := server.NewSqliteStore(mgmtDataDir, nil) + if err != nil { + return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err) + } + + sqliteStoreAccounts := len(sqlstore.GetAllAccounts()) + log.Infof("%d account will be migrated from sqlite store %s to file store %s", + sqliteStoreAccounts, sqliteStorePath, fileStorePath) + + store, err := server.NewFilestoreFromSqliteStore(sqlstore, mgmtDataDir, nil) + if err != nil { + return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err) + } + + fsStoreAccounts := len(store.GetAllAccounts()) + if fsStoreAccounts != sqliteStoreAccounts { + return fmt.Errorf("failed to migrate accounts from sqlite to file[]. Expected accounts: %d, got: %d", + sqliteStoreAccounts, fsStoreAccounts) + } + + log.Info("Migration finished successfully") + + return nil + }, +} diff --git a/management/cmd/migration_up.go b/management/cmd/migration_up.go new file mode 100644 index 00000000000..5c7505cfcea --- /dev/null +++ b/management/cmd/migration_up.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "errors" + "flag" + "fmt" + "os" + "path" + + "github.com/netbirdio/netbird/management/server" + "github.com/netbirdio/netbird/util" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var shortUp = "Migrate JSON file store to SQLite store. Please make a backup of the JSON file before running this command." + +var upCmd = &cobra.Command{ + Use: "upgrade [--datadir directory] [--log-file console]", + Aliases: []string{"up"}, + Short: shortUp, + Long: shortUp + + "\n\n" + + "This command reads the content of {datadir}/store.json and migrates it to {datadir}/store.db that can be used by SQLite store driver.", + RunE: func(cmd *cobra.Command, args []string) error { + flag.Parse() + err := util.InitLog(logLevel, logFile) + if err != nil { + return fmt.Errorf("failed initializing log %v", err) + } + + fileStorePath := path.Join(mgmtDataDir, "store.json") + if _, err := os.Stat(fileStorePath); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s doesn't exist, couldn't continue the operation", fileStorePath) + } + + sqlStorePath := path.Join(mgmtDataDir, "store.db") + if _, err := os.Stat(sqlStorePath); err == nil { + return fmt.Errorf("%s already exists, couldn't continue the operation", sqlStorePath) + } + + fstore, err := server.NewFileStore(mgmtDataDir, nil) + if err != nil { + return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err) + } + + fsStoreAccounts := len(fstore.GetAllAccounts()) + log.Infof("%d account will be migrated from file store %s to sqlite store %s", + fsStoreAccounts, fileStorePath, sqlStorePath) + + store, err := server.NewSqliteStoreFromFileStore(fstore, mgmtDataDir, nil) + if err != nil { + return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err) + } + + sqliteStoreAccounts := len(store.GetAllAccounts()) + if fsStoreAccounts != sqliteStoreAccounts { + return fmt.Errorf("failed to migrate accounts from file to sqlite. Expected accounts: %d, got: %d", + fsStoreAccounts, sqliteStoreAccounts) + } + + log.Info("Migration finished successfully") + + return nil + }, +} diff --git a/management/cmd/root.go b/management/cmd/root.go index 2080a6b29f2..d8a9da53f03 100644 --- a/management/cmd/root.go +++ b/management/cmd/root.go @@ -34,6 +34,12 @@ var ( SilenceUsage: true, } + migrationCmd = &cobra.Command{ + Use: "sqlite-migration", + Short: "Contains sub-commands to perform JSON file store to SQLite store migration and rollback", + Long: "", + SilenceUsage: true, + } // Execution control channel for stopCh signal stopCh chan int ) @@ -63,6 +69,14 @@ func init() { rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "") rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the the log will be output to stdout") rootCmd.AddCommand(mgmtCmd) + + migrationCmd.PersistentFlags().StringVar(&mgmtDataDir, "datadir", defaultMgmtDataDir, "server data directory location") + migrationCmd.MarkFlagRequired("datadir") //nolint + + migrationCmd.AddCommand(upCmd) + migrationCmd.AddCommand(downCmd) + + rootCmd.AddCommand(migrationCmd) } // SetupCloseHandler handles SIGTERM signal and exits with success diff --git a/management/server/account.go b/management/server/account.go index 0e583e17f3f..f78530b4440 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -165,24 +165,33 @@ func (s *Settings) Copy() *Settings { // Account represents a unique account of the system type Account struct { - Id string + // we have to name column to aid as it collides with Network.Id when work with associations + Id string `gorm:"primaryKey"` + // User.Id it was created by CreatedBy string - Domain string + Domain string `gorm:"index"` DomainCategory string IsDomainPrimaryAccount bool - SetupKeys map[string]*SetupKey - Network *Network - Peers map[string]*Peer - Users map[string]*User - Groups map[string]*Group - Rules map[string]*Rule - Policies []*Policy - Routes map[string]*route.Route - NameServerGroups map[string]*nbdns.NameServerGroup - DNSSettings DNSSettings + SetupKeys map[string]*SetupKey `gorm:"-"` + SetupKeysG []SetupKey `json:"-" gorm:"foreignKey:AccountID;references:id"` + Network *Network `gorm:"embedded;embeddedPrefix:network_"` + Peers map[string]*Peer `gorm:"-"` + PeersG []Peer `json:"-" gorm:"foreignKey:AccountID;references:id"` + Users map[string]*User `gorm:"-"` + UsersG []User `json:"-" gorm:"foreignKey:AccountID;references:id"` + Groups map[string]*Group `gorm:"-"` + GroupsG []Group `json:"-" gorm:"foreignKey:AccountID;references:id"` + Rules map[string]*Rule `gorm:"-"` + RulesG []Rule `json:"-" gorm:"foreignKey:AccountID;references:id"` + Policies []*Policy `gorm:"foreignKey:AccountID;references:id"` + Routes map[string]*route.Route `gorm:"-"` + RoutesG []route.Route `json:"-" gorm:"foreignKey:AccountID;references:id"` + NameServerGroups map[string]*nbdns.NameServerGroup `gorm:"-"` + NameServerGroupsG []nbdns.NameServerGroup `json:"-" gorm:"foreignKey:AccountID;references:id"` + DNSSettings DNSSettings `gorm:"embedded;embeddedPrefix:dns_settings_"` // Settings is a dictionary of Account settings - Settings *Settings + Settings *Settings `gorm:"embedded;embeddedPrefix:settings_"` } type UserInfo struct { diff --git a/management/server/account_test.go b/management/server/account_test.go index e47b3b854d6..181e1c3feaf 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -198,11 +198,11 @@ func TestAccount_GetPeerNetworkMap(t *testing.T) { netIP := net.IP{100, 64, 0, 0} netMask := net.IPMask{255, 255, 0, 0} network := &Network{ - Id: "network", - Net: net.IPNet{IP: netIP, Mask: netMask}, - Dns: "netbird.selfhosted", - Serial: 0, - mu: sync.Mutex{}, + Identifier: "network", + Net: net.IPNet{IP: netIP, Mask: netMask}, + Dns: "netbird.selfhosted", + Serial: 0, + mu: sync.Mutex{}, } for _, testCase := range tt { @@ -476,7 +476,7 @@ func TestDefaultAccountManager_GetGroupsFromTheToken(t *testing.T) { // as initAccount was created without account id we have to take the id after account initialization // that happens inside the GetAccountByUserOrAccountID where the id is getting generated // it is important to set the id as it help to avoid creating additional account with empty Id and re-pointing indices to it - initAccount.Id = acc.Id + initAccount = acc claims := jwtclaims.AuthorizationClaims{ AccountId: accountID, // is empty as it is based on accountID right after initialization of initAccount @@ -1025,7 +1025,6 @@ func TestAccountManager_NetworkUpdates(t *testing.T) { wg.Wait() }) - t.Run("delete peer update", func(t *testing.T) { wg.Add(1) go func() { @@ -1309,7 +1308,7 @@ func TestAccount_Copy(t *testing.T) { }, }, Network: &Network{ - Id: "net1", + Identifier: "net1", }, Peers: map[string]*Peer{ "peer1": { @@ -1400,6 +1399,10 @@ func hasNilField(x interface{}) error { rv := reflect.ValueOf(x) rv = rv.Elem() for i := 0; i < rv.NumField(); i++ { + // skip gorm internal fields + if json, ok := rv.Type().Field(i).Tag.Lookup("json"); ok && json == "-" { + continue + } if f := rv.Field(i); f.IsValid() { k := f.Kind() switch k { @@ -2045,7 +2048,7 @@ func createManager(t *testing.T) (*DefaultAccountManager, error) { func createStore(t *testing.T) (Store, error) { dataDir := t.TempDir() - store, err := NewFileStore(dataDir, nil) + store, err := NewStoreFromJson(dataDir, nil) if err != nil { return nil, err } diff --git a/management/server/config.go b/management/server/config.go index 31c1cf45c5d..19a71ff7aae 100644 --- a/management/server/config.go +++ b/management/server/config.go @@ -45,6 +45,8 @@ type Config struct { DeviceAuthorizationFlow *DeviceAuthorizationFlow PKCEAuthorizationFlow *PKCEAuthorizationFlow + + StoreKind StoreKind } // GetAuthAudiences returns the audience from the http config and device authorization flow config diff --git a/management/server/dns.go b/management/server/dns.go index 9707cc372b6..7b25e230f49 100644 --- a/management/server/dns.go +++ b/management/server/dns.go @@ -20,7 +20,7 @@ type lookupMap map[string]struct{} // DNSSettings defines dns settings at the account level type DNSSettings struct { // DisabledManagementGroups groups whose DNS management is disabled - DisabledManagementGroups []string + DisabledManagementGroups []string `gorm:"serializer:json"` } // Copy returns a copy of the DNS settings diff --git a/management/server/dns_test.go b/management/server/dns_test.go index 8c979c2a6ac..a2c9d3aa2f7 100644 --- a/management/server/dns_test.go +++ b/management/server/dns_test.go @@ -196,7 +196,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) { func createDNSStore(t *testing.T) (Store, error) { dataDir := t.TempDir() - store, err := NewFileStore(dataDir, nil) + store, err := NewStoreFromJson(dataDir, nil) if err != nil { return nil, err } diff --git a/management/server/file_store.go b/management/server/file_store.go index b90b1d607eb..c8d24433fa1 100644 --- a/management/server/file_store.go +++ b/management/server/file_store.go @@ -54,6 +54,25 @@ func NewFileStore(dataDir string, metrics telemetry.AppMetrics) (*FileStore, err return fs, nil } +// NewFilestoreFromSqliteStore restores a store from Sqlite and stores to Filestore json in the file located in datadir +func NewFilestoreFromSqliteStore(sqlitestore *SqliteStore, dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) { + store, err := NewFileStore(dataDir, metrics) + if err != nil { + return nil, err + } + + err = store.SaveInstallationID(sqlitestore.GetInstallationID()) + if err != nil { + return nil, err + } + + for _, account := range sqlitestore.GetAllAccounts() { + store.Accounts[account.Id] = account + } + + return store, store.persist(store.storeFile) +} + // restore the state of the store from the file. // Creates a new empty store file if doesn't exist func restore(file string) (*FileStore, error) { @@ -595,3 +614,8 @@ func (s *FileStore) Close() error { return s.persist(s.storeFile) } + +// GetStoreKind returns FileStoreKind +func (s *FileStore) GetStoreKind() StoreKind { + return FileStoreKind +} diff --git a/management/server/file_store_test.go b/management/server/file_store_test.go index e2f07acda6e..705e9f14935 100644 --- a/management/server/file_store_test.go +++ b/management/server/file_store_test.go @@ -387,7 +387,7 @@ func TestFileStore_GetAccount(t *testing.T) { assert.Equal(t, expected.DomainCategory, account.DomainCategory) assert.Equal(t, expected.Domain, account.Domain) assert.Equal(t, expected.CreatedBy, account.CreatedBy) - assert.Equal(t, expected.Network.Id, account.Network.Id) + assert.Equal(t, expected.Network.Identifier, account.Network.Identifier) assert.Len(t, account.Peers, len(expected.Peers)) assert.Len(t, account.Users, len(expected.Users)) assert.Len(t, account.SetupKeys, len(expected.SetupKeys)) diff --git a/management/server/group.go b/management/server/group.go index a7502134aa3..28606e02d6f 100644 --- a/management/server/group.go +++ b/management/server/group.go @@ -23,6 +23,9 @@ type Group struct { // ID of the group ID string + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` + // Name visible in the UI Name string @@ -30,7 +33,7 @@ type Group struct { Issued string // Peers list of the group - Peers []string + Peers []string `gorm:"serializer:json"` } // EventMeta returns activity event meta related to the group diff --git a/management/server/group_test.go b/management/server/group_test.go index 3e2d6d3cc64..e300fe7fbfb 100644 --- a/management/server/group_test.go +++ b/management/server/group_test.go @@ -80,6 +80,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { groupForRoute := &Group{ "grp-for-route", + "account-id", "Group for route", GroupIssuedAPI, make([]string, 0), @@ -87,6 +88,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { groupForNameServerGroups := &Group{ "grp-for-name-server-grp", + "account-id", "Group for name server groups", GroupIssuedAPI, make([]string, 0), @@ -94,6 +96,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { groupForPolicies := &Group{ "grp-for-policies", + "account-id", "Group for policies", GroupIssuedAPI, make([]string, 0), @@ -101,6 +104,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { groupForSetupKeys := &Group{ "grp-for-keys", + "account-id", "Group for setup keys", GroupIssuedAPI, make([]string, 0), @@ -108,6 +112,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) { groupForUsers := &Group{ "grp-for-users", + "account-id", "Group for users", GroupIssuedAPI, make([]string, 0), diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index b4a527e463d..06fc6669de1 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -405,7 +405,7 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error) return nil, "", err } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - store, err := NewFileStore(config.Datadir, nil) + store, err := NewStoreFromJson(config.Datadir, nil) if err != nil { return nil, "", err } diff --git a/management/server/management_test.go b/management/server/management_test.go index fa35cfdef4e..375e7e634e2 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -393,6 +393,7 @@ var _ = Describe("Management service", func() { ipChannel := make(chan string, 20) for i := 0; i < initialPeers; i++ { go func() { + defer GinkgoRecover() key, _ := wgtypes.GenerateKey() loginPeerWithValidSetupKey(serverPubKey, key, client) encryptedBytes, err := encryption.EncryptMessage(serverPubKey, key, &mgmtProto.SyncRequest{}) @@ -496,7 +497,7 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) { Expect(err).NotTo(HaveOccurred()) s := grpc.NewServer() - store, err := server.NewFileStore(config.Datadir, nil) + store, err := server.NewStoreFromJson(config.Datadir, nil) if err != nil { log.Fatalf("failed creating a store: %s: %v", config.Datadir, err) } diff --git a/management/server/metrics/selfhosted.go b/management/server/metrics/selfhosted.go index 3b3db0baa87..59364b940e9 100644 --- a/management/server/metrics/selfhosted.go +++ b/management/server/metrics/selfhosted.go @@ -48,6 +48,7 @@ type properties map[string]interface{} // DataSource metric data source type DataSource interface { GetAllAccounts() []*server.Account + GetStoreKind() server.StoreKind } // ConnManager peer connection manager that holds state for current active connections @@ -295,6 +296,7 @@ func (w *Worker) generateProperties() properties { metricsProperties["max_active_peer_version"] = maxActivePeerVersion metricsProperties["ui_clients"] = uiClient metricsProperties["idp_manager"] = w.idpManager + metricsProperties["store_kind"] = w.dataSource.GetStoreKind() for protocol, count := range rulesProtocol { metricsProperties["rules_protocol_"+protocol] = count diff --git a/management/server/metrics/selfhosted_test.go b/management/server/metrics/selfhosted_test.go index c61613fd26f..f69c0f8f8ec 100644 --- a/management/server/metrics/selfhosted_test.go +++ b/management/server/metrics/selfhosted_test.go @@ -151,6 +151,11 @@ func (mockDatasource) GetAllAccounts() []*server.Account { } } +// GetStoreKind returns FileStoreKind +func (mockDatasource) GetStoreKind() server.StoreKind { + return server.FileStoreKind +} + // TestGenerateProperties tests and validate the properties generation by using the mockDatasource for the Worker.generateProperties func TestGenerateProperties(t *testing.T) { ds := mockDatasource{} @@ -236,4 +241,8 @@ func TestGenerateProperties(t *testing.T) { if properties["user_peers"] != 2 { t.Errorf("expected 2 user_peers, got %d", properties["user_peers"]) } + + if properties["store_kind"] != server.FileStoreKind { + t.Errorf("expected JsonFile, got %s", properties["store_kind"]) + } } diff --git a/management/server/nameserver_test.go b/management/server/nameserver_test.go index 26977116b86..8809dc8ad9b 100644 --- a/management/server/nameserver_test.go +++ b/management/server/nameserver_test.go @@ -749,7 +749,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) { func createNSStore(t *testing.T) (Store, error) { dataDir := t.TempDir() - store, err := NewFileStore(dataDir, nil) + store, err := NewStoreFromJson(dataDir, nil) if err != nil { return nil, err } diff --git a/management/server/network.go b/management/server/network.go index 70f218f6688..c5b165caeda 100644 --- a/management/server/network.go +++ b/management/server/network.go @@ -34,14 +34,14 @@ type NetworkMap struct { } type Network struct { - Id string - Net net.IPNet - Dns string + Identifier string `json:"id"` + Net net.IPNet `gorm:"serializer:gob"` + Dns string // Serial is an ID that increments by 1 when any change to the network happened (e.g. new peer has been added). // Used to synchronize state to the client apps. Serial uint64 - mu sync.Mutex `json:"-"` + mu sync.Mutex `json:"-" gorm:"-"` } // NewNetwork creates a new Network initializing it with a Serial=0 @@ -56,10 +56,10 @@ func NewNetwork() *Network { intn := r.Intn(len(sub)) return &Network{ - Id: xid.New().String(), - Net: sub[intn].IPNet, - Dns: "", - Serial: 0} + Identifier: xid.New().String(), + Net: sub[intn].IPNet, + Dns: "", + Serial: 0} } // IncSerial increments Serial by 1 reflecting that the network state has been changed @@ -78,10 +78,10 @@ func (n *Network) CurrentSerial() uint64 { func (n *Network) Copy() *Network { return &Network{ - Id: n.Id, - Net: n.Net, - Dns: n.Dns, - Serial: n.Serial, + Identifier: n.Identifier, + Net: n.Net, + Dns: n.Dns, + Serial: n.Serial, } } diff --git a/management/server/peer.go b/management/server/peer.go index e5c6e39d65d..f38e19e870d 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -72,22 +72,24 @@ type PeerLogin struct { // The Peer is a WireGuard peer identified by a public key type Peer struct { // ID is an internal ID of the peer - ID string + ID string `gorm:"primaryKey"` + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index;uniqueIndex:idx_peers_account_id_ip"` // WireGuard public key - Key string + Key string `gorm:"index"` // A setup key this peer was registered with SetupKey string // IP address of the Peer - IP net.IP + IP net.IP `gorm:"uniqueIndex:idx_peers_account_id_ip"` // Meta is a Peer system meta data - Meta PeerSystemMeta + Meta PeerSystemMeta `gorm:"embedded;embeddedPrefix:meta_"` // Name is peer's name (machine name) Name string // DNSLabel is the parsed peer name for domain resolution. It is used to form an FQDN by appending the account's // domain to the peer label. e.g. peer-dns-label.netbird.cloud DNSLabel string // Status peer's management connection status - Status *PeerStatus + Status *PeerStatus `gorm:"embedded;embeddedPrefix:peer_status_"` // The user ID that registered the peer UserID string // SSHKey is a public SSH key of the peer @@ -116,6 +118,7 @@ func (p *Peer) Copy() *Peer { } return &Peer{ ID: p.ID, + AccountID: p.AccountID, Key: p.Key, SetupKey: p.SetupKey, IP: p.IP, diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 36e96df4311..9d5a8bfb99d 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -369,8 +369,8 @@ func TestAccountManager_GetPeerNetwork(t *testing.T) { return } - if account.Network.Id != network.Id { - t.Errorf("expecting Account Networks ID to be equal, got %s expected %s", network.Id, account.Network.Id) + if account.Network.Identifier != network.Identifier { + t.Errorf("expecting Account Networks ID to be equal, got %s expected %s", network.Identifier, account.Network.Identifier) } } diff --git a/management/server/personal_access_token.go b/management/server/personal_access_token.go index c7deca9dee4..f466661120f 100644 --- a/management/server/personal_access_token.go +++ b/management/server/personal_access_token.go @@ -26,7 +26,9 @@ const ( // PersonalAccessToken holds all information about a PAT including a hashed version of it for verification type PersonalAccessToken struct { - ID string + ID string `gorm:"primaryKey"` + // User is a reference to Account that this object belongs + UserID string `gorm:"index"` Name string HashedToken string ExpirationDate time.Time diff --git a/management/server/policy.go b/management/server/policy.go index 308a5c3c0db..d470ab4bf72 100644 --- a/management/server/policy.go +++ b/management/server/policy.go @@ -63,7 +63,10 @@ type PolicyUpdateOperation struct { // PolicyRule is the metadata of the policy type PolicyRule struct { // ID of the policy rule - ID string + ID string `gorm:"primaryKey"` + + // PolicyID is a reference to Policy that this object belongs + PolicyID string `json:"-" gorm:"index"` // Name of the rule visible in the UI Name string @@ -78,10 +81,10 @@ type PolicyRule struct { Action PolicyTrafficActionType // Destinations policy destination groups - Destinations []string + Destinations []string `gorm:"serializer:json"` // Sources policy source groups - Sources []string + Sources []string `gorm:"serializer:json"` // Bidirectional define if the rule is applicable in both directions, sources, and destinations Bidirectional bool @@ -90,7 +93,7 @@ type PolicyRule struct { Protocol PolicyRuleProtocolType // Ports or it ranges list - Ports []string + Ports []string `gorm:"serializer:json"` } // Copy returns a copy of a policy rule @@ -128,8 +131,11 @@ func (pm *PolicyRule) ToRule() *Rule { // Policy of the Rego query type Policy struct { - // ID of the policy - ID string + // ID of the policy' + ID string `gorm:"primaryKey"` + + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` // Name of the Policy Name string @@ -141,7 +147,7 @@ type Policy struct { Enabled bool // Rules of the policy - Rules []*PolicyRule + Rules []*PolicyRule `gorm:"foreignKey:PolicyID;references:id"` } // Copy returns a copy of the policy. @@ -201,7 +207,6 @@ type FirewallRule struct { // This function returns the list of peers and firewall rules that are applicable to a given peer. func (a *Account) getPeerConnectionResources(peerID string) ([]*Peer, []*FirewallRule) { generateResources, getAccumulatedResources := a.connResourcesGenerator() - for _, policy := range a.Policies { if !policy.Enabled { continue diff --git a/management/server/route_test.go b/management/server/route_test.go index 00ef3e93a4d..efd73d6c2d1 100644 --- a/management/server/route_test.go +++ b/management/server/route_test.go @@ -1017,7 +1017,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) { func createRouterStore(t *testing.T) (Store, error) { dataDir := t.TempDir() - store, err := NewFileStore(dataDir, nil) + store, err := NewStoreFromJson(dataDir, nil) if err != nil { return nil, err } diff --git a/management/server/rule.go b/management/server/rule.go index cb85d633d25..19085840cc7 100644 --- a/management/server/rule.go +++ b/management/server/rule.go @@ -25,6 +25,9 @@ type Rule struct { // ID of the rule ID string + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` + // Name of the rule visible in the UI Name string @@ -35,10 +38,10 @@ type Rule struct { Disabled bool // Source list of groups IDs of peers - Source []string + Source []string `gorm:"serializer:json"` // Destination list of groups IDs of peers - Destination []string + Destination []string `gorm:"serializer:json"` // Flow of the traffic allowed by the rule Flow TrafficFlowType diff --git a/management/server/setupkey.go b/management/server/setupkey.go index 6e626d08411..a33f537a7e6 100644 --- a/management/server/setupkey.go +++ b/management/server/setupkey.go @@ -68,13 +68,15 @@ type SetupKeyType string // SetupKey represents a pre-authorized key used to register machines (peers) type SetupKey struct { - Id string + Id string + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` Key string Name string Type SetupKeyType CreatedAt time.Time ExpiresAt time.Time - UpdatedAt time.Time + UpdatedAt time.Time `gorm:"autoUpdateTime:false"` // Revoked indicates whether the key was revoked or not (we don't remove them for tracking purposes) Revoked bool // UsedTimes indicates how many times the key was used @@ -82,7 +84,7 @@ type SetupKey struct { // LastUsed last time the key was used for peer registration LastUsed time.Time // AutoGroups is a list of Group IDs that are auto assigned to a Peer when it uses this key to register - AutoGroups []string + AutoGroups []string `gorm:"serializer:json"` // UsageLimit indicates the number of times this key can be used to enroll a machine. // The value of 0 indicates the unlimited usage. UsageLimit int @@ -99,6 +101,7 @@ func (key *SetupKey) Copy() *SetupKey { } return &SetupKey{ Id: key.Id, + AccountID: key.AccountID, Key: key.Key, Name: key.Name, Type: key.Type, diff --git a/management/server/sqlite_store.go b/management/server/sqlite_store.go new file mode 100644 index 00000000000..dfe6c3dfa95 --- /dev/null +++ b/management/server/sqlite_store.go @@ -0,0 +1,457 @@ +package server + +import ( + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + nbdns "github.com/netbirdio/netbird/dns" + "github.com/netbirdio/netbird/management/server/status" + "github.com/netbirdio/netbird/management/server/telemetry" + "github.com/netbirdio/netbird/route" + log "github.com/sirupsen/logrus" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +// SqliteStore represents an account storage backed by a Sqlite DB persisted to disk +type SqliteStore struct { + db *gorm.DB + storeFile string + accountLocks sync.Map + globalAccountLock sync.Mutex + metrics telemetry.AppMetrics + installationPK int +} + +type installation struct { + ID uint `gorm:"primaryKey"` + InstallationIDValue string +} + +// NewSqliteStore restores a store from the file located in the datadir +func NewSqliteStore(dataDir string, metrics telemetry.AppMetrics) (*SqliteStore, error) { + storeStr := "store.db?cache=shared" + if runtime.GOOS == "windows" { + // Vo avoid `The process cannot access the file because it is being used by another process` on Windows + storeStr = "store.db" + } + + file := filepath.Join(dataDir, storeStr) + db, err := gorm.Open(sqlite.Open(file), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + PrepareStmt: true, + }) + if err != nil { + return nil, err + } + + sql, err := db.DB() + if err != nil { + return nil, err + } + conns := runtime.NumCPU() + sql.SetMaxOpenConns(conns) // TODO: make it configurable + + err = db.AutoMigrate( + &SetupKey{}, &Peer{}, &User{}, &PersonalAccessToken{}, &Group{}, &Rule{}, + &Account{}, &Policy{}, &PolicyRule{}, &route.Route{}, &nbdns.NameServerGroup{}, + &installation{}, + ) + if err != nil { + return nil, err + } + + return &SqliteStore{db: db, storeFile: file, metrics: metrics, installationPK: 1}, nil +} + +// NewSqliteStoreFromFileStore restores a store from FileStore and stores SQLite DB in the file located in datadir +func NewSqliteStoreFromFileStore(filestore *FileStore, dataDir string, metrics telemetry.AppMetrics) (*SqliteStore, error) { + store, err := NewSqliteStore(dataDir, metrics) + if err != nil { + return nil, err + } + + err = store.SaveInstallationID(filestore.InstallationID) + if err != nil { + return nil, err + } + + for _, account := range filestore.GetAllAccounts() { + err := store.SaveAccount(account) + if err != nil { + return nil, err + } + } + + return store, nil +} + +// AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock +func (s *SqliteStore) AcquireGlobalLock() (unlock func()) { + log.Debugf("acquiring global lock") + start := time.Now() + s.globalAccountLock.Lock() + + unlock = func() { + s.globalAccountLock.Unlock() + log.Debugf("released global lock in %v", time.Since(start)) + } + + took := time.Since(start) + log.Debugf("took %v to acquire global lock", took) + if s.metrics != nil { + s.metrics.StoreMetrics().CountGlobalLockAcquisitionDuration(took) + } + + return unlock +} + +func (s *SqliteStore) AcquireAccountLock(accountID string) (unlock func()) { + log.Debugf("acquiring lock for account %s", accountID) + + start := time.Now() + value, _ := s.accountLocks.LoadOrStore(accountID, &sync.Mutex{}) + mtx := value.(*sync.Mutex) + mtx.Lock() + + unlock = func() { + mtx.Unlock() + log.Debugf("released lock for account %s in %v", accountID, time.Since(start)) + } + + return unlock +} + +func (s *SqliteStore) SaveAccount(account *Account) error { + start := time.Now() + + for _, key := range account.SetupKeys { + account.SetupKeysG = append(account.SetupKeysG, *key) + } + + for id, peer := range account.Peers { + peer.ID = id + account.PeersG = append(account.PeersG, *peer) + } + + for id, user := range account.Users { + user.Id = id + for id, pat := range user.PATs { + pat.ID = id + user.PATsG = append(user.PATsG, *pat) + } + account.UsersG = append(account.UsersG, *user) + } + + for id, group := range account.Groups { + group.ID = id + account.GroupsG = append(account.GroupsG, *group) + } + + for id, rule := range account.Rules { + rule.ID = id + account.RulesG = append(account.RulesG, *rule) + } + + for id, route := range account.Routes { + route.ID = id + account.RoutesG = append(account.RoutesG, *route) + } + + for id, ns := range account.NameServerGroups { + ns.ID = id + account.NameServerGroupsG = append(account.NameServerGroupsG, *ns) + } + + err := s.db.Transaction(func(tx *gorm.DB) error { + result := tx.Select(clause.Associations).Delete(account.Policies, "account_id = ?", account.Id) + if result.Error != nil { + return result.Error + } + + result = tx.Select(clause.Associations).Delete(account.UsersG, "account_id = ?", account.Id) + if result.Error != nil { + return result.Error + } + + result = tx.Select(clause.Associations).Delete(account) + if result.Error != nil { + return result.Error + } + + result = tx. + Session(&gorm.Session{FullSaveAssociations: true}). + Clauses(clause.OnConflict{UpdateAll: true}).Create(account) + if result.Error != nil { + return result.Error + } + return nil + }) + + took := time.Since(start) + if s.metrics != nil { + s.metrics.StoreMetrics().CountPersistenceDuration(took) + } + log.Debugf("took %d ms to persist an account to the SQLite", took.Milliseconds()) + + return err +} + +func (s *SqliteStore) SaveInstallationID(ID string) error { + installation := installation{InstallationIDValue: ID} + installation.ID = uint(s.installationPK) + + return s.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(&installation).Error +} + +func (s *SqliteStore) GetInstallationID() string { + var installation installation + + if result := s.db.First(&installation, "id = ?", s.installationPK); result.Error != nil { + return "" + } + + return installation.InstallationIDValue +} + +func (s *SqliteStore) SavePeerStatus(accountID, peerID string, peerStatus PeerStatus) error { + var peer Peer + + result := s.db.First(&peer, "account_id = ? and id = ?", accountID, peerID) + if result.Error != nil { + return status.Errorf(status.NotFound, "peer %s not found", peerID) + } + + peer.Status = &peerStatus + + return s.db.Save(peer).Error +} + +// DeleteHashedPAT2TokenIDIndex is noop in Sqlite +func (s *SqliteStore) DeleteHashedPAT2TokenIDIndex(hashedToken string) error { + return nil +} + +// DeleteTokenID2UserIDIndex is noop in Sqlite +func (s *SqliteStore) DeleteTokenID2UserIDIndex(tokenID string) error { + return nil +} + +func (s *SqliteStore) GetAccountByPrivateDomain(domain string) (*Account, error) { + var account Account + + result := s.db.First(&account, "domain = ?", strings.ToLower(domain)) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: provided domain is not registered or is not private") + } + + // TODO: rework to not call GetAccount + return s.GetAccount(account.Id) +} + +func (s *SqliteStore) GetAccountBySetupKey(setupKey string) (*Account, error) { + var key SetupKey + result := s.db.Select("account_id").First(&key, "key = ?", strings.ToUpper(setupKey)) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + if key.AccountID == "" { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + return s.GetAccount(key.AccountID) +} + +func (s *SqliteStore) GetTokenIDByHashedToken(hashedToken string) (string, error) { + var token PersonalAccessToken + result := s.db.First(&token, "hashed_token = ?", hashedToken) + if result.Error != nil { + return "", status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + return token.ID, nil +} + +func (s *SqliteStore) GetUserByTokenID(tokenID string) (*User, error) { + var token PersonalAccessToken + result := s.db.First(&token, "id = ?", tokenID) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + if token.UserID == "" { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + var user User + result = s.db.Preload("PATsG").First(&user, "id = ?", token.UserID) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + user.PATs = make(map[string]*PersonalAccessToken, len(user.PATsG)) + for _, pat := range user.PATsG { + user.PATs[pat.ID] = &pat + } + + return &user, nil +} + +func (s *SqliteStore) GetAllAccounts() (all []*Account) { + var accounts []Account + result := s.db.Find(&accounts) + if result.Error != nil { + return all + } + + for _, account := range accounts { + if acc, err := s.GetAccount(account.Id); err == nil { + all = append(all, acc) + } + } + + return all +} + +func (s *SqliteStore) GetAccount(accountID string) (*Account, error) { + var account Account + + result := s.db.Model(&account). + Preload("UsersG.PATsG"). // have to be specifies as this is nester reference + Preload(clause.Associations). + First(&account, "id = ?", accountID) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found") + } + + // we have to manually preload policy rules as it seems that gorm preloading doesn't do it for us + for i, policy := range account.Policies { + var rules []*PolicyRule + err := s.db.Model(&PolicyRule{}).Find(&rules, "policy_id = ?", policy.ID).Error + if err != nil { + return nil, status.Errorf(status.NotFound, "account not found") + } + account.Policies[i].Rules = rules + } + + account.SetupKeys = make(map[string]*SetupKey, len(account.SetupKeysG)) + for _, key := range account.SetupKeysG { + account.SetupKeys[key.Key] = key.Copy() + } + account.SetupKeysG = nil + + account.Peers = make(map[string]*Peer, len(account.PeersG)) + for _, peer := range account.PeersG { + account.Peers[peer.ID] = peer.Copy() + } + account.PeersG = nil + + account.Users = make(map[string]*User, len(account.UsersG)) + for _, user := range account.UsersG { + user.PATs = make(map[string]*PersonalAccessToken, len(user.PATs)) + for _, pat := range user.PATsG { + user.PATs[pat.ID] = pat.Copy() + } + account.Users[user.Id] = user.Copy() + } + account.UsersG = nil + + account.Groups = make(map[string]*Group, len(account.GroupsG)) + for _, group := range account.GroupsG { + account.Groups[group.ID] = group.Copy() + } + account.GroupsG = nil + + account.Rules = make(map[string]*Rule, len(account.RulesG)) + for _, rule := range account.RulesG { + account.Rules[rule.ID] = rule.Copy() + } + account.RulesG = nil + + account.Routes = make(map[string]*route.Route, len(account.RoutesG)) + for _, route := range account.RoutesG { + account.Routes[route.ID] = route.Copy() + } + account.RoutesG = nil + + account.NameServerGroups = make(map[string]*nbdns.NameServerGroup, len(account.NameServerGroupsG)) + for _, ns := range account.NameServerGroupsG { + account.NameServerGroups[ns.ID] = ns.Copy() + } + account.NameServerGroupsG = nil + + return &account, nil +} + +func (s *SqliteStore) GetAccountByUser(userID string) (*Account, error) { + var user User + result := s.db.Select("account_id").First(&user, "id = ?", userID) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + if user.AccountID == "" { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + return s.GetAccount(user.AccountID) +} + +func (s *SqliteStore) GetAccountByPeerID(peerID string) (*Account, error) { + var peer Peer + result := s.db.Select("account_id").First(&peer, "id = ?", peerID) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + if peer.AccountID == "" { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + return s.GetAccount(peer.AccountID) +} + +func (s *SqliteStore) GetAccountByPeerPubKey(peerKey string) (*Account, error) { + var peer Peer + + result := s.db.Select("account_id").First(&peer, "key = ?", peerKey) + if result.Error != nil { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + if peer.AccountID == "" { + return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") + } + + return s.GetAccount(peer.AccountID) +} + +// SaveUserLastLogin stores the last login time for a user in DB. +func (s *SqliteStore) SaveUserLastLogin(accountID, userID string, lastLogin time.Time) error { + var peer Peer + + result := s.db.First(&peer, "account_id = ? and user_id = ?", accountID, userID) + if result.Error != nil { + return status.Errorf(status.NotFound, "user %s not found", userID) + } + + peer.LastLogin = lastLogin + + return s.db.Save(peer).Error +} + +// Close is noop in Sqlite +func (s *SqliteStore) Close() error { + return nil +} + +// GetStoreKind returns SqliteStoreKind +func (s *SqliteStore) GetStoreKind() StoreKind { + return SqliteStoreKind +} diff --git a/management/server/sqlite_store_test.go b/management/server/sqlite_store_test.go new file mode 100644 index 00000000000..4a16e25255d --- /dev/null +++ b/management/server/sqlite_store_test.go @@ -0,0 +1,229 @@ +package server + +import ( + "fmt" + "net" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/google/uuid" + "github.com/netbirdio/netbird/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSqlite_NewStore(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStore(t) + + if len(store.GetAllAccounts()) != 0 { + t.Errorf("expected to create a new empty Accounts map when creating a new FileStore") + } +} + +func TestSqlite_SaveAccount(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStore(t) + + account := newAccountWithId("account_id", "testuser", "") + setupKey := GenerateDefaultSetupKey() + account.SetupKeys[setupKey.Key] = setupKey + account.Peers["testpeer"] = &Peer{ + Key: "peerkey", + SetupKey: "peerkeysetupkey", + IP: net.IP{127, 0, 0, 1}, + Meta: PeerSystemMeta{}, + Name: "peer name", + Status: &PeerStatus{Connected: true, LastSeen: time.Now().UTC()}, + } + + err := store.SaveAccount(account) + require.NoError(t, err) + + account2 := newAccountWithId("account_id2", "testuser2", "") + setupKey = GenerateDefaultSetupKey() + account2.SetupKeys[setupKey.Key] = setupKey + account2.Peers["testpeer2"] = &Peer{ + Key: "peerkey2", + SetupKey: "peerkeysetupkey2", + IP: net.IP{127, 0, 0, 2}, + Meta: PeerSystemMeta{}, + Name: "peer name 2", + Status: &PeerStatus{Connected: true, LastSeen: time.Now().UTC()}, + } + + err = store.SaveAccount(account2) + require.NoError(t, err) + + if len(store.GetAllAccounts()) != 2 { + t.Errorf("expecting 2 Accounts to be stored after SaveAccount()") + } + + a, err := store.GetAccount(account.Id) + if a == nil { + t.Errorf("expecting Account to be stored after SaveAccount(): %v", err) + } + + if a != nil && len(a.Policies) != 1 { + t.Errorf("expecting Account to have one policy stored after SaveAccount(), got %d", len(a.Policies)) + } + + if a != nil && len(a.Policies[0].Rules) != 1 { + t.Errorf("expecting Account to have one policy rule stored after SaveAccount(), got %d", len(a.Policies[0].Rules)) + return + } + + if a, err := store.GetAccountByPeerPubKey("peerkey"); a == nil { + t.Errorf("expecting PeerKeyID2AccountID index updated after SaveAccount(): %v", err) + } + + if a, err := store.GetAccountByUser("testuser"); a == nil { + t.Errorf("expecting UserID2AccountID index updated after SaveAccount(): %v", err) + } + + if a, err := store.GetAccountByPeerID("testpeer"); a == nil { + t.Errorf("expecting PeerID2AccountID index updated after SaveAccount(): %v", err) + } + + if a, err := store.GetAccountBySetupKey(setupKey.Key); a == nil { + t.Errorf("expecting SetupKeyID2AccountID index updated after SaveAccount(): %v", err) + } +} + +func TestSqlite_SavePeerStatus(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStoreFromFile(t, "testdata/store.json") + + account, err := store.GetAccount("bf1c8084-ba50-4ce7-9439-34653001fc3b") + require.NoError(t, err) + + // save status of non-existing peer + newStatus := PeerStatus{Connected: true, LastSeen: time.Now().UTC()} + err = store.SavePeerStatus(account.Id, "non-existing-peer", newStatus) + assert.Error(t, err) + + // save new status of existing peer + account.Peers["testpeer"] = &Peer{ + Key: "peerkey", + ID: "testpeer", + SetupKey: "peerkeysetupkey", + IP: net.IP{127, 0, 0, 1}, + Meta: PeerSystemMeta{}, + Name: "peer name", + Status: &PeerStatus{Connected: false, LastSeen: time.Now().UTC()}, + } + + err = store.SaveAccount(account) + require.NoError(t, err) + + err = store.SavePeerStatus(account.Id, "testpeer", newStatus) + require.NoError(t, err) + + account, err = store.GetAccount(account.Id) + require.NoError(t, err) + + actual := account.Peers["testpeer"].Status + assert.Equal(t, newStatus, *actual) +} + +func TestSqlite_TestGetAccountByPrivateDomain(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStoreFromFile(t, "testdata/store.json") + + existingDomain := "test.com" + + account, err := store.GetAccountByPrivateDomain(existingDomain) + require.NoError(t, err, "should found account") + require.Equal(t, existingDomain, account.Domain, "domains should match") + + _, err = store.GetAccountByPrivateDomain("missing-domain.com") + require.Error(t, err, "should return error on domain lookup") +} + +func TestSqlite_GetTokenIDByHashedToken(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStoreFromFile(t, "testdata/store.json") + + hashed := "SoMeHaShEdToKeN" + id := "9dj38s35-63fb-11ec-90d6-0242ac120003" + + token, err := store.GetTokenIDByHashedToken(hashed) + require.NoError(t, err) + require.Equal(t, id, token) +} + +func TestSqlite_GetUserByTokenID(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("The SQLite store is not properly supported by Windows yet") + } + + store := newSqliteStoreFromFile(t, "testdata/store.json") + + id := "9dj38s35-63fb-11ec-90d6-0242ac120003" + + user, err := store.GetUserByTokenID(id) + require.NoError(t, err) + require.Equal(t, id, user.PATs[id].ID) +} + +func newSqliteStore(t *testing.T) *SqliteStore { + t.Helper() + + store, err := NewSqliteStore(t.TempDir(), nil) + require.NoError(t, err) + require.NotNil(t, store) + + return store +} + +func newSqliteStoreFromFile(t *testing.T, filename string) *SqliteStore { + t.Helper() + + storeDir := t.TempDir() + + err := util.CopyFileContents(filename, filepath.Join(storeDir, "store.json")) + require.NoError(t, err) + + fStore, err := NewFileStore(storeDir, nil) + require.NoError(t, err) + + store, err := NewSqliteStoreFromFileStore(fStore, storeDir, nil) + require.NoError(t, err) + require.NotNil(t, store) + + return store +} + +func newAccount(store Store, id int) error { + str := fmt.Sprintf("%s-%d", uuid.New().String(), id) + account := newAccountWithId(str, str+"-testuser", "example.com") + setupKey := GenerateDefaultSetupKey() + account.SetupKeys[setupKey.Key] = setupKey + account.Peers["p"+str] = &Peer{ + Key: "peerkey" + str, + SetupKey: "peerkeysetupkey", + IP: net.IP{127, 0, 0, 1}, + Meta: PeerSystemMeta{}, + Name: "peer name", + Status: &PeerStatus{Connected: true, LastSeen: time.Now().UTC()}, + } + + return store.SaveAccount(account) +} diff --git a/management/server/store.go b/management/server/store.go index 9ebe4123517..6606c91e63d 100644 --- a/management/server/store.go +++ b/management/server/store.go @@ -1,6 +1,12 @@ package server -import "time" +import ( + "fmt" + "os" + "time" + + "github.com/netbirdio/netbird/management/server/telemetry" +) type Store interface { GetAllAccounts() []*Account @@ -25,4 +31,63 @@ type Store interface { SaveUserLastLogin(accountID, userID string, lastLogin time.Time) error // Close should close the store persisting all unsaved data. Close() error + // GetStoreKind should return StoreKind of the current store implementation. + // This is also a method of metrics.DataSource interface. + GetStoreKind() StoreKind +} + +type StoreKind string + +const ( + FileStoreKind StoreKind = "JsonFile" + SqliteStoreKind StoreKind = "Sqlite" +) + +func GetStoreKindFromEnv() StoreKind { + kind, ok := os.LookupEnv("NETBIRD_STORE_KIND") + if !ok { + return FileStoreKind + } + + value := StoreKind(kind) + + if value == FileStoreKind || value == SqliteStoreKind { + return value + } + + return FileStoreKind +} + +func NewStore(kind StoreKind, dataDir string, metrics telemetry.AppMetrics) (Store, error) { + if kind == "" { + // fallback to env. Normally this only should be used from tests + kind = GetStoreKindFromEnv() + } + switch kind { + case FileStoreKind: + return NewFileStore(dataDir, metrics) + case SqliteStoreKind: + return NewSqliteStore(dataDir, metrics) + default: + return nil, fmt.Errorf("unsupported kind of store %s", kind) + } +} + +func NewStoreFromJson(dataDir string, metrics telemetry.AppMetrics) (Store, error) { + fstore, err := NewFileStore(dataDir, nil) + if err != nil { + return nil, err + } + + kind := GetStoreKindFromEnv() + + switch kind { + case FileStoreKind: + return fstore, nil + case SqliteStoreKind: + return NewSqliteStoreFromFileStore(fstore, dataDir, metrics) + default: + return nil, fmt.Errorf("unsupported kind of store %s", kind) + } + } diff --git a/management/server/store_test.go b/management/server/store_test.go new file mode 100644 index 00000000000..72bbaf9498e --- /dev/null +++ b/management/server/store_test.go @@ -0,0 +1,88 @@ +package server + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +type benchCase struct { + name string + storeFn func(b *testing.B) Store + size int +} + +var newFs = func(b *testing.B) Store { + store, _ := NewFileStore(b.TempDir(), nil) + return store +} + +var newSqlite = func(b *testing.B) Store { + store, _ := NewSqliteStore(b.TempDir(), nil) + return store +} + +func BenchmarkTest_StoreWrite(b *testing.B) { + cases := []benchCase{ + {name: "FileStore_Write", storeFn: newFs, size: 100}, + {name: "SqliteStore_Write", storeFn: newSqlite, size: 100}, + {name: "FileStore_Write", storeFn: newFs, size: 500}, + {name: "SqliteStore_Write", storeFn: newSqlite, size: 500}, + {name: "FileStore_Write", storeFn: newFs, size: 1000}, + {name: "SqliteStore_Write", storeFn: newSqlite, size: 1000}, + {name: "FileStore_Write", storeFn: newFs, size: 2000}, + {name: "SqliteStore_Write", storeFn: newSqlite, size: 2000}, + } + + for _, c := range cases { + name := fmt.Sprintf("%s_%d", c.name, c.size) + store := c.storeFn(b) + + for i := 0; i < c.size; i++ { + _ = newAccount(store, i) + } + + b.Run(name, func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := c.size + for pb.Next() { + i++ + err := newAccount(store, i) + require.NoError(b, err) + } + }) + }) + } +} + +func BenchmarkTest_StoreRead(b *testing.B) { + cases := []benchCase{ + {name: "FileStore_Read", storeFn: newFs, size: 100}, + {name: "SqliteStore_Read", storeFn: newSqlite, size: 100}, + {name: "FileStore_Read", storeFn: newFs, size: 500}, + {name: "SqliteStore_Read", storeFn: newSqlite, size: 500}, + {name: "FileStore_Read", storeFn: newFs, size: 1000}, + {name: "SqliteStore_Read", storeFn: newSqlite, size: 1000}, + } + + for _, c := range cases { + name := fmt.Sprintf("%s_%d", c.name, c.size) + store := c.storeFn(b) + + for i := 0; i < c.size; i++ { + _ = newAccount(store, i) + } + + accounts := store.GetAllAccounts() + id := accounts[c.size-1].Id + + b.Run(name, func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = store.GetAccount(id) + } + }) + }) + } +} diff --git a/management/server/testdata/store.json b/management/server/testdata/store.json index ecde766c374..1fa4e3a9a32 100644 --- a/management/server/testdata/store.json +++ b/management/server/testdata/store.json @@ -2,52 +2,87 @@ "Accounts": { "bf1c8084-ba50-4ce7-9439-34653001fc3b": { "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b", + "CreatedBy": "", "Domain": "test.com", "DomainCategory": "private", "IsDomainPrimaryAccount": true, "SetupKeys": { "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": { + "Id": "", + "AccountID": "", "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", "Name": "Default key", "Type": "reusable", "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", + "UpdatedAt": "0001-01-01T00:00:00Z", "Revoked": false, - "UsedTimes": 0 - + "UsedTimes": 0, + "LastUsed": "0001-01-01T00:00:00Z", + "AutoGroups": null, + "UsageLimit": 0, + "Ephemeral": false } }, "Network": { - "Id": "af1c8024-ha40-4ce2-9418-34653101fc3c", + "id": "af1c8024-ha40-4ce2-9418-34653101fc3c", "Net": { "IP": "100.64.0.0", "Mask": "//8AAA==" }, - "Dns": null + "Dns": "", + "Serial": 0 }, "Peers": {}, "Users": { "edafee4e-63fb-11ec-90d6-0242ac120003": { "Id": "edafee4e-63fb-11ec-90d6-0242ac120003", + "AccountID": "", "Role": "admin", - "PATs": {} + "IsServiceUser": false, + "ServiceUserName": "", + "AutoGroups": null, + "PATs": {}, + "Blocked": false, + "LastLogin": "0001-01-01T00:00:00Z" }, "f4f6d672-63fb-11ec-90d6-0242ac120003": { "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003", + "AccountID": "", "Role": "user", + "IsServiceUser": false, + "ServiceUserName": "", + "AutoGroups": null, "PATs": { "9dj38s35-63fb-11ec-90d6-0242ac120003": { - "ID":"9dj38s35-63fb-11ec-90d6-0242ac120003", - "Description":"some Description", - "HashedToken":"SoMeHaShEdToKeN", - "ExpirationDate":"2023-02-27T00:00:00Z", - "CreatedBy":"user", - "CreatedAt":"2023-01-01T00:00:00Z", - "LastUsed":"2023-02-01T00:00:00Z" + "ID": "9dj38s35-63fb-11ec-90d6-0242ac120003", + "UserID": "", + "Name": "", + "HashedToken": "SoMeHaShEdToKeN", + "ExpirationDate": "2023-02-27T00:00:00Z", + "CreatedBy": "user", + "CreatedAt": "2023-01-01T00:00:00Z", + "LastUsed": "2023-02-01T00:00:00Z" } - } + }, + "Blocked": false, + "LastLogin": "0001-01-01T00:00:00Z" } + }, + "Groups": null, + "Rules": null, + "Policies": [], + "Routes": null, + "NameServerGroups": null, + "DNSSettings": null, + "Settings": { + "PeerLoginExpirationEnabled": false, + "PeerLoginExpiration": 86400000000000, + "GroupsPropagationEnabled": false, + "JWTGroupsEnabled": false, + "JWTGroupsClaimName": "" } } - } + }, + "InstallationID": "" } \ No newline at end of file diff --git a/management/server/user.go b/management/server/user.go index 3169c784f14..5858720805e 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -44,14 +44,17 @@ type UserRole string // User represents a user of the system type User struct { - Id string + Id string `gorm:"primaryKey"` + // AccountID is a reference to Account that this object belongs + AccountID string `json:"-" gorm:"index"` Role UserRole IsServiceUser bool // ServiceUserName is only set if IsServiceUser is true ServiceUserName string // AutoGroups is a list of Group IDs to auto-assign to peers registered by this user - AutoGroups []string - PATs map[string]*PersonalAccessToken + AutoGroups []string `gorm:"serializer:json"` + PATs map[string]*PersonalAccessToken `gorm:"-"` + PATsG []PersonalAccessToken `json:"-" gorm:"foreignKey:UserID;references:id"` // Blocked indicates whether the user is blocked. Blocked users can't use the system. Blocked bool // LastLogin is the last time the user logged in to IdP @@ -124,6 +127,7 @@ func (u *User) Copy() *User { } return &User{ Id: u.Id, + AccountID: u.AccountID, Role: u.Role, AutoGroups: autoGroups, IsServiceUser: u.IsServiceUser, diff --git a/management/server/user_test.go b/management/server/user_test.go index 1565814b81b..fdaffc693ea 100644 --- a/management/server/user_test.go +++ b/management/server/user_test.go @@ -251,6 +251,7 @@ func TestUser_Copy(t *testing.T) { // this is an imaginary case which will never be in DB this way user := User{ Id: "userId", + AccountID: "accountId", Role: "role", IsServiceUser: true, ServiceUserName: "servicename", @@ -291,6 +292,11 @@ func validateStruct(s interface{}) (err error) { field := structVal.Field(i) fieldName := structType.Field(i).Name + // skip gorm internal fields + if json, ok := structType.Field(i).Tag.Lookup("json"); ok && json == "-" { + continue + } + isSet := field.IsValid() && (!field.IsZero() || field.Type().String() == "bool") if !isSet { diff --git a/route/route.go b/route/route.go index eb7bcba2f32..194e0c80d0f 100644 --- a/route/route.go +++ b/route/route.go @@ -65,17 +65,19 @@ func ToPrefixType(prefix string) NetworkType { // Route represents a route type Route struct { - ID string - Network netip.Prefix + ID string `gorm:"primaryKey"` + // AccountID is a reference to Account that this object belongs + AccountID string `gorm:"index"` + Network netip.Prefix `gorm:"serializer:gob"` NetID string Description string Peer string - PeerGroups []string + PeerGroups []string `gorm:"serializer:gob"` NetworkType NetworkType Masquerade bool Metric int Enabled bool - Groups []string + Groups []string `gorm:"serializer:json"` } // EventMeta returns activity event meta related to the route