From 82ab433a122b530264d827980df1f91a3f2ec649 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Tue, 5 Dec 2023 15:06:57 +0100 Subject: [PATCH 01/10] change gomigrate for dbmate --- db/schema.sql | 47 ++++++++++++++++++++++++++ storage/db/schema.sql | 47 ++++++++++++++++++++++++++ storage/engine.go | 7 ++-- storage/sql_migrations/001_initial.sql | 5 +++ 4 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 db/schema.sql create mode 100644 storage/db/schema.sql create mode 100644 storage/sql_migrations/001_initial.sql diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 0000000000..9df248474c --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,47 @@ +CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(128) primary key); +CREATE TABLE discovery_service +( + -- id is the unique identifier for the service. It comes from the service definition. + id varchar(200) not null primary key, + lamport_timestamp integer not null +); +CREATE TABLE discovery_presentation +( + id varchar(36) not null primary key, + service_id varchar(36) not null, + lamport_timestamp integer not null, + credential_subject_id varchar not null, + presentation_id varchar not null, + presentation_raw varchar not null, + presentation_expiration integer not null, + unique (service_id, credential_subject_id), + constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade +); +CREATE INDEX idx_discovery_presentation_expiration on discovery_presentation (presentation_expiration); +CREATE TABLE discovery_credential +( + id varchar(36) not null primary key, + -- presentation_id is NOT the ID of the presentation (VerifiablePresentation.ID), but refers to the presentation record in the discovery_presentation table. + presentation_id varchar(36) not null, + credential_id varchar not null, + credential_issuer varchar not null, + credential_subject_id varchar not null, + -- for now, credentials with at most 2 types are supported. + -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. + -- When credentials with 3 or more types appear, we could have to use a separate table for the types. + credential_type varchar, + constraint fk_discovery_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade +); +CREATE TABLE discovery_credential_prop +( + credential_id varchar(36) not null, + key varchar not null, + value varchar, + PRIMARY KEY (credential_id, key), + -- cascading delete: if the presentation gets deleted, the properties get deleted as well + constraint fk_discovery_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade +); +-- Dbmate schema migrations +INSERT INTO "schema_migrations" (version) VALUES + ('001'), + ('002'); diff --git a/storage/db/schema.sql b/storage/db/schema.sql new file mode 100644 index 0000000000..9df248474c --- /dev/null +++ b/storage/db/schema.sql @@ -0,0 +1,47 @@ +CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(128) primary key); +CREATE TABLE discovery_service +( + -- id is the unique identifier for the service. It comes from the service definition. + id varchar(200) not null primary key, + lamport_timestamp integer not null +); +CREATE TABLE discovery_presentation +( + id varchar(36) not null primary key, + service_id varchar(36) not null, + lamport_timestamp integer not null, + credential_subject_id varchar not null, + presentation_id varchar not null, + presentation_raw varchar not null, + presentation_expiration integer not null, + unique (service_id, credential_subject_id), + constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade +); +CREATE INDEX idx_discovery_presentation_expiration on discovery_presentation (presentation_expiration); +CREATE TABLE discovery_credential +( + id varchar(36) not null primary key, + -- presentation_id is NOT the ID of the presentation (VerifiablePresentation.ID), but refers to the presentation record in the discovery_presentation table. + presentation_id varchar(36) not null, + credential_id varchar not null, + credential_issuer varchar not null, + credential_subject_id varchar not null, + -- for now, credentials with at most 2 types are supported. + -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. + -- When credentials with 3 or more types appear, we could have to use a separate table for the types. + credential_type varchar, + constraint fk_discovery_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade +); +CREATE TABLE discovery_credential_prop +( + credential_id varchar(36) not null, + key varchar not null, + value varchar, + PRIMARY KEY (credential_id, key), + -- cascading delete: if the presentation gets deleted, the properties get deleted as well + constraint fk_discovery_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade +); +-- Dbmate schema migrations +INSERT INTO "schema_migrations" (version) VALUES + ('001'), + ('002'); diff --git a/storage/engine.go b/storage/engine.go index 2d91ce1141..cdefe022e2 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -32,13 +32,14 @@ import ( "time" "github.com/amacneil/dbmate/v2/pkg/dbmate" - _ "github.com/amacneil/dbmate/v2/pkg/driver/mysql" - _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" - _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage/log" "github.com/redis/go-redis/v9" + + _ "github.com/amacneil/dbmate/v2/pkg/driver/mysql" + _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" + _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" ) const storeShutdownTimeout = 5 * time.Second diff --git a/storage/sql_migrations/001_initial.sql b/storage/sql_migrations/001_initial.sql new file mode 100644 index 0000000000..fdd74f6191 --- /dev/null +++ b/storage/sql_migrations/001_initial.sql @@ -0,0 +1,5 @@ +-- migrate:up +select 1; + +-- migrate:down +select 1; \ No newline at end of file From 487abaed2698b38a135435fe87205f1b5561b6a4 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Tue, 5 Dec 2023 15:08:18 +0100 Subject: [PATCH 02/10] remove obsolete file --- storage/db/schema.sql | 47 ------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 storage/db/schema.sql diff --git a/storage/db/schema.sql b/storage/db/schema.sql deleted file mode 100644 index 9df248474c..0000000000 --- a/storage/db/schema.sql +++ /dev/null @@ -1,47 +0,0 @@ -CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(128) primary key); -CREATE TABLE discovery_service -( - -- id is the unique identifier for the service. It comes from the service definition. - id varchar(200) not null primary key, - lamport_timestamp integer not null -); -CREATE TABLE discovery_presentation -( - id varchar(36) not null primary key, - service_id varchar(36) not null, - lamport_timestamp integer not null, - credential_subject_id varchar not null, - presentation_id varchar not null, - presentation_raw varchar not null, - presentation_expiration integer not null, - unique (service_id, credential_subject_id), - constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade -); -CREATE INDEX idx_discovery_presentation_expiration on discovery_presentation (presentation_expiration); -CREATE TABLE discovery_credential -( - id varchar(36) not null primary key, - -- presentation_id is NOT the ID of the presentation (VerifiablePresentation.ID), but refers to the presentation record in the discovery_presentation table. - presentation_id varchar(36) not null, - credential_id varchar not null, - credential_issuer varchar not null, - credential_subject_id varchar not null, - -- for now, credentials with at most 2 types are supported. - -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. - -- When credentials with 3 or more types appear, we could have to use a separate table for the types. - credential_type varchar, - constraint fk_discovery_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade -); -CREATE TABLE discovery_credential_prop -( - credential_id varchar(36) not null, - key varchar not null, - value varchar, - PRIMARY KEY (credential_id, key), - -- cascading delete: if the presentation gets deleted, the properties get deleted as well - constraint fk_discovery_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade -); --- Dbmate schema migrations -INSERT INTO "schema_migrations" (version) VALUES - ('001'), - ('002'); From 7f351eae7cb6cdd03c241f5621ed3d611682d165 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Tue, 5 Dec 2023 15:11:41 +0100 Subject: [PATCH 03/10] remove obsolete file --- db/schema.sql | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 db/schema.sql diff --git a/db/schema.sql b/db/schema.sql deleted file mode 100644 index 9df248474c..0000000000 --- a/db/schema.sql +++ /dev/null @@ -1,47 +0,0 @@ -CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(128) primary key); -CREATE TABLE discovery_service -( - -- id is the unique identifier for the service. It comes from the service definition. - id varchar(200) not null primary key, - lamport_timestamp integer not null -); -CREATE TABLE discovery_presentation -( - id varchar(36) not null primary key, - service_id varchar(36) not null, - lamport_timestamp integer not null, - credential_subject_id varchar not null, - presentation_id varchar not null, - presentation_raw varchar not null, - presentation_expiration integer not null, - unique (service_id, credential_subject_id), - constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade -); -CREATE INDEX idx_discovery_presentation_expiration on discovery_presentation (presentation_expiration); -CREATE TABLE discovery_credential -( - id varchar(36) not null primary key, - -- presentation_id is NOT the ID of the presentation (VerifiablePresentation.ID), but refers to the presentation record in the discovery_presentation table. - presentation_id varchar(36) not null, - credential_id varchar not null, - credential_issuer varchar not null, - credential_subject_id varchar not null, - -- for now, credentials with at most 2 types are supported. - -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. - -- When credentials with 3 or more types appear, we could have to use a separate table for the types. - credential_type varchar, - constraint fk_discovery_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade -); -CREATE TABLE discovery_credential_prop -( - credential_id varchar(36) not null, - key varchar not null, - value varchar, - PRIMARY KEY (credential_id, key), - -- cascading delete: if the presentation gets deleted, the properties get deleted as well - constraint fk_discovery_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade -); --- Dbmate schema migrations -INSERT INTO "schema_migrations" (version) VALUES - ('001'), - ('002'); From 02141a8514d961e59db77235eae904d81a2d9c2d Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 6 Dec 2023 08:50:42 +0100 Subject: [PATCH 04/10] PR feedback and logger --- storage/engine_test.go | 2 +- storage/sql_migrations/001_initial.sql | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 storage/sql_migrations/001_initial.sql diff --git a/storage/engine_test.go b/storage/engine_test.go index aa27b500a1..f4a6ceb4b9 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -143,6 +143,6 @@ func Test_engine_sqlDatabase(t *testing.T) { require.NoError(t, row.Err()) var count int assert.NoError(t, row.Scan(&count)) - assert.Equal(t, 2, count) + assert.Equal(t, 1, count) }) } diff --git a/storage/sql_migrations/001_initial.sql b/storage/sql_migrations/001_initial.sql deleted file mode 100644 index fdd74f6191..0000000000 --- a/storage/sql_migrations/001_initial.sql +++ /dev/null @@ -1,5 +0,0 @@ --- migrate:up -select 1; - --- migrate:down -select 1; \ No newline at end of file From 9ea6ca5525a477b90236417f270f4a353f6bcd64 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 6 Dec 2023 09:19:42 +0100 Subject: [PATCH 05/10] rebase on master --- storage/engine_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/engine_test.go b/storage/engine_test.go index f4a6ceb4b9..aa27b500a1 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -143,6 +143,6 @@ func Test_engine_sqlDatabase(t *testing.T) { require.NoError(t, row.Err()) var count int assert.NoError(t, row.Scan(&count)) - assert.Equal(t, 1, count) + assert.Equal(t, 2, count) }) } From 4c06ce66e40824f1e3f59bbc3681d4f959fdfc20 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 1 Dec 2023 16:11:28 +0100 Subject: [PATCH 06/10] Postgres: add e2e test make sure DDL statements are compatible --- e2e-tests/storage/postgres/docker-compose.yml | 30 ++++++++++++++ e2e-tests/storage/postgres/nuts.yaml | 16 ++++++++ e2e-tests/storage/postgres/run-test.sh | 23 +++++++++++ e2e-tests/storage/run-tests.sh | 7 ++++ go.mod | 7 ++++ go.sum | 9 ++++ storage/engine.go | 41 +++++++++++++++++-- 7 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 e2e-tests/storage/postgres/docker-compose.yml create mode 100644 e2e-tests/storage/postgres/nuts.yaml create mode 100755 e2e-tests/storage/postgres/run-test.sh diff --git a/e2e-tests/storage/postgres/docker-compose.yml b/e2e-tests/storage/postgres/docker-compose.yml new file mode 100644 index 0000000000..81acbc4fef --- /dev/null +++ b/e2e-tests/storage/postgres/docker-compose.yml @@ -0,0 +1,30 @@ +version: "3.7" +services: + node: + image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:local}" + environment: + NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + volumes: + - "./nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../../tls-certs/nodeA-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" + - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" + ports: + - "1323:1323" + healthcheck: + interval: 1s # Make test run quicker by checking health status more often + depends_on: + db: + condition: service_healthy + db: + image: postgres:16-alpine + restart: always + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] # this makes sure the container only reports healthy it can be connected to + interval: 1s + timeout: 5s + retries: 20 diff --git a/e2e-tests/storage/postgres/nuts.yaml b/e2e-tests/storage/postgres/nuts.yaml new file mode 100644 index 0000000000..7ef8e253eb --- /dev/null +++ b/e2e-tests/storage/postgres/nuts.yaml @@ -0,0 +1,16 @@ +url: https://node +verbosity: debug +auth: + contractvalidators: + - dummy + irma: + autoupdateschemas: false +tls: + truststorefile: /opt/nuts/truststore.pem + certfile: /opt/nuts/certificate-and-key.pem + certkeyfile: /opt/nuts/certificate-and-key.pem +crypto: + storage: fs +storage: + sql: + connection: postgres:host=db user=postgres password=postgres dbname=postgres sslmode=disable TimeZone=Europe/Amsterdam diff --git a/e2e-tests/storage/postgres/run-test.sh b/e2e-tests/storage/postgres/run-test.sh new file mode 100755 index 0000000000..84d413ba41 --- /dev/null +++ b/e2e-tests/storage/postgres/run-test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source ../../util.sh + +echo "------------------------------------" +echo "Cleaning up running Docker containers and volumes, and key material..." +echo "------------------------------------" +docker compose stop +docker compose rm -f -v + +echo "------------------------------------" +echo "Starting Docker containers..." +echo "------------------------------------" +docker compose up --wait +if [ $? -ne 0 ]; then + echo "ERROR: node failed to start" + exitWithDockerLogs 1 +fi + +echo "------------------------------------" +echo "Stopping Docker containers..." +echo "------------------------------------" +docker compose stop diff --git a/e2e-tests/storage/run-tests.sh b/e2e-tests/storage/run-tests.sh index 1d66b81d07..87388e9257 100755 --- a/e2e-tests/storage/run-tests.sh +++ b/e2e-tests/storage/run-tests.sh @@ -9,6 +9,13 @@ pushd redis ./run-test.sh popd +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +echo "!! Running test: Postgres !!" +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +pushd postgres +./run-test.sh +popd + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "!! Running test: Backup/Restore !!" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" diff --git a/go.mod b/go.mod index e8c5e2a68f..a479c0c853 100644 --- a/go.mod +++ b/go.mod @@ -176,3 +176,10 @@ require ( gorm.io/gorm v1.25.5 rsc.io/qr v0.2.0 // indirect ) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect + gorm.io/driver/postgres v1.5.4 // indirect +) diff --git a/go.sum b/go.sum index 2bd1eafc3b..81353a6f12 100644 --- a/go.sum +++ b/go.sum @@ -303,6 +303,13 @@ github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEF github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -863,6 +870,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= diff --git a/storage/engine.go b/storage/engine.go index cdefe022e2..e1031b5771 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -23,6 +23,7 @@ import ( "embed" "errors" "fmt" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "net/url" @@ -170,14 +171,48 @@ func (e *engine) initSQLDatabase() error { connectionString = sqliteConnectionString(e.datadir) } + // Find right SQL adapter + type sqlAdapter struct { + connector func(dsn string) gorm.Dialector + connectionString func(config string) string + } + adapters := map[string]sqlAdapter{ + "sqlite": { + connector: sqlite.Open, + connectionString: func(config string) string { + return fmt.Sprintf("sqlite:%s", config) + }, + }, + "postgres": { + connector: postgres.Open, + connectionString: func(config string) string { + return fmt.Sprintf("postgres:%s", config) + }, + }, + } + var adapter *sqlAdapter + for prefix, curr := range adapters { + parsedConnectionString := strings.TrimPrefix(connectionString, prefix+":") + if len(parsedConnectionString) != len(connectionString) { + adapter = &curr + connectionString = parsedConnectionString + break + } + } + + // Open connection and migrate var err error - e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{}) + e.sqlDB, err = gorm.Open(adapter.connector(connectionString), &gorm.Config{}) if err != nil { return err } log.Logger().Debug("Running database migrations...") - dbURL, _ := url.Parse(fmt.Sprintf("sqlite:%s", connectionString)) + // we need the connectionString with adapter specific prefix here + dbURL, err := url.Parse(adapter.connectionString(e.config.SQL.ConnectionString)) + if err != nil { + return err + } db := dbmate.New(dbURL) db.FS = sqlMigrationsFS db.MigrationsDir = []string{"sql_migrations"} @@ -188,7 +223,7 @@ func (e *engine) initSQLDatabase() error { } func sqliteConnectionString(datadir string) string { - return "file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on") + return "sqlite:file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on") } type provider struct { From 7c60d153d5c4362d0d499588a0b31ee0f6a9dbb2 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 1 Dec 2023 16:12:50 +0100 Subject: [PATCH 07/10] gomodtidy --- go.mod | 3 ++- go.sum | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a479c0c853..3ed728f767 100644 --- a/go.mod +++ b/go.mod @@ -177,9 +177,10 @@ require ( rsc.io/qr v0.2.0 // indirect ) +require gorm.io/driver/postgres v1.5.4 + require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.4.3 // indirect - gorm.io/driver/postgres v1.5.4 // indirect ) diff --git a/go.sum b/go.sum index 81353a6f12..2afa3f95b3 100644 --- a/go.sum +++ b/go.sum @@ -307,7 +307,6 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= From f9c50d8c02dc48ae8e0f319adbedf3e51a780531 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Sat, 2 Dec 2023 07:24:54 +0100 Subject: [PATCH 08/10] masterimg --- e2e-tests/storage/postgres/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/storage/postgres/docker-compose.yml b/e2e-tests/storage/postgres/docker-compose.yml index 81acbc4fef..85ebb6ff72 100644 --- a/e2e-tests/storage/postgres/docker-compose.yml +++ b/e2e-tests/storage/postgres/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.7" services: node: - image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:local}" + image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}" environment: NUTS_CONFIGFILE: /opt/nuts/nuts.yaml volumes: From 6c62ce77ab63d39bff2a3076afc7fb74d54cd2fe Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 6 Dec 2023 10:56:06 +0100 Subject: [PATCH 09/10] connectionString fix in engine --- storage/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/engine.go b/storage/engine.go index e1031b5771..9f0bc5f729 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -209,7 +209,7 @@ func (e *engine) initSQLDatabase() error { log.Logger().Debug("Running database migrations...") // we need the connectionString with adapter specific prefix here - dbURL, err := url.Parse(adapter.connectionString(e.config.SQL.ConnectionString)) + dbURL, err := url.Parse(adapter.connectionString(connectionString)) if err != nil { return err } From 8e86d0a4f0c2e1dacc516950b1c6cb23449ea3a8 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 6 Dec 2023 12:04:16 +0100 Subject: [PATCH 10/10] fix postgres e2e test --- e2e-tests/storage/postgres/nuts.yaml | 2 +- storage/engine.go | 30 ++++++++++++++++----------- storage/sql_migrations/002_didweb.sql | 2 +- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/e2e-tests/storage/postgres/nuts.yaml b/e2e-tests/storage/postgres/nuts.yaml index 7ef8e253eb..bba4d11a02 100644 --- a/e2e-tests/storage/postgres/nuts.yaml +++ b/e2e-tests/storage/postgres/nuts.yaml @@ -13,4 +13,4 @@ crypto: storage: fs storage: sql: - connection: postgres:host=db user=postgres password=postgres dbname=postgres sslmode=disable TimeZone=Europe/Amsterdam + connection: postgres://postgres:postgres@db:5432/postgres?sslmode=disable diff --git a/storage/engine.go b/storage/engine.go index 9f0bc5f729..30a28f3c5a 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -173,43 +173,46 @@ func (e *engine) initSQLDatabase() error { // Find right SQL adapter type sqlAdapter struct { - connector func(dsn string) gorm.Dialector - connectionString func(config string) string + connector func(dsn string) gorm.Dialector + gormConnectionString func(config string) string } adapters := map[string]sqlAdapter{ "sqlite": { connector: sqlite.Open, - connectionString: func(config string) string { - return fmt.Sprintf("sqlite:%s", config) + gormConnectionString: func(trimmed string) string { + return trimmed }, }, "postgres": { connector: postgres.Open, - connectionString: func(config string) string { - return fmt.Sprintf("postgres:%s", config) + gormConnectionString: func(trimmed string) string { + return fmt.Sprintf("postgres:%s", trimmed) }, }, } var adapter *sqlAdapter + var trimmedConnectionString string for prefix, curr := range adapters { - parsedConnectionString := strings.TrimPrefix(connectionString, prefix+":") - if len(parsedConnectionString) != len(connectionString) { + trimmedConnectionString = strings.TrimPrefix(connectionString, prefix+":") + if len(trimmedConnectionString) != len(connectionString) { adapter = &curr - connectionString = parsedConnectionString break } } + if adapter == nil { + return fmt.Errorf("unsupported SQL database connection: %s", connectionString) + } // Open connection and migrate var err error - e.sqlDB, err = gorm.Open(adapter.connector(connectionString), &gorm.Config{}) + e.sqlDB, err = gorm.Open(adapter.connector(adapter.gormConnectionString(trimmedConnectionString)), &gorm.Config{}) if err != nil { return err } log.Logger().Debug("Running database migrations...") // we need the connectionString with adapter specific prefix here - dbURL, err := url.Parse(adapter.connectionString(connectionString)) + dbURL, err := url.Parse(connectionString) if err != nil { return err } @@ -219,7 +222,10 @@ func (e *engine) initSQLDatabase() error { db.AutoDumpSchema = false db.Log = sqlMigrationLogger{} - return db.CreateAndMigrate() + if err = db.CreateAndMigrate(); err != nil { + return fmt.Errorf("failed to migrate database: %w on %s", err, dbURL.String()) + } + return nil } func sqliteConnectionString(datadir string) string { diff --git a/storage/sql_migrations/002_didweb.sql b/storage/sql_migrations/002_didweb.sql index 42584367a5..44d70a47cd 100644 --- a/storage/sql_migrations/002_didweb.sql +++ b/storage/sql_migrations/002_didweb.sql @@ -16,7 +16,7 @@ create table vdr_didweb_verificationmethod did varchar(255) not null, -- data is a JSON object containing the verification method data, e.g. the public key. -- When producing the verificationMethod, data is used as JSON base object and the id and type are added. - data blob not null, + data text not null, primary key (did, id), foreign key (did) references vdr_didweb (did) on delete cascade );