diff --git a/e2e-tests/storage/postgres/docker-compose.yml b/e2e-tests/storage/postgres/docker-compose.yml new file mode 100644 index 0000000000..85ebb6ff72 --- /dev/null +++ b/e2e-tests/storage/postgres/docker-compose.yml @@ -0,0 +1,30 @@ +version: "3.7" +services: + node: + image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}" + environment: + NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + volumes: + - "./nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../../tls-certs/nodeA-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" + - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" + ports: + - "1323:1323" + healthcheck: + interval: 1s # Make test run quicker by checking health status more often + depends_on: + db: + condition: service_healthy + db: + image: postgres:16-alpine + restart: always + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] # this makes sure the container only reports healthy it can be connected to + interval: 1s + timeout: 5s + retries: 20 diff --git a/e2e-tests/storage/postgres/nuts.yaml b/e2e-tests/storage/postgres/nuts.yaml new file mode 100644 index 0000000000..bba4d11a02 --- /dev/null +++ b/e2e-tests/storage/postgres/nuts.yaml @@ -0,0 +1,16 @@ +url: https://node +verbosity: debug +auth: + contractvalidators: + - dummy + irma: + autoupdateschemas: false +tls: + truststorefile: /opt/nuts/truststore.pem + certfile: /opt/nuts/certificate-and-key.pem + certkeyfile: /opt/nuts/certificate-and-key.pem +crypto: + storage: fs +storage: + sql: + connection: postgres://postgres:postgres@db:5432/postgres?sslmode=disable diff --git a/e2e-tests/storage/postgres/run-test.sh b/e2e-tests/storage/postgres/run-test.sh new file mode 100755 index 0000000000..84d413ba41 --- /dev/null +++ b/e2e-tests/storage/postgres/run-test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source ../../util.sh + +echo "------------------------------------" +echo "Cleaning up running Docker containers and volumes, and key material..." +echo "------------------------------------" +docker compose stop +docker compose rm -f -v + +echo "------------------------------------" +echo "Starting Docker containers..." +echo "------------------------------------" +docker compose up --wait +if [ $? -ne 0 ]; then + echo "ERROR: node failed to start" + exitWithDockerLogs 1 +fi + +echo "------------------------------------" +echo "Stopping Docker containers..." +echo "------------------------------------" +docker compose stop diff --git a/e2e-tests/storage/run-tests.sh b/e2e-tests/storage/run-tests.sh index 1d66b81d07..87388e9257 100755 --- a/e2e-tests/storage/run-tests.sh +++ b/e2e-tests/storage/run-tests.sh @@ -9,6 +9,13 @@ pushd redis ./run-test.sh popd +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +echo "!! Running test: Postgres !!" +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +pushd postgres +./run-test.sh +popd + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "!! Running test: Backup/Restore !!" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" diff --git a/go.mod b/go.mod index e8c5e2a68f..3ed728f767 100644 --- a/go.mod +++ b/go.mod @@ -176,3 +176,11 @@ require ( gorm.io/gorm v1.25.5 rsc.io/qr v0.2.0 // indirect ) + +require gorm.io/driver/postgres v1.5.4 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect +) diff --git a/go.sum b/go.sum index 2bd1eafc3b..2afa3f95b3 100644 --- a/go.sum +++ b/go.sum @@ -303,6 +303,12 @@ github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEF github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -863,6 +869,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= diff --git a/storage/engine.go b/storage/engine.go index 2d91ce1141..30a28f3c5a 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -23,6 +23,7 @@ import ( "embed" "errors" "fmt" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "net/url" @@ -32,13 +33,14 @@ import ( "time" "github.com/amacneil/dbmate/v2/pkg/dbmate" - _ "github.com/amacneil/dbmate/v2/pkg/driver/mysql" - _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" - _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage/log" "github.com/redis/go-redis/v9" + + _ "github.com/amacneil/dbmate/v2/pkg/driver/mysql" + _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" + _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" ) const storeShutdownTimeout = 5 * time.Second @@ -169,25 +171,65 @@ func (e *engine) initSQLDatabase() error { connectionString = sqliteConnectionString(e.datadir) } + // Find right SQL adapter + type sqlAdapter struct { + connector func(dsn string) gorm.Dialector + gormConnectionString func(config string) string + } + adapters := map[string]sqlAdapter{ + "sqlite": { + connector: sqlite.Open, + gormConnectionString: func(trimmed string) string { + return trimmed + }, + }, + "postgres": { + connector: postgres.Open, + gormConnectionString: func(trimmed string) string { + return fmt.Sprintf("postgres:%s", trimmed) + }, + }, + } + var adapter *sqlAdapter + var trimmedConnectionString string + for prefix, curr := range adapters { + trimmedConnectionString = strings.TrimPrefix(connectionString, prefix+":") + if len(trimmedConnectionString) != len(connectionString) { + adapter = &curr + break + } + } + if adapter == nil { + return fmt.Errorf("unsupported SQL database connection: %s", connectionString) + } + + // Open connection and migrate var err error - e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{}) + e.sqlDB, err = gorm.Open(adapter.connector(adapter.gormConnectionString(trimmedConnectionString)), &gorm.Config{}) if err != nil { return err } log.Logger().Debug("Running database migrations...") - dbURL, _ := url.Parse(fmt.Sprintf("sqlite:%s", connectionString)) + // we need the connectionString with adapter specific prefix here + dbURL, err := url.Parse(connectionString) + if err != nil { + return err + } db := dbmate.New(dbURL) db.FS = sqlMigrationsFS db.MigrationsDir = []string{"sql_migrations"} db.AutoDumpSchema = false db.Log = sqlMigrationLogger{} - return db.CreateAndMigrate() + if err = db.CreateAndMigrate(); err != nil { + return fmt.Errorf("failed to migrate database: %w on %s", err, dbURL.String()) + } + return nil } func sqliteConnectionString(datadir string) string { - return "file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on") + return "sqlite:file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on") } type provider struct { diff --git a/storage/sql_migrations/002_didweb.sql b/storage/sql_migrations/002_didweb.sql index 42584367a5..44d70a47cd 100644 --- a/storage/sql_migrations/002_didweb.sql +++ b/storage/sql_migrations/002_didweb.sql @@ -16,7 +16,7 @@ create table vdr_didweb_verificationmethod did varchar(255) not null, -- data is a JSON object containing the verification method data, e.g. the public key. -- When producing the verificationMethod, data is used as JSON base object and the id and type are added. - data blob not null, + data text not null, primary key (did, id), foreign key (did) references vdr_didweb (did) on delete cascade );