Postgres support (#2638)

e2e-tests/storage/postgres/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.7"
+services:
+  node:
+    image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+    volumes:
+      - "./nuts.yaml:/opt/nuts/nuts.yaml:ro"
+      - "../../tls-certs/nodeA-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    ports:
+      - "1323:1323"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often
+    depends_on:
+      db:
+        condition: service_healthy
+  db:
+    image: postgres:16-alpine
+    restart: always
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ] # this makes sure the container only reports healthy it can be connected to
+      interval: 1s
+      timeout: 5s
+      retries: 20

e2e-tests/storage/postgres/nuts.yaml

@@ -0,0 +1,16 @@
+url: https://node
+verbosity: debug
+auth:
+  contractvalidators:
+    - dummy
+  irma:
+    autoupdateschemas: false
+tls:
+  truststorefile: /opt/nuts/truststore.pem
+  certfile: /opt/nuts/certificate-and-key.pem
+  certkeyfile: /opt/nuts/certificate-and-key.pem
+crypto:
+  storage: fs
+storage:
+  sql:
+    connection: postgres://postgres:postgres@db:5432/postgres?sslmode=disable

e2e-tests/storage/postgres/run-test.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+source ../../util.sh
+
+echo "------------------------------------"
+echo "Cleaning up running Docker containers and volumes, and key material..."
+echo "------------------------------------"
+docker compose stop
+docker compose rm -f -v
+
+echo "------------------------------------"
+echo "Starting Docker containers..."
+echo "------------------------------------"
+docker compose up --wait
+if [ $? -ne 0 ]; then
+	echo "ERROR: node failed to start"
+	exitWithDockerLogs 1
+fi
+
+echo "------------------------------------"
+echo "Stopping Docker containers..."
+echo "------------------------------------"
+docker compose stop

e2e-tests/storage/run-tests.sh

@@ -9,6 +9,13 @@ pushd redis
 ./run-test.sh
 popd
 
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+echo "!! Running test: Postgres         !!"
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+pushd postgres
+./run-test.sh
+popd
+
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 echo "!! Running test: Backup/Restore   !!"
 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"

go.mod

@@ -176,3 +176,11 @@ require (
 	gorm.io/gorm v1.25.5
 	rsc.io/qr v0.2.0 // indirect
 )
+
+require gorm.io/driver/postgres v1.5.4
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.4.3 // indirect
+)

go.sum

@@ -303,6 +303,12 @@ github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEF
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
+github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
 github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
 github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -863,6 +869,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo=
+gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0=
 gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=
 gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=

storage/engine.go

@@ -23,6 +23,7 @@ import (
 	"embed"
 	"errors"
 	"fmt"
+	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"net/url"
@@ -32,13 +33,14 @@ import (
 	"time"
 
 	"github.com/amacneil/dbmate/v2/pkg/dbmate"
-	_ "github.com/amacneil/dbmate/v2/pkg/driver/mysql"
-	_ "github.com/amacneil/dbmate/v2/pkg/driver/postgres"
-	_ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite"
 	"github.com/nuts-foundation/go-stoabs"
 	"github.com/nuts-foundation/nuts-node/core"
 	"github.com/nuts-foundation/nuts-node/storage/log"
 	"github.com/redis/go-redis/v9"
+
+	_ "github.com/amacneil/dbmate/v2/pkg/driver/mysql"
+	_ "github.com/amacneil/dbmate/v2/pkg/driver/postgres"
+	_ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite"
 )
 
 const storeShutdownTimeout = 5 * time.Second
@@ -169,25 +171,65 @@ func (e *engine) initSQLDatabase() error {
 		connectionString = sqliteConnectionString(e.datadir)
 	}
 
+	// Find right SQL adapter
+	type sqlAdapter struct {
+		connector            func(dsn string) gorm.Dialector
+		gormConnectionString func(config string) string
+	}
+	adapters := map[string]sqlAdapter{
+		"sqlite": {
+			connector: sqlite.Open,
+			gormConnectionString: func(trimmed string) string {
+				return trimmed
+			},
+		},
+		"postgres": {
+			connector: postgres.Open,
+			gormConnectionString: func(trimmed string) string {
+				return fmt.Sprintf("postgres:%s", trimmed)
+			},
+		},
+	}
+	var adapter *sqlAdapter
+	var trimmedConnectionString string
+	for prefix, curr := range adapters {
+		trimmedConnectionString = strings.TrimPrefix(connectionString, prefix+":")
+		if len(trimmedConnectionString) != len(connectionString) {
+			adapter = &curr
+			break
+		}
+	}
+	if adapter == nil {
+		return fmt.Errorf("unsupported SQL database connection: %s", connectionString)
+	}
+
+	// Open connection and migrate
 	var err error
-	e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{})
+	e.sqlDB, err = gorm.Open(adapter.connector(adapter.gormConnectionString(trimmedConnectionString)), &gorm.Config{})
 	if err != nil {
 		return err
 	}
 	log.Logger().Debug("Running database migrations...")
 
-	dbURL, _ := url.Parse(fmt.Sprintf("sqlite:%s", connectionString))
+	// we need the connectionString with adapter specific prefix here
+	dbURL, err := url.Parse(connectionString)
+	if err != nil {
+		return err
+	}
 	db := dbmate.New(dbURL)
 	db.FS = sqlMigrationsFS
 	db.MigrationsDir = []string{"sql_migrations"}
 	db.AutoDumpSchema = false
 	db.Log = sqlMigrationLogger{}
 
-	return db.CreateAndMigrate()
+	if err = db.CreateAndMigrate(); err != nil {
+		return fmt.Errorf("failed to migrate database: %w on %s", err, dbURL.String())
+	}
+	return nil
 }
 
 func sqliteConnectionString(datadir string) string {
-	return "file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on")
+	return "sqlite:file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on")
 }
 
 type provider struct {

storage/sql_migrations/002_didweb.sql

@@ -16,7 +16,7 @@ create table vdr_didweb_verificationmethod
     did  varchar(255) not null,
     -- data is a JSON object containing the verification method data, e.g. the public key.
     -- When producing the verificationMethod, data is used as JSON base object and the id and type are added.
-    data blob         not null,
+    data text         not null,
     primary key (did, id),
     foreign key (did) references vdr_didweb (did) on delete cascade
 );