[VNEXT] feat: Multi-DB type support

.github/FUNDING.yml

@@ -1 +1 @@
-github: [tankerkiller125,katosdev]
+github: [tankerkiller125,katosdev,tonyaellie]

.github/workflows/docker-publish-arm.yaml

@@ -11,7 +11,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
@@ -22,7 +22,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/docker-publish-rootless-arm.yaml

@@ -11,7 +11,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
@@ -22,7 +22,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/docker-publish-rootless.yaml

@@ -11,7 +11,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
@@ -22,7 +22,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
 
 
 env:

.github/workflows/docker-publish.yaml

@@ -11,7 +11,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
@@ -22,7 +22,7 @@ on:
       - 'Dockerfile'
       - 'Dockerfile.rootless'
       - '.dockerignore'
-      - '.github/workflows'
+      - '.github/workflows/**'
 
 env:
   # Use docker.io for Docker Hub if empty

.github/workflows/pull-requests.yaml

@@ -8,6 +8,7 @@ on:
     paths:
       - 'backend/**'
       - 'frontend/**'
+      - '.github/workflows/**'
 
 jobs:
   backend-tests:

.gitignore

@@ -2,6 +2,9 @@
 backend/.data/*
 config.yml
 homebox.db
+homebox.db-journal
+homebox.db-shm
+homebox.db-wal
 .idea
 .DS_Store
 test-mailer.json

Taskfile.yml

@@ -2,7 +2,8 @@ version: "3"
 
 env:
   HBOX_LOG_LEVEL: debug
-  HBOX_STORAGE_SQLITE_URL: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1
+  HBOX_DATABASE_DRIVER: sqlite3
+  HBOX_DATABASE_SQLITE_PATH: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1
   HBOX_OPTIONS_ALLOW_REGISTRATION: true
   UNSAFE_DISABLE_PASSWORD_PROJECTION: "yes_i_am_sure"
 tasks:
@@ -60,6 +61,23 @@ tasks:
       - go run ./app/api/ {{ .CLI_ARGS }}
     silent: false
 
+  go:run:postgresql:
+    env:
+        HBOX_DATABASE_DRIVER: postgres
+        HBOX_DATABASE_USERNAME: homebox
+        HBOX_DATABASE_PASSWORD: homebox
+        HBOX_DATABASE_DATABASE: homebox
+        HBOX_DATABASE_HOST: localhost
+        HBOX_DATABASE_PORT: 5432
+        HBOX_DATABASE_SSLMODE: disable
+    desc: Starts the backend api server with postgresql (depends on generate task)
+    dir: backend
+    deps:
+      - generate
+    cmds:
+      - go run ./app/api/ {{ .CLI_ARGS }}
+    silent: false
+
   go:test:
     desc: Runs all go tests using gotestsum - supports passing gotestsum args
     dir: backend
@@ -114,6 +132,21 @@ tasks:
     cmds:
       - cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}
 
+  db:migration:postgresql:
+    env:
+        HBOX_DATABASE_DRIVER: postgres
+        HBOX_DATABASE_USERNAME: homebox
+        HBOX_DATABASE_PASSWORD: homebox
+        HBOX_DATABASE_DATABASE: homebox
+        HBOX_DATABASE_HOST: localhost
+        HBOX_DATABASE_PORT: 5432
+        HBOX_DATABASE_SSLMODE: disable
+    desc: Runs the database diff engine to generate a SQL migration files for postgresql
+    deps:
+      - db:generate
+    cmds:
+      - cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}
+
   ui:watch:
     desc: Starts the vitest test runner in watch mode
     dir: frontend
@@ -147,6 +180,23 @@ tasks:
       - cd frontend && pnpm run test:ci
     silent: true
 
+  test:ci:postgresql:
+    env:
+      HBOX_DATABASE_DRIVER: postgres
+      HBOX_DATABASE_USERNAME: homebox
+      HBOX_DATABASE_PASSWORD: homebox
+      HBOX_DATABASE_DATABASE: homebox
+      HBOX_DATABASE_HOST: localhost
+      HBOX_DATABASE_PORT: 5432
+      HBOX_DATABASE_SSLMODE: disable
+    desc: Runs end-to-end test on a live server with postgresql (only for use in CI)
+    cmds:
+      - cd backend && go build ./app/api
+      - backend/api &
+      - sleep 5
+      - cd frontend && pnpm run test:ci
+    silent: true
+
   pr:
     desc: Runs all tasks required for a PR
     cmds:

backend/app/api/main.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/google/uuid"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	atlas "ariga.io/atlas/sql/migrate"
@@ -28,6 +30,7 @@ import (
 	"github.com/sysadminsmedia/homebox/backend/internal/sys/config"
 	"github.com/sysadminsmedia/homebox/backend/internal/web/mid"
 
+	_ "github.com/lib/pq"
 	_ "github.com/sysadminsmedia/homebox/backend/pkgs/cgofreesqlite"
 )
 
@@ -46,6 +49,19 @@ func build() string {
 	return fmt.Sprintf("%s, commit %s, built at %s", version, short, buildTime)
 }
 
+func validatePostgresSSLMode(sslMode string) bool {
+	validModes := map[string]bool{
+		"":            true,
+		"disable":     true,
+		"allow":       true,
+		"prefer":      true,
+		"require":     true,
+		"verify-ca":   true,
+		"verify-full": true,
+	}
+	return validModes[strings.ToLower(strings.TrimSpace(sslMode))]
+}
+
 // @title                      Homebox API
 // @version                    1.0
 // @description                Track, Manage, and Organize your Things.
@@ -80,13 +96,32 @@ func run(cfg *config.Config) error {
 		log.Fatal().Err(err).Msg("failed to create data directory")
 	}
 
-	c, err := ent.Open("sqlite3", cfg.Storage.SqliteURL)
+	if strings.ToLower(cfg.Database.Driver) == "postgres" {
+		if !validatePostgresSSLMode(cfg.Database.SslMode) {
+			log.Fatal().Str("sslmode", cfg.Database.SslMode).Msg("invalid sslmode")
+		}
+	}
+
+	// Set up the database URL based on the driver because for some reason a common URL format is not used
+	databaseURL := ""
+	switch strings.ToLower(cfg.Database.Driver) {
+	case "sqlite3":
+		databaseURL = cfg.Database.SqlitePath
+	case "postgres":
+		databaseURL = fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", cfg.Database.Host, cfg.Database.Port, cfg.Database.Username, cfg.Database.Password, cfg.Database.Database, cfg.Database.SslMode)
+	default:
+		log.Fatal().Str("driver", cfg.Database.Driver).Msg("unsupported database driver {driver}")
+	}
+
+	c, err := ent.Open(strings.ToLower(cfg.Database.Driver), databaseURL)
 	if err != nil {
 		log.Fatal().
 			Err(err).
-			Str("driver", "sqlite").
-			Str("url", cfg.Storage.SqliteURL).
-			Msg("failed opening connection to sqlite")
+			Str("driver", strings.ToLower(cfg.Database.Driver)).
+			Str("host", cfg.Database.Host).
+			Str("port", cfg.Database.Port).
+			Str("database", cfg.Database.Database).
+			Msg("failed opening connection to {driver} database at {host}:{port}/{database}")
 	}
 	defer func(c *ent.Client) {
 		err := c.Close()
@@ -95,9 +130,14 @@ func run(cfg *config.Config) error {
 		}
 	}(c)
 
-	temp := filepath.Join(os.TempDir(), "migrations")
+	// Always create a random temporary directory for migrations
+	tempUuid, err := uuid.NewUUID()
+	if err != nil {
+		return err
+	}
+	temp := filepath.Join(os.TempDir(), "homebox-", tempUuid.String())
 
-	err = migrations.Write(temp)
+	err = migrations.Write(temp, cfg.Database.Driver)
 	if err != nil {
 		return err
 	}
@@ -117,17 +157,18 @@ func run(cfg *config.Config) error {
 	if err != nil {
 		log.Error().
 			Err(err).
-			Str("driver", "sqlite").
-			Str("url", cfg.Storage.SqliteURL).
+			Str("driver", cfg.Database.Driver).
+			Str("url", databaseURL).
 			Msg("failed creating schema resources")
 		return err
 	}
 
-	err = os.RemoveAll(temp)
-	if err != nil {
-		log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
-		return err
-	}
+	defer func() {
+		err := os.RemoveAll(temp)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
+		}
+	}()
 
 	collectFuncs := []currencies.CollectorFunc{
 		currencies.CollectDefaults(),