Skip to content

Commit

Permalink
[VNEXT] feat: Multi-DB type support (#291)
Browse files Browse the repository at this point in the history
* feat: Multi-DB type URL formats and config

* fix: remove legacy sqlite path config and minor other things

* fix: dumb eslint issues

* fix: dumb eslint issues

* fix: application can be tested with sqlite

* fix: minor config formatting

* chore: some cleanup

* feat: postgres migration creation now works

The migration creation for postgres now works properly.
Removed MySQL support, having too many issues with it at this time.

* chore: revert some strings back to bytes as they should be

* feat: improve languages support

* feat: add locale time ago formatting and the local name for the language in language dropdown

* Update FUNDING.yml

* chore: remove some more mysql stuff

* fix: coderabbit security recommendations

* fix: validate postgres sslmode

* Update migrations.go

* fix: postgres migration creation now works

* fix: errors in raw sql queries

* fix: lint error, and simpler SQL query

* fix: migrations directory string

* fix: stats related test

* fix: sql query

* Update TextArea.vue

* Update TextField.vue

* chore: run integration testing on multiple postgresql versions

* chore: jobs should run for vnext branch PRs

* fix: missed $ for Postgres testing

* fix: environment variable for db ssl mode

* fix: lint issue from a merge

* chore: trying to fix postgresql testing

* chore: trying to fix postgresql testing

* fix: trying to fix postgresql testing

* fix: trying to fix postgresql testing

---------

Co-authored-by: tonya <[email protected]>
  • Loading branch information
tankerkiller125 and tonyaellie committed Nov 14, 2024
1 parent b5c7566 commit 79ec169
Show file tree
Hide file tree
Showing 36 changed files with 387 additions and 107 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/docker-publish-arm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ on:
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ "main" ]
branches: [ "main", "vnext" ]
paths:
- 'backend/**'
- 'frontend/**'
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'

env:
# Use docker.io for Docker Hub if empty
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/docker-publish-rootless-arm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ on:
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ "main" ]
branches: [ "main", "vnext" ]
paths:
- 'backend/**'
- 'frontend/**'
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'

env:
# Use docker.io for Docker Hub if empty
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/docker-publish-rootless.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ on:
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ "main" ]
branches: [ "main", "vnext" ]
paths:
- 'backend/**'
- 'frontend/**'
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'


env:
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/docker-publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ on:
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ "main" ]
branches: [ "main", "vnext" ]
paths:
- 'backend/**'
- 'frontend/**'
- 'Dockerfile'
- 'Dockerfile.rootless'
- '.dockerignore'
- '.github/workflows'
- '.github/workflows/**'

env:
# Use docker.io for Docker Hub if empty
Expand Down
64 changes: 64 additions & 0 deletions .github/workflows/partial-frontend.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,20 @@ jobs:
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
services:
postgres:
image: postgres:17
env:
POSTGRES_USER: homebox
POSTGRES_PASSWORD: homebox
POSTGRES_DB: homebox
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -62,3 +76,53 @@ jobs:

- name: Run Integration Tests
run: task test:ci
integration-tests-pgsql:
strategy:
matrix:
database_version: [17,16,15]
name: Integration Tests PGSQL ${{ matrix.database_version }}
runs-on: ubuntu-latest
services:
postgres:
image: postgres:${{ matrix.database_version }}
env:
POSTGRES_USER: homebox
POSTGRES_PASSWORD: homebox
POSTGRES_DB: homebox
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Install Task
uses: arduino/setup-task@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.21"

- uses: actions/setup-node@v4
with:
node-version: 18

- uses: pnpm/[email protected]
with:
version: 9.12.2

- name: Install dependencies
run: pnpm install
working-directory: frontend

- name: Run Integration Tests
run: task test:ci:postgresql
2 changes: 2 additions & 0 deletions .github/workflows/pull-requests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ on:
pull_request:
branches:
- main
- vnext

paths:
- 'backend/**'
- 'frontend/**'
- '.github/workflows/**'

jobs:
backend-tests:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
backend/.data/*
config.yml
homebox.db
homebox.db-journal
homebox.db-shm
homebox.db-wal
.idea
.DS_Store
test-mailer.json
Expand Down
54 changes: 52 additions & 2 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ version: "3"

env:
HBOX_LOG_LEVEL: debug
HBOX_STORAGE_SQLITE_URL: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1
HBOX_DATABASE_DRIVER: sqlite3
HBOX_DATABASE_SQLITE_PATH: .data/homebox.db?_pragma=busy_timeout=1000&_pragma=journal_mode=WAL&_fk=1
HBOX_OPTIONS_ALLOW_REGISTRATION: true
UNSAFE_DISABLE_PASSWORD_PROJECTION: "yes_i_am_sure"
tasks:
Expand Down Expand Up @@ -60,6 +61,23 @@ tasks:
- go run ./app/api/ {{ .CLI_ARGS }}
silent: false

go:run:postgresql:
env:
HBOX_DATABASE_DRIVER: postgres
HBOX_DATABASE_USERNAME: homebox
HBOX_DATABASE_PASSWORD: homebox
HBOX_DATABASE_DATABASE: homebox
HBOX_DATABASE_HOST: localhost
HBOX_DATABASE_PORT: 5432
HBOX_DATABASE_SSL_MODE: disable
desc: Starts the backend api server with postgresql (depends on generate task)
dir: backend
deps:
- generate
cmds:
- go run ./app/api/ {{ .CLI_ARGS }}
silent: false

go:test:
desc: Runs all go tests using gotestsum - supports passing gotestsum args
dir: backend
Expand Down Expand Up @@ -114,6 +132,21 @@ tasks:
cmds:
- cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}

db:migration:postgresql:
env:
HBOX_DATABASE_DRIVER: postgres
HBOX_DATABASE_USERNAME: homebox
HBOX_DATABASE_PASSWORD: homebox
HBOX_DATABASE_DATABASE: homebox
HBOX_DATABASE_HOST: localhost
HBOX_DATABASE_PORT: 5432
HBOX_DATABASE_SSL_MODE: disable
desc: Runs the database diff engine to generate a SQL migration files for postgresql
deps:
- db:generate
cmds:
- cd backend && go run app/tools/migrations/main.go {{ .CLI_ARGS }}

ui:watch:
desc: Starts the vitest test runner in watch mode
dir: frontend
Expand Down Expand Up @@ -143,7 +176,24 @@ tasks:
cmds:
- cd backend && go build ./app/api
- backend/api &
- sleep 5
- sleep 10
- cd frontend && pnpm run test:ci
silent: true

test:ci:postgresql:
env:
HBOX_DATABASE_DRIVER: postgres
HBOX_DATABASE_USERNAME: homebox
HBOX_DATABASE_PASSWORD: homebox
HBOX_DATABASE_DATABASE: homebox
HBOX_DATABASE_HOST: 127.0.0.1
HBOX_DATABASE_PORT: 5432
HBOX_DATABASE_SSL_MODE: disable
desc: Runs end-to-end test on a live server with postgresql (only for use in CI)
cmds:
- cd backend && go build ./app/api
- backend/api &
- sleep 10
- cd frontend && pnpm run test:ci
silent: true

Expand Down
67 changes: 54 additions & 13 deletions backend/app/api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"bytes"
"context"
"fmt"
"github.com/google/uuid"
"net/http"
"os"
"path/filepath"
"strings"
"time"

atlas "ariga.io/atlas/sql/migrate"
Expand All @@ -28,6 +30,7 @@ import (
"github.com/sysadminsmedia/homebox/backend/internal/sys/config"
"github.com/sysadminsmedia/homebox/backend/internal/web/mid"

_ "github.com/lib/pq"
_ "github.com/sysadminsmedia/homebox/backend/pkgs/cgofreesqlite"
)

Expand All @@ -46,6 +49,19 @@ func build() string {
return fmt.Sprintf("%s, commit %s, built at %s", version, short, buildTime)
}

func validatePostgresSSLMode(sslMode string) bool {
validModes := map[string]bool{
"": true,
"disable": true,
"allow": true,
"prefer": true,
"require": true,
"verify-ca": true,
"verify-full": true,
}
return validModes[strings.ToLower(strings.TrimSpace(sslMode))]
}

// @title Homebox API
// @version 1.0
// @description Track, Manage, and Organize your Things.
Expand Down Expand Up @@ -80,13 +96,32 @@ func run(cfg *config.Config) error {
log.Fatal().Err(err).Msg("failed to create data directory")
}

c, err := ent.Open("sqlite3", cfg.Storage.SqliteURL)
if strings.ToLower(cfg.Database.Driver) == "postgres" {
if !validatePostgresSSLMode(cfg.Database.SslMode) {
log.Fatal().Str("sslmode", cfg.Database.SslMode).Msg("invalid sslmode")
}
}

// Set up the database URL based on the driver because for some reason a common URL format is not used
databaseURL := ""
switch strings.ToLower(cfg.Database.Driver) {
case "sqlite3":
databaseURL = cfg.Database.SqlitePath
case "postgres":
databaseURL = fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", cfg.Database.Host, cfg.Database.Port, cfg.Database.Username, cfg.Database.Password, cfg.Database.Database, cfg.Database.SslMode)
default:
log.Fatal().Str("driver", cfg.Database.Driver).Msg("unsupported database driver")
}

c, err := ent.Open(strings.ToLower(cfg.Database.Driver), databaseURL)
if err != nil {
log.Fatal().
Err(err).
Str("driver", "sqlite").
Str("url", cfg.Storage.SqliteURL).
Msg("failed opening connection to sqlite")
Str("driver", strings.ToLower(cfg.Database.Driver)).
Str("host", cfg.Database.Host).
Str("port", cfg.Database.Port).
Str("database", cfg.Database.Database).
Msg("failed opening connection to {driver} database at {host}:{port}/{database}")
}
defer func(c *ent.Client) {
err := c.Close()
Expand All @@ -95,9 +130,14 @@ func run(cfg *config.Config) error {
}
}(c)

temp := filepath.Join(os.TempDir(), "migrations")
// Always create a random temporary directory for migrations
tempUUID, err := uuid.NewUUID()
if err != nil {
return err
}
temp := filepath.Join(os.TempDir(), fmt.Sprintf("homebox-%s", tempUUID.String()))

err = migrations.Write(temp)
err = migrations.Write(temp, cfg.Database.Driver)
if err != nil {
return err
}
Expand All @@ -117,17 +157,18 @@ func run(cfg *config.Config) error {
if err != nil {
log.Error().
Err(err).
Str("driver", "sqlite").
Str("url", cfg.Storage.SqliteURL).
Str("driver", cfg.Database.Driver).
Str("url", databaseURL).
Msg("failed creating schema resources")
return err
}

err = os.RemoveAll(temp)
if err != nil {
log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
return err
}
defer func() {
err := os.RemoveAll(temp)
if err != nil {
log.Error().Err(err).Msg("failed to remove temporary directory for database migrations")
}
}()

collectFuncs := []currencies.CollectorFunc{
currencies.CollectDefaults(),
Expand Down
Loading

0 comments on commit 79ec169

Please sign in to comment.