From 28d2bd82f904d5e2c5fa0c62e39e7d0df22d3e92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:04:33 +0000 Subject: [PATCH 1/5] Bump github.com/pressly/goose/v3 from 3.22.0 to 3.22.1 Bumps [github.com/pressly/goose/v3](https://github.com/pressly/goose) from 3.22.0 to 3.22.1. - [Release notes](https://github.com/pressly/goose/releases) - [Changelog](https://github.com/pressly/goose/blob/master/CHANGELOG.md) - [Commits](https://github.com/pressly/goose/compare/v3.22.0...v3.22.1) --- updated-dependencies: - dependency-name: github.com/pressly/goose/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3051f71bd..6f37a1875 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( // check the oapi-codegen tool version in the makefile when upgrading the runtime github.com/oapi-codegen/runtime v1.1.1 github.com/piprate/json-gold v0.5.1-0.20230111113000-6ddbe6e6f19f - github.com/pressly/goose/v3 v3.22.0 + github.com/pressly/goose/v3 v3.22.1 github.com/privacybydesign/irmago v0.16.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 @@ -123,9 +123,9 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -199,6 +199,6 @@ require ( modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.32.0 // indirect + modernc.org/sqlite v1.33.0 // indirect rsc.io/qr v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index 795b027c9..deb6af356 100644 --- a/go.sum +++ b/go.sum @@ -213,12 +213,12 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= @@ -378,8 +378,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= -github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= -github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= +github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuioLc= +github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= github.com/privacybydesign/gabi v0.0.0-20221212095008-68a086907750 h1:3RuYOQTlArQ6Uw2TgySusmZGluP+18WdQL56YSfkM3Q= github.com/privacybydesign/gabi v0.0.0-20221212095008-68a086907750/go.mod h1:QZI8hX8Ff2GfZ7UJuxyWw3nAGgt2s5+U4hxY6rmwQvs= github.com/privacybydesign/irmago v0.16.0 h1:PxIPRvpitxfJSocIRwIoYDSETarsopxtByZ5XZ3yzcE= @@ -634,8 +634,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= -modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= +modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From ccbba4044e7e45359803d1e129fe9c32acbe3f09 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 16 Oct 2024 09:11:56 +0200 Subject: [PATCH 2/5] update image for circlCI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a4e5c757..202fc79b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: build: parallelism: 8 docker: - - image: cimg/go:1.22 + - image: cimg/go:1.23 steps: - checkout @@ -36,7 +36,7 @@ jobs: report: docker: - - image: cimg/go:1.22 + - image: cimg/go:1.23 steps: - checkout - attach_workspace: From a26364fe07f95637fcd8ee13388245171afe39c3 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 16 Oct 2024 10:18:26 +0200 Subject: [PATCH 3/5] replace github.com/glebarez/sqlite with local code --- go.mod | 13 +- go.sum | 4 + storage/engine.go | 8 +- storage/sqlite/sqlite_ddlmod.go | 317 ++++++++++++++++++++ storage/sqlite/sqlite_dialector.go | 297 +++++++++++++++++++ storage/sqlite/sqlite_migrator.go | 454 +++++++++++++++++++++++++++++ 6 files changed, 1088 insertions(+), 5 deletions(-) create mode 100644 storage/sqlite/sqlite_ddlmod.go create mode 100644 storage/sqlite/sqlite_dialector.go create mode 100644 storage/sqlite/sqlite_migrator.go diff --git a/go.mod b/go.mod index 6f37a1875..f8490d7a6 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 + gorm.io/driver/sqlite v1.5.6 gorm.io/driver/sqlserver v1.5.3 schneider.vip/problem v1.9.1 ) @@ -94,7 +95,6 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor v1.5.1 // indirect - github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-co-op/gocron v1.28.3 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -199,6 +199,15 @@ require ( modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.33.0 // indirect + modernc.org/sqlite v1.33.0 rsc.io/qr v0.2.0 // indirect ) + +require ( + github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/go.sum b/go.sum index deb6af356..02322a3e8 100644 --- a/go.sum +++ b/go.sum @@ -288,6 +288,8 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mdp/qrterminal/v3 v3.2.0 h1:qteQMXO3oyTK4IHwj2mWsKYYRBOp1Pj2WRYFYYNTCdk= github.com/mdp/qrterminal/v3 v3.2.0/go.mod h1:XGGuua4Lefrl7TLEsSONiD+UEjQXJZ4mPzF+gWYIJkk= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= @@ -608,6 +610,8 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/driver/sqlserver v1.5.3 h1:rjupPS4PVw+rjJkfvr8jn2lJ8BMhT4UW5FwuJY0P3Z0= gorm.io/driver/sqlserver v1.5.3/go.mod h1:B+CZ0/7oFJ6tAlefsKoyxdgDCXJKSgwS2bMOQZT0I00= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= diff --git a/storage/engine.go b/storage/engine.go index c1834217e..6c3d1c978 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -29,7 +29,6 @@ import ( "sync" "time" - "github.com/glebarez/sqlite" _ "github.com/microsoft/go-mssqldb/azuread" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" @@ -40,8 +39,10 @@ import ( "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" "gorm.io/driver/sqlserver" "gorm.io/gorm" + _ "modernc.org/sqlite" ) const storeShutdownTimeout = 5 * time.Second @@ -255,8 +256,9 @@ func (e *engine) initSQLDatabase() error { // With 1 connection, all actions will be performed sequentially. This impacts performance, but SQLite should not be used in production. // See https://github.com/nuts-foundation/nuts-node/pull/2589#discussion_r1399130608 db.SetMaxOpenConns(1) - dialector := sqlite.Dialector{Conn: db} - e.sqlDB, err = gorm.Open(dialector, gormConfig) + e.sqlDB, err = gorm.Open(sqlite.New(sqlite.Config{ + Conn: db, + }), gormConfig) if err != nil { return err } diff --git a/storage/sqlite/sqlite_ddlmod.go b/storage/sqlite/sqlite_ddlmod.go new file mode 100644 index 000000000..e5bcb3ad7 --- /dev/null +++ b/storage/sqlite/sqlite_ddlmod.go @@ -0,0 +1,317 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2013-NOW Jinzhu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// sqlite contains the GORM dialect for SQLite. This is copied from https://github.com/glebarez/sqlite +// This is needed to have GORM work with modernc.org/sqlite. +// github.com/glebarez/sqlite is late with updates and is/was incompatible with Go 1.23 +// The code here is from master (2024-04-14) +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "gorm.io/gorm/migrator" +) + +var ( + sqliteSeparator = "`|\"|'|\t" + uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^CONSTRAINT [%v]?[\w-]+[%v]? UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator)) + indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator)) + tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator)) + separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator)) + columnsRegexp = regexp.MustCompile(fmt.Sprintf(`[(,][%v]?(\w+)[%v]?`, sqliteSeparator, sqliteSeparator)) + columnRegexp = regexp.MustCompile(fmt.Sprintf(`^[%v]?([\w\d]+)[%v]?\s+([\w\(\)\d]+)(.*)$`, sqliteSeparator, sqliteSeparator)) + defaultValueRegexp = regexp.MustCompile(`(?i) DEFAULT \(?(.+)?\)?( |COLLATE|GENERATED|$)`) + regRealDataType = regexp.MustCompile(`[^\d](\d+)[^\d]?`) +) + +func getAllColumns(s string) []string { + allMatches := columnsRegexp.FindAllStringSubmatch(s, -1) + columns := make([]string, 0, len(allMatches)) + for _, matches := range allMatches { + if len(matches) > 1 { + columns = append(columns, matches[1]) + } + } + return columns +} + +type ddl struct { + head string + fields []string + columns []migrator.ColumnType +} + +func parseDDL(strs ...string) (*ddl, error) { + var result ddl + for _, str := range strs { + if sections := tableRegexp.FindStringSubmatch(str); len(sections) > 0 { + var ( + ddlBody = sections[2] + ddlBodyRunes = []rune(ddlBody) + bracketLevel int + quote rune + buf string + ) + ddlBodyRunesLen := len(ddlBodyRunes) + + result.head = sections[1] + + for idx := 0; idx < ddlBodyRunesLen; idx++ { + var ( + next rune = 0 + c = ddlBodyRunes[idx] + ) + if idx+1 < ddlBodyRunesLen { + next = ddlBodyRunes[idx+1] + } + + if sc := string(c); separatorRegexp.MatchString(sc) { + if c == next { + buf += sc // Skip escaped quote + idx++ + } else if quote > 0 { + quote = 0 + } else { + quote = c + } + } else if quote == 0 { + if c == '(' { + bracketLevel++ + } else if c == ')' { + bracketLevel-- + } else if bracketLevel == 0 { + if c == ',' { + result.fields = append(result.fields, strings.TrimSpace(buf)) + buf = "" + continue + } + } + } + + if bracketLevel < 0 { + return nil, errors.New("invalid DDL, unbalanced brackets") + } + + buf += string(c) + } + + if bracketLevel != 0 { + return nil, errors.New("invalid DDL, unbalanced brackets") + } + + if buf != "" { + result.fields = append(result.fields, strings.TrimSpace(buf)) + } + + for _, f := range result.fields { + fUpper := strings.ToUpper(f) + if strings.HasPrefix(fUpper, "CHECK") { + continue + } + if strings.HasPrefix(fUpper, "CONSTRAINT") { + matches := uniqueRegexp.FindStringSubmatch(f) + if len(matches) > 0 { + if columns := getAllColumns(matches[1]); len(columns) == 1 { + for idx, column := range result.columns { + if column.NameValue.String == columns[0] { + column.UniqueValue = sql.NullBool{Bool: true, Valid: true} + result.columns[idx] = column + break + } + } + } + } + continue + } + if strings.HasPrefix(fUpper, "PRIMARY KEY") { + for _, name := range getAllColumns(f) { + for idx, column := range result.columns { + if column.NameValue.String == name { + column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} + result.columns[idx] = column + break + } + } + } + } else if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 0 { + columnType := migrator.ColumnType{ + NameValue: sql.NullString{String: matches[1], Valid: true}, + DataTypeValue: sql.NullString{String: matches[2], Valid: true}, + ColumnTypeValue: sql.NullString{String: matches[2], Valid: true}, + PrimaryKeyValue: sql.NullBool{Valid: true}, + UniqueValue: sql.NullBool{Valid: true}, + NullableValue: sql.NullBool{Bool: true, Valid: true}, + DefaultValueValue: sql.NullString{Valid: false}, + } + + matchUpper := strings.ToUpper(matches[3]) + if strings.Contains(matchUpper, " NOT NULL") { + columnType.NullableValue = sql.NullBool{Bool: false, Valid: true} + } else if strings.Contains(matchUpper, " NULL") { + columnType.NullableValue = sql.NullBool{Bool: true, Valid: true} + } + if strings.Contains(matchUpper, " UNIQUE") { + columnType.UniqueValue = sql.NullBool{Bool: true, Valid: true} + } + if strings.Contains(matchUpper, " PRIMARY") { + columnType.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} + } + if defaultMatches := defaultValueRegexp.FindStringSubmatch(matches[3]); len(defaultMatches) > 1 { + if strings.ToLower(defaultMatches[1]) != "null" { + columnType.DefaultValueValue = sql.NullString{String: strings.Trim(defaultMatches[1], `"`), Valid: true} + } + } + + // data type length + matches := regRealDataType.FindAllStringSubmatch(columnType.DataTypeValue.String, -1) + if len(matches) == 1 && len(matches[0]) == 2 { + size, _ := strconv.Atoi(matches[0][1]) + columnType.LengthValue = sql.NullInt64{Valid: true, Int64: int64(size)} + columnType.DataTypeValue.String = strings.TrimSuffix(columnType.DataTypeValue.String, matches[0][0]) + } + + result.columns = append(result.columns, columnType) + } + } + } else if matches := indexRegexp.FindStringSubmatch(str); len(matches) > 0 { + // don't report Unique by UniqueIndex + } else { + return nil, errors.New("invalid DDL") + } + } + + return &result, nil +} + +func (d *ddl) clone() *ddl { + copied := new(ddl) + *copied = *d + + copied.fields = make([]string, len(d.fields)) + copy(copied.fields, d.fields) + copied.columns = make([]migrator.ColumnType, len(d.columns)) + copy(copied.columns, d.columns) + + return copied +} + +func (d *ddl) compile() string { + if len(d.fields) == 0 { + return d.head + } + + return fmt.Sprintf("%s (%s)", d.head, strings.Join(d.fields, ",")) +} + +func (d *ddl) renameTable(dst, src string) error { + tableReg, err := regexp.Compile("\\s*('|`|\")?\\b" + regexp.QuoteMeta(src) + "\\b('|`|\")?\\s*") + if err != nil { + return err + } + + replaced := tableReg.ReplaceAllString(d.head, fmt.Sprintf(" `%s` ", dst)) + if replaced == d.head { + return fmt.Errorf("failed to look up tablename `%s` from DDL head '%s'", src, d.head) + } + + d.head = replaced + return nil +} + +func (d *ddl) addConstraint(name string, sql string) { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for i := 0; i < len(d.fields); i++ { + if reg.MatchString(d.fields[i]) { + d.fields[i] = sql + return + } + } + + d.fields = append(d.fields, sql) +} + +func (d *ddl) removeConstraint(name string) bool { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for i := 0; i < len(d.fields); i++ { + if reg.MatchString(d.fields[i]) { + d.fields = append(d.fields[:i], d.fields[i+1:]...) + return true + } + } + return false +} + +func (d *ddl) hasConstraint(name string) bool { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for _, f := range d.fields { + if reg.MatchString(f) { + return true + } + } + return false +} + +func (d *ddl) getColumns() []string { + res := []string{} + + for _, f := range d.fields { + fUpper := strings.ToUpper(f) + if strings.HasPrefix(fUpper, "PRIMARY KEY") || + strings.HasPrefix(fUpper, "CHECK") || + strings.HasPrefix(fUpper, "CONSTRAINT") || + strings.Contains(fUpper, "GENERATED ALWAYS AS") { + continue + } + + reg := regexp.MustCompile("^[\"`']?([\\w\\d]+)[\"`']?") + match := reg.FindStringSubmatch(f) + + if match != nil { + res = append(res, "`"+match[1]+"`") + } + } + return res +} + +func (d *ddl) removeColumn(name string) bool { + reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$") + + for i := 0; i < len(d.fields); i++ { + if reg.MatchString(d.fields[i]) { + d.fields = append(d.fields[:i], d.fields[i+1:]...) + return true + } + } + + return false +} diff --git a/storage/sqlite/sqlite_dialector.go b/storage/sqlite/sqlite_dialector.go new file mode 100644 index 000000000..2a2f60e94 --- /dev/null +++ b/storage/sqlite/sqlite_dialector.go @@ -0,0 +1,297 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2013-NOW Jinzhu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package sqlite + +import ( + "context" + "database/sql" + "strconv" + + "gorm.io/gorm/callbacks" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" +) + +// DriverName is the default driver name for SQLite. +const DriverName = "sqlite" + +type Dialector struct { + DriverName string + DSN string + Conn gorm.ConnPool +} + +func Open(dsn string) gorm.Dialector { + return &Dialector{DSN: dsn} +} + +func (dialector Dialector) Name() string { + return "sqlite" +} + +func (dialector Dialector) Initialize(db *gorm.DB) (err error) { + if dialector.DriverName == "" { + dialector.DriverName = DriverName + } + + if dialector.Conn != nil { + db.ConnPool = dialector.Conn + } else { + conn, err := sql.Open(dialector.DriverName, dialector.DSN) + if err != nil { + return err + } + db.ConnPool = conn + } + + var version string + if err := db.ConnPool.QueryRowContext(context.Background(), "select sqlite_version()").Scan(&version); err != nil { + return err + } + // https://www.sqlite.org/releaselog/3_35_0.html + if compareVersion(version, "3.35.0") >= 0 { + callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ + CreateClauses: []string{"INSERT", "VALUES", "ON CONFLICT", "RETURNING"}, + UpdateClauses: []string{"UPDATE", "SET", "FROM", "WHERE", "RETURNING"}, + DeleteClauses: []string{"DELETE", "FROM", "WHERE", "RETURNING"}, + LastInsertIDReversed: true, + }) + } else { + callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ + LastInsertIDReversed: true, + }) + } + + for k, v := range dialector.ClauseBuilders() { + db.ClauseBuilders[k] = v + } + return +} + +func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder { + return map[string]clause.ClauseBuilder{ + "INSERT": func(c clause.Clause, builder clause.Builder) { + if insert, ok := c.Expression.(clause.Insert); ok { + if stmt, ok := builder.(*gorm.Statement); ok { + stmt.WriteString("INSERT ") + if insert.Modifier != "" { + stmt.WriteString(insert.Modifier) + stmt.WriteByte(' ') + } + + stmt.WriteString("INTO ") + if insert.Table.Name == "" { + stmt.WriteQuoted(stmt.Table) + } else { + stmt.WriteQuoted(insert.Table) + } + return + } + } + + c.Build(builder) + }, + "LIMIT": func(c clause.Clause, builder clause.Builder) { + if limit, ok := c.Expression.(clause.Limit); ok { + var lmt = -1 + if limit.Limit != nil && *limit.Limit >= 0 { + lmt = *limit.Limit + } + if lmt >= 0 || limit.Offset > 0 { + builder.WriteString("LIMIT ") + builder.WriteString(strconv.Itoa(lmt)) + } + if limit.Offset > 0 { + builder.WriteString(" OFFSET ") + builder.WriteString(strconv.Itoa(limit.Offset)) + } + } + }, + "FOR": func(c clause.Clause, builder clause.Builder) { + if _, ok := c.Expression.(clause.Locking); ok { + // SQLite3 does not support row-level locking. + return + } + c.Build(builder) + }, + } +} + +func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression { + if field.AutoIncrement { + return clause.Expr{SQL: "NULL"} + } + + // doesn't work, will raise error + return clause.Expr{SQL: "DEFAULT"} +} + +func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator { + return Migrator{migrator.Migrator{Config: migrator.Config{ + DB: db, + Dialector: dialector, + CreateIndexAfterCreateTable: true, + }}} +} + +func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) { + writer.WriteByte('?') +} + +func (dialector Dialector) QuoteTo(writer clause.Writer, str string) { + var ( + underQuoted, selfQuoted bool + continuousBacktick int8 + shiftDelimiter int8 + ) + + for _, v := range []byte(str) { + switch v { + case '`': + continuousBacktick++ + if continuousBacktick == 2 { + writer.WriteString("``") + continuousBacktick = 0 + } + case '.': + if continuousBacktick > 0 || !selfQuoted { + shiftDelimiter = 0 + underQuoted = false + continuousBacktick = 0 + writer.WriteString("`") + } + writer.WriteByte(v) + continue + default: + if shiftDelimiter-continuousBacktick <= 0 && !underQuoted { + writer.WriteString("`") + underQuoted = true + if selfQuoted = continuousBacktick > 0; selfQuoted { + continuousBacktick -= 1 + } + } + + for ; continuousBacktick > 0; continuousBacktick -= 1 { + writer.WriteString("``") + } + + writer.WriteByte(v) + } + shiftDelimiter++ + } + + if continuousBacktick > 0 && !selfQuoted { + writer.WriteString("``") + } + writer.WriteString("`") +} + +func (dialector Dialector) Explain(sql string, vars ...interface{}) string { + return logger.ExplainSQL(sql, nil, `"`, vars...) +} + +func (dialector Dialector) DataTypeOf(field *schema.Field) string { + switch field.DataType { + case schema.Bool: + return "numeric" + case schema.Int, schema.Uint: + if field.AutoIncrement { + // doesn't check `PrimaryKey`, to keep backward compatibility + // https://www.sqlite.org/autoinc.html + return "integer PRIMARY KEY AUTOINCREMENT" + } else { + return "integer" + } + case schema.Float: + return "real" + case schema.String: + return "text" + case schema.Time: + // Distinguish between schema.Time and tag time + if val, ok := field.TagSettings["TYPE"]; ok { + return val + } else { + return "datetime" + } + case schema.Bytes: + return "blob" + } + + return string(field.DataType) +} + +func (dialectopr Dialector) SavePoint(tx *gorm.DB, name string) error { + tx.Exec("SAVEPOINT " + name) + return nil +} + +func (dialectopr Dialector) RollbackTo(tx *gorm.DB, name string) error { + tx.Exec("ROLLBACK TO SAVEPOINT " + name) + return nil +} + +func (dialector Dialector) Translate(err error) error { + return err + //switch terr := err.(type) { + //case *gosqlite.Error: + // switch terr.Code() { + // case sqlite3.SQLITE_CONSTRAINT_UNIQUE: + // return gorm.ErrDuplicatedKey + // case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + // return gorm.ErrDuplicatedKey + // case sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY: + // return gorm.ErrForeignKeyViolated + // } + //} + //return err +} + +func compareVersion(version1, version2 string) int { + n, m := len(version1), len(version2) + i, j := 0, 0 + for i < n || j < m { + x := 0 + for ; i < n && version1[i] != '.'; i++ { + x = x*10 + int(version1[i]-'0') + } + i++ + y := 0 + for ; j < m && version2[j] != '.'; j++ { + y = y*10 + int(version2[j]-'0') + } + j++ + if x > y { + return 1 + } + if x < y { + return -1 + } + } + return 0 +} diff --git a/storage/sqlite/sqlite_migrator.go b/storage/sqlite/sqlite_migrator.go new file mode 100644 index 000000000..120ef4050 --- /dev/null +++ b/storage/sqlite/sqlite_migrator.go @@ -0,0 +1,454 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2013-NOW Jinzhu + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package sqlite + +import ( + "database/sql" + "fmt" + "strings" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" +) + +type Migrator struct { + migrator.Migrator +} + +func (m *Migrator) RunWithoutForeignKey(fc func() error) error { + var enabled int + m.DB.Raw("PRAGMA foreign_keys").Scan(&enabled) + if enabled == 1 { + m.DB.Exec("PRAGMA foreign_keys = OFF") + defer m.DB.Exec("PRAGMA foreign_keys = ON") + } + + return fc() +} + +func (m Migrator) HasTable(value interface{}) bool { + var count int + m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { + return m.DB.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", stmt.Table).Row().Scan(&count) + }) + return count > 0 +} + +func (m Migrator) DropTable(values ...interface{}) error { + return m.RunWithoutForeignKey(func() error { + values = m.ReorderModels(values, false) + tx := m.DB.Session(&gorm.Session{}) + + for i := len(values) - 1; i >= 0; i-- { + if err := m.RunWithValue(values[i], func(stmt *gorm.Statement) error { + return tx.Exec("DROP TABLE IF EXISTS ?", clause.Table{Name: stmt.Table}).Error + }); err != nil { + return err + } + } + + return nil + }) +} + +func (m Migrator) GetTables() (tableList []string, err error) { + return tableList, m.DB.Raw("SELECT name FROM sqlite_master where type=?", "table").Scan(&tableList).Error +} + +func (m Migrator) HasColumn(value interface{}, name string) bool { + var count int + m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { + if stmt.Schema != nil { + if field := stmt.Schema.LookUpField(name); field != nil { + name = field.DBName + } + } + + if name != "" { + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", + "table", stmt.Table, `%"`+name+`" %`, `%`+name+` %`, "%`"+name+"`%", "%["+name+"]%", "%\t"+name+"\t%", + ).Row().Scan(&count) + } + return nil + }) + return count > 0 +} + +func (m Migrator) AlterColumn(value interface{}, name string) error { + return m.RunWithoutForeignKey(func() error { + return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { + if field := stmt.Schema.LookUpField(name); field != nil { + var sqlArgs []interface{} + for i, f := range ddl.fields { + if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 1 && matches[1] == field.DBName { + ddl.fields[i] = fmt.Sprintf("`%v` ?", field.DBName) + sqlArgs = []interface{}{m.FullDataTypeOf(field)} + // table created by old version might look like `CREATE TABLE ? (? varchar(10) UNIQUE)`. + // FullDataTypeOf doesn't contain UNIQUE, so we need to add unique constraint. + if strings.Contains(strings.ToUpper(matches[3]), " UNIQUE") { + uniName := m.DB.NamingStrategy.UniqueName(stmt.Table, field.DBName) + uni, _ := m.GuessConstraintInterfaceAndTable(stmt, uniName) + if uni != nil { + uniSQL, uniArgs := uni.Build() + ddl.addConstraint(uniName, uniSQL) + sqlArgs = append(sqlArgs, uniArgs...) + } + } + break + } + } + return ddl, sqlArgs, nil + } + return nil, nil, fmt.Errorf("failed to alter field with name %v", name) + }) + }) +} + +// ColumnTypes return columnTypes []gorm.ColumnType and execErr error +func (m Migrator) ColumnTypes(value interface{}) ([]gorm.ColumnType, error) { + columnTypes := make([]gorm.ColumnType, 0) + execErr := m.RunWithValue(value, func(stmt *gorm.Statement) (err error) { + var ( + sqls []string + sqlDDL *ddl + ) + + if err := m.DB.Raw("SELECT sql FROM sqlite_master WHERE type IN ? AND tbl_name = ? AND sql IS NOT NULL order by type = ? desc", []string{"table", "index"}, stmt.Table, "table").Scan(&sqls).Error; err != nil { + return err + } + + if sqlDDL, err = parseDDL(sqls...); err != nil { + return err + } + + rows, err := m.DB.Session(&gorm.Session{}).Table(stmt.Table).Limit(1).Rows() + if err != nil { + return err + } + defer func() { + err = rows.Close() + }() + + var rawColumnTypes []*sql.ColumnType + rawColumnTypes, err = rows.ColumnTypes() + if err != nil { + return err + } + + for _, c := range rawColumnTypes { + columnType := migrator.ColumnType{SQLColumnType: c} + for _, column := range sqlDDL.columns { + if column.NameValue.String == c.Name() { + column.SQLColumnType = c + columnType = column + break + } + } + columnTypes = append(columnTypes, columnType) + } + + return err + }) + + return columnTypes, execErr +} + +func (m Migrator) DropColumn(value interface{}, name string) error { + return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { + if field := stmt.Schema.LookUpField(name); field != nil { + name = field.DBName + } + + ddl.removeColumn(name) + return ddl, nil, nil + }) +} + +func (m Migrator) CreateConstraint(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) + + return m.recreateTable(value, &table, + func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { + var ( + constraintName string + constraintSql string + constraintValues []interface{} + ) + + if constraint != nil { + constraintName = constraint.GetName() + constraintSql, constraintValues = constraint.Build() + } else { + return nil, nil, nil + } + + ddl.addConstraint(constraintName, constraintSql) + return ddl, constraintValues, nil + }) + }) +} + +func (m Migrator) DropConstraint(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) + if constraint != nil { + name = constraint.GetName() + } + + return m.recreateTable(value, &table, + func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { + ddl.removeConstraint(name) + return ddl, nil, nil + }) + }) +} + +func (m Migrator) HasConstraint(value interface{}, name string) bool { + var count int64 + m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) + if constraint != nil { + name = constraint.GetName() + } + + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", + "table", table, `%CONSTRAINT "`+name+`" %`, `%CONSTRAINT `+name+` %`, "%CONSTRAINT `"+name+"`%", "%CONSTRAINT ["+name+"]%", "%CONSTRAINT \t"+name+"\t%", + ).Row().Scan(&count) + + return nil + }) + + return count > 0 +} + +func (m Migrator) CurrentDatabase() (name string) { + var null interface{} + m.DB.Raw("PRAGMA database_list").Row().Scan(&null, &name, &null) + return +} + +func (m Migrator) BuildIndexOptions(opts []schema.IndexOption, stmt *gorm.Statement) (results []interface{}) { + for _, opt := range opts { + str := stmt.Quote(opt.DBName) + if opt.Expression != "" { + str = opt.Expression + } + + if opt.Collate != "" { + str += " COLLATE " + opt.Collate + } + + if opt.Sort != "" { + str += " " + opt.Sort + } + results = append(results, clause.Expr{SQL: str}) + } + return +} + +func (m Migrator) CreateIndex(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + if stmt.Schema != nil { + if idx := stmt.Schema.LookIndex(name); idx != nil { + opts := m.BuildIndexOptions(idx.Fields, stmt) + values := []interface{}{clause.Column{Name: idx.Name}, clause.Table{Name: stmt.Table}, opts} + + createIndexSQL := "CREATE " + if idx.Class != "" { + createIndexSQL += idx.Class + " " + } + createIndexSQL += "INDEX ?" + + if idx.Type != "" { + createIndexSQL += " USING " + idx.Type + } + createIndexSQL += " ON ??" + + if idx.Where != "" { + createIndexSQL += " WHERE " + idx.Where + } + + return m.DB.Exec(createIndexSQL, values...).Error + } + } + return fmt.Errorf("failed to create index with name %v", name) + }) +} + +func (m Migrator) HasIndex(value interface{}, name string) bool { + var count int + m.RunWithValue(value, func(stmt *gorm.Statement) error { + if stmt.Schema != nil { + if idx := stmt.Schema.LookIndex(name); idx != nil { + name = idx.Name + } + } + + if name != "" { + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, name, + ).Row().Scan(&count) + } + return nil + }) + return count > 0 +} + +func (m Migrator) RenameIndex(value interface{}, oldName, newName string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + var sql string + m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, oldName).Row().Scan(&sql) + if sql != "" { + if err := m.DropIndex(value, oldName); err != nil { + return err + } + return m.DB.Exec(strings.Replace(sql, oldName, newName, 1)).Error + } + return fmt.Errorf("failed to find index with name %v", oldName) + }) +} + +func (m Migrator) DropIndex(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + if stmt.Schema != nil { + if idx := stmt.Schema.LookIndex(name); idx != nil { + name = idx.Name + } + } + + return m.DB.Exec("DROP INDEX ?", clause.Column{Name: name}).Error + }) +} + +type Index struct { + Seq int + Name string + Unique bool + Origin string + Partial bool +} + +// GetIndexes return Indexes []gorm.Index and execErr error, +// See the [doc] +// +// [doc]: https://www.sqlite.org/pragma.html#pragma_index_list +func (m Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) { + indexes := make([]gorm.Index, 0) + err := m.RunWithValue(value, func(stmt *gorm.Statement) error { + rst := make([]*Index, 0) + if err := m.DB.Debug().Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)` + return err + } + for _, index := range rst { + if index.Origin == "u" { // skip the index was created by a UNIQUE constraint + continue + } + var columns []string + if err := m.DB.Raw("SELECT name FROM PRAGMA_index_info(?)", index.Name).Scan(&columns).Error; err != nil { // alias `PRAGMA index_info(?)` + return err + } + indexes = append(indexes, &migrator.Index{ + TableName: stmt.Table, + NameValue: index.Name, + ColumnList: columns, + PrimaryKeyValue: sql.NullBool{Bool: index.Origin == "pk", Valid: true}, // The exceptions are INTEGER PRIMARY KEY + UniqueValue: sql.NullBool{Bool: index.Unique, Valid: true}, + }) + } + return nil + }) + return indexes, err +} + +func (m Migrator) getRawDDL(table string) (string, error) { + var createSQL string + m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "table", table, table).Row().Scan(&createSQL) + + if m.DB.Error != nil { + return "", m.DB.Error + } + return createSQL, nil +} + +func (m Migrator) recreateTable( + value interface{}, tablePtr *string, + getCreateSQL func(ddl *ddl, stmt *gorm.Statement) (sql *ddl, sqlArgs []interface{}, err error), +) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + table := stmt.Table + if tablePtr != nil { + table = *tablePtr + } + + rawDDL, err := m.getRawDDL(table) + if err != nil { + return err + } + + originDDL, err := parseDDL(rawDDL) + if err != nil { + return err + } + + createDDL, sqlArgs, err := getCreateSQL(originDDL.clone(), stmt) + if err != nil { + return err + } + if createDDL == nil { + return nil + } + + newTableName := table + "__temp" + if err := createDDL.renameTable(newTableName, table); err != nil { + return err + } + + columns := createDDL.getColumns() + createSQL := createDDL.compile() + + return m.DB.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(createSQL, sqlArgs...).Error; err != nil { + return err + } + + queries := []string{ + fmt.Sprintf("INSERT INTO `%v`(%v) SELECT %v FROM `%v`", newTableName, strings.Join(columns, ","), strings.Join(columns, ","), table), + fmt.Sprintf("DROP TABLE `%v`", table), + fmt.Sprintf("ALTER TABLE `%v` RENAME TO `%v`", newTableName, table), + } + for _, query := range queries { + if err := tx.Exec(query).Error; err != nil { + return err + } + } + return nil + }) + }) +} From 7b31a2d800fe5eff1a100fb675adda89a289e546 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 16 Oct 2024 10:54:07 +0200 Subject: [PATCH 4/5] select correct dialector --- go.mod | 5 ----- go.sum | 20 -------------------- storage/engine.go | 6 +++--- storage/sqlite/sqlite_dialector.go | 25 +++++++++++++------------ vcr/revocation/bitstring_test.go | 2 +- 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index f8490d7a6..0ee36fd73 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/cbroglie/mustache v1.4.0 github.com/chromedp/chromedp v0.11.0 github.com/dlclark/regexp2 v1.11.4 - github.com/glebarez/sqlite v1.11.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/goodsign/monday v1.0.2 github.com/google/uuid v1.6.0 @@ -60,7 +59,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 - gorm.io/driver/sqlite v1.5.6 gorm.io/driver/sqlserver v1.5.3 schneider.vip/problem v1.9.1 ) @@ -196,7 +194,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/Regis24GmbH/go-diacritics.v2 v2.0.3 // indirect gorm.io/gorm v1.25.12 - modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect modernc.org/sqlite v1.33.0 @@ -204,9 +201,7 @@ require ( ) require ( - github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect diff --git a/go.sum b/go.sum index 02322a3e8..8073b4ac6 100644 --- a/go.sum +++ b/go.sum @@ -109,10 +109,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor v1.5.1 h1:XjQWBgdmQyqimslUh5r4tUGmoqzHmBFQOImkWGi2awg= github.com/fxamacker/cbor v1.5.1/go.mod h1:3aPGItF174ni7dDzd6JZ206H8cmr4GDNBGpPa971zsU= -github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= -github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= -github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= -github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-co-op/gocron v1.28.3 h1:swTsge6u/1Ei51b9VLMz/YTzEzWpbsk5SiR7m5fklTI= @@ -288,8 +284,6 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mdp/qrterminal/v3 v3.2.0 h1:qteQMXO3oyTK4IHwj2mWsKYYRBOp1Pj2WRYFYYNTCdk= github.com/mdp/qrterminal/v3 v3.2.0/go.mod h1:XGGuua4Lefrl7TLEsSONiD+UEjQXJZ4mPzF+gWYIJkk= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= @@ -610,34 +604,20 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= -gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= -gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/driver/sqlserver v1.5.3 h1:rjupPS4PVw+rjJkfvr8jn2lJ8BMhT4UW5FwuJY0P3Z0= gorm.io/driver/sqlserver v1.5.3/go.mod h1:B+CZ0/7oFJ6tAlefsKoyxdgDCXJKSgwS2bMOQZT0I00= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= diff --git a/storage/engine.go b/storage/engine.go index 6c3d1c978..92a05c0d3 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -34,12 +34,12 @@ import ( "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage/log" "github.com/nuts-foundation/nuts-node/storage/sql_migrations" + "github.com/nuts-foundation/nuts-node/storage/sqlite" "github.com/pressly/goose/v3" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/driver/sqlserver" "gorm.io/gorm" _ "modernc.org/sqlite" @@ -256,9 +256,9 @@ func (e *engine) initSQLDatabase() error { // With 1 connection, all actions will be performed sequentially. This impacts performance, but SQLite should not be used in production. // See https://github.com/nuts-foundation/nuts-node/pull/2589#discussion_r1399130608 db.SetMaxOpenConns(1) - e.sqlDB, err = gorm.Open(sqlite.New(sqlite.Config{ + e.sqlDB, err = gorm.Open(sqlite.Dialector{ Conn: db, - }), gormConfig) + }, gormConfig) if err != nil { return err } diff --git a/storage/sqlite/sqlite_dialector.go b/storage/sqlite/sqlite_dialector.go index 2a2f60e94..9332639d3 100644 --- a/storage/sqlite/sqlite_dialector.go +++ b/storage/sqlite/sqlite_dialector.go @@ -27,6 +27,8 @@ package sqlite import ( "context" "database/sql" + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" "strconv" "gorm.io/gorm/callbacks" @@ -257,19 +259,18 @@ func (dialectopr Dialector) RollbackTo(tx *gorm.DB, name string) error { } func (dialector Dialector) Translate(err error) error { + switch terr := err.(type) { + case *sqlite.Error: + switch terr.Code() { + case sqlite3.SQLITE_CONSTRAINT_UNIQUE: + return gorm.ErrDuplicatedKey + case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + return gorm.ErrDuplicatedKey + case sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY: + return gorm.ErrForeignKeyViolated + } + } return err - //switch terr := err.(type) { - //case *gosqlite.Error: - // switch terr.Code() { - // case sqlite3.SQLITE_CONSTRAINT_UNIQUE: - // return gorm.ErrDuplicatedKey - // case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - // return gorm.ErrDuplicatedKey - // case sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY: - // return gorm.ErrForeignKeyViolated - // } - //} - //return err } func compareVersion(version1, version2 string) int { diff --git a/vcr/revocation/bitstring_test.go b/vcr/revocation/bitstring_test.go index 2de832f32..8be8bfbb3 100644 --- a/vcr/revocation/bitstring_test.go +++ b/vcr/revocation/bitstring_test.go @@ -20,7 +20,7 @@ package revocation import ( "database/sql" - "github.com/glebarez/sqlite" + "github.com/nuts-foundation/nuts-node/storage/sqlite" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "math/rand" From 7d7b9d6b5188e066f0fbd1bf153f48f24cc69bb0 Mon Sep 17 00:00:00 2001 From: Wout Slakhorst Date: Wed, 16 Oct 2024 11:19:29 +0200 Subject: [PATCH 5/5] replace copied code with fork --- go.mod | 4 +- go.sum | 18 +- storage/engine.go | 2 +- storage/sqlite/sqlite_ddlmod.go | 317 -------------------- storage/sqlite/sqlite_dialector.go | 298 ------------------- storage/sqlite/sqlite_migrator.go | 454 ----------------------------- vcr/revocation/bitstring_test.go | 2 +- 7 files changed, 21 insertions(+), 1074 deletions(-) delete mode 100644 storage/sqlite/sqlite_ddlmod.go delete mode 100644 storage/sqlite/sqlite_dialector.go delete mode 100644 storage/sqlite/sqlite_migrator.go diff --git a/go.mod b/go.mod index 0ee36fd73..3d7b8b797 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/nuts-foundation/go-did v0.14.0 github.com/nuts-foundation/go-leia/v4 v4.0.3 github.com/nuts-foundation/go-stoabs v1.10.0 + github.com/nuts-foundation/sqlite v1.0.0 // check the oapi-codegen tool version in the makefile when upgrading the runtime github.com/oapi-codegen/runtime v1.1.1 github.com/piprate/json-gold v0.5.1-0.20230111113000-6ddbe6e6f19f @@ -196,13 +197,14 @@ require ( gorm.io/gorm v1.25.12 modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.33.0 + modernc.org/sqlite v1.33.1 rsc.io/qr v0.2.0 // indirect ) require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 8073b4ac6..8ea203664 100644 --- a/go.sum +++ b/go.sum @@ -350,6 +350,8 @@ github.com/nuts-foundation/go-leia/v4 v4.0.3 h1:xNZznXWvcIwonXIDmpDDvF7KmP9BOK0M github.com/nuts-foundation/go-leia/v4 v4.0.3/go.mod h1:tYveGED8tSbQYhZNv2DVTc51c2zEWmSF+MG96PAtalY= github.com/nuts-foundation/go-stoabs v1.10.0 h1:mNzm9jgraMc69a8gTgteli8t1CMxr1+gyI7A9Eh0NDk= github.com/nuts-foundation/go-stoabs v1.10.0/go.mod h1:So6S7ninucyJUU7I+JK1zcpoGDsZtd+jLXXacVtSWew= +github.com/nuts-foundation/sqlite v1.0.0 h1:gLKyVIHZqYfYpEy5Ji6vjNUH8rs0luiY3DWNcOSBBzM= +github.com/nuts-foundation/sqlite v1.0.0/go.mod h1:2GHDXCw5Sul9L3h8T5k0+558scm4ol4iXG+wVyYqueI= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= @@ -610,16 +612,28 @@ gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= -modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/storage/engine.go b/storage/engine.go index 92a05c0d3..3619c7115 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -34,7 +34,7 @@ import ( "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage/log" "github.com/nuts-foundation/nuts-node/storage/sql_migrations" - "github.com/nuts-foundation/nuts-node/storage/sqlite" + "github.com/nuts-foundation/sqlite" "github.com/pressly/goose/v3" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" diff --git a/storage/sqlite/sqlite_ddlmod.go b/storage/sqlite/sqlite_ddlmod.go deleted file mode 100644 index e5bcb3ad7..000000000 --- a/storage/sqlite/sqlite_ddlmod.go +++ /dev/null @@ -1,317 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2013-NOW Jinzhu - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -// sqlite contains the GORM dialect for SQLite. This is copied from https://github.com/glebarez/sqlite -// This is needed to have GORM work with modernc.org/sqlite. -// github.com/glebarez/sqlite is late with updates and is/was incompatible with Go 1.23 -// The code here is from master (2024-04-14) -package sqlite - -import ( - "database/sql" - "errors" - "fmt" - "regexp" - "strconv" - "strings" - - "gorm.io/gorm/migrator" -) - -var ( - sqliteSeparator = "`|\"|'|\t" - uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^CONSTRAINT [%v]?[\w-]+[%v]? UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator)) - indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator)) - tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator)) - separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator)) - columnsRegexp = regexp.MustCompile(fmt.Sprintf(`[(,][%v]?(\w+)[%v]?`, sqliteSeparator, sqliteSeparator)) - columnRegexp = regexp.MustCompile(fmt.Sprintf(`^[%v]?([\w\d]+)[%v]?\s+([\w\(\)\d]+)(.*)$`, sqliteSeparator, sqliteSeparator)) - defaultValueRegexp = regexp.MustCompile(`(?i) DEFAULT \(?(.+)?\)?( |COLLATE|GENERATED|$)`) - regRealDataType = regexp.MustCompile(`[^\d](\d+)[^\d]?`) -) - -func getAllColumns(s string) []string { - allMatches := columnsRegexp.FindAllStringSubmatch(s, -1) - columns := make([]string, 0, len(allMatches)) - for _, matches := range allMatches { - if len(matches) > 1 { - columns = append(columns, matches[1]) - } - } - return columns -} - -type ddl struct { - head string - fields []string - columns []migrator.ColumnType -} - -func parseDDL(strs ...string) (*ddl, error) { - var result ddl - for _, str := range strs { - if sections := tableRegexp.FindStringSubmatch(str); len(sections) > 0 { - var ( - ddlBody = sections[2] - ddlBodyRunes = []rune(ddlBody) - bracketLevel int - quote rune - buf string - ) - ddlBodyRunesLen := len(ddlBodyRunes) - - result.head = sections[1] - - for idx := 0; idx < ddlBodyRunesLen; idx++ { - var ( - next rune = 0 - c = ddlBodyRunes[idx] - ) - if idx+1 < ddlBodyRunesLen { - next = ddlBodyRunes[idx+1] - } - - if sc := string(c); separatorRegexp.MatchString(sc) { - if c == next { - buf += sc // Skip escaped quote - idx++ - } else if quote > 0 { - quote = 0 - } else { - quote = c - } - } else if quote == 0 { - if c == '(' { - bracketLevel++ - } else if c == ')' { - bracketLevel-- - } else if bracketLevel == 0 { - if c == ',' { - result.fields = append(result.fields, strings.TrimSpace(buf)) - buf = "" - continue - } - } - } - - if bracketLevel < 0 { - return nil, errors.New("invalid DDL, unbalanced brackets") - } - - buf += string(c) - } - - if bracketLevel != 0 { - return nil, errors.New("invalid DDL, unbalanced brackets") - } - - if buf != "" { - result.fields = append(result.fields, strings.TrimSpace(buf)) - } - - for _, f := range result.fields { - fUpper := strings.ToUpper(f) - if strings.HasPrefix(fUpper, "CHECK") { - continue - } - if strings.HasPrefix(fUpper, "CONSTRAINT") { - matches := uniqueRegexp.FindStringSubmatch(f) - if len(matches) > 0 { - if columns := getAllColumns(matches[1]); len(columns) == 1 { - for idx, column := range result.columns { - if column.NameValue.String == columns[0] { - column.UniqueValue = sql.NullBool{Bool: true, Valid: true} - result.columns[idx] = column - break - } - } - } - } - continue - } - if strings.HasPrefix(fUpper, "PRIMARY KEY") { - for _, name := range getAllColumns(f) { - for idx, column := range result.columns { - if column.NameValue.String == name { - column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} - result.columns[idx] = column - break - } - } - } - } else if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 0 { - columnType := migrator.ColumnType{ - NameValue: sql.NullString{String: matches[1], Valid: true}, - DataTypeValue: sql.NullString{String: matches[2], Valid: true}, - ColumnTypeValue: sql.NullString{String: matches[2], Valid: true}, - PrimaryKeyValue: sql.NullBool{Valid: true}, - UniqueValue: sql.NullBool{Valid: true}, - NullableValue: sql.NullBool{Bool: true, Valid: true}, - DefaultValueValue: sql.NullString{Valid: false}, - } - - matchUpper := strings.ToUpper(matches[3]) - if strings.Contains(matchUpper, " NOT NULL") { - columnType.NullableValue = sql.NullBool{Bool: false, Valid: true} - } else if strings.Contains(matchUpper, " NULL") { - columnType.NullableValue = sql.NullBool{Bool: true, Valid: true} - } - if strings.Contains(matchUpper, " UNIQUE") { - columnType.UniqueValue = sql.NullBool{Bool: true, Valid: true} - } - if strings.Contains(matchUpper, " PRIMARY") { - columnType.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} - } - if defaultMatches := defaultValueRegexp.FindStringSubmatch(matches[3]); len(defaultMatches) > 1 { - if strings.ToLower(defaultMatches[1]) != "null" { - columnType.DefaultValueValue = sql.NullString{String: strings.Trim(defaultMatches[1], `"`), Valid: true} - } - } - - // data type length - matches := regRealDataType.FindAllStringSubmatch(columnType.DataTypeValue.String, -1) - if len(matches) == 1 && len(matches[0]) == 2 { - size, _ := strconv.Atoi(matches[0][1]) - columnType.LengthValue = sql.NullInt64{Valid: true, Int64: int64(size)} - columnType.DataTypeValue.String = strings.TrimSuffix(columnType.DataTypeValue.String, matches[0][0]) - } - - result.columns = append(result.columns, columnType) - } - } - } else if matches := indexRegexp.FindStringSubmatch(str); len(matches) > 0 { - // don't report Unique by UniqueIndex - } else { - return nil, errors.New("invalid DDL") - } - } - - return &result, nil -} - -func (d *ddl) clone() *ddl { - copied := new(ddl) - *copied = *d - - copied.fields = make([]string, len(d.fields)) - copy(copied.fields, d.fields) - copied.columns = make([]migrator.ColumnType, len(d.columns)) - copy(copied.columns, d.columns) - - return copied -} - -func (d *ddl) compile() string { - if len(d.fields) == 0 { - return d.head - } - - return fmt.Sprintf("%s (%s)", d.head, strings.Join(d.fields, ",")) -} - -func (d *ddl) renameTable(dst, src string) error { - tableReg, err := regexp.Compile("\\s*('|`|\")?\\b" + regexp.QuoteMeta(src) + "\\b('|`|\")?\\s*") - if err != nil { - return err - } - - replaced := tableReg.ReplaceAllString(d.head, fmt.Sprintf(" `%s` ", dst)) - if replaced == d.head { - return fmt.Errorf("failed to look up tablename `%s` from DDL head '%s'", src, d.head) - } - - d.head = replaced - return nil -} - -func (d *ddl) addConstraint(name string, sql string) { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") - - for i := 0; i < len(d.fields); i++ { - if reg.MatchString(d.fields[i]) { - d.fields[i] = sql - return - } - } - - d.fields = append(d.fields, sql) -} - -func (d *ddl) removeConstraint(name string) bool { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") - - for i := 0; i < len(d.fields); i++ { - if reg.MatchString(d.fields[i]) { - d.fields = append(d.fields[:i], d.fields[i+1:]...) - return true - } - } - return false -} - -func (d *ddl) hasConstraint(name string) bool { - reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") - - for _, f := range d.fields { - if reg.MatchString(f) { - return true - } - } - return false -} - -func (d *ddl) getColumns() []string { - res := []string{} - - for _, f := range d.fields { - fUpper := strings.ToUpper(f) - if strings.HasPrefix(fUpper, "PRIMARY KEY") || - strings.HasPrefix(fUpper, "CHECK") || - strings.HasPrefix(fUpper, "CONSTRAINT") || - strings.Contains(fUpper, "GENERATED ALWAYS AS") { - continue - } - - reg := regexp.MustCompile("^[\"`']?([\\w\\d]+)[\"`']?") - match := reg.FindStringSubmatch(f) - - if match != nil { - res = append(res, "`"+match[1]+"`") - } - } - return res -} - -func (d *ddl) removeColumn(name string) bool { - reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$") - - for i := 0; i < len(d.fields); i++ { - if reg.MatchString(d.fields[i]) { - d.fields = append(d.fields[:i], d.fields[i+1:]...) - return true - } - } - - return false -} diff --git a/storage/sqlite/sqlite_dialector.go b/storage/sqlite/sqlite_dialector.go deleted file mode 100644 index 9332639d3..000000000 --- a/storage/sqlite/sqlite_dialector.go +++ /dev/null @@ -1,298 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2013-NOW Jinzhu - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package sqlite - -import ( - "context" - "database/sql" - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" - "strconv" - - "gorm.io/gorm/callbacks" - - "gorm.io/gorm" - "gorm.io/gorm/clause" - "gorm.io/gorm/logger" - "gorm.io/gorm/migrator" - "gorm.io/gorm/schema" -) - -// DriverName is the default driver name for SQLite. -const DriverName = "sqlite" - -type Dialector struct { - DriverName string - DSN string - Conn gorm.ConnPool -} - -func Open(dsn string) gorm.Dialector { - return &Dialector{DSN: dsn} -} - -func (dialector Dialector) Name() string { - return "sqlite" -} - -func (dialector Dialector) Initialize(db *gorm.DB) (err error) { - if dialector.DriverName == "" { - dialector.DriverName = DriverName - } - - if dialector.Conn != nil { - db.ConnPool = dialector.Conn - } else { - conn, err := sql.Open(dialector.DriverName, dialector.DSN) - if err != nil { - return err - } - db.ConnPool = conn - } - - var version string - if err := db.ConnPool.QueryRowContext(context.Background(), "select sqlite_version()").Scan(&version); err != nil { - return err - } - // https://www.sqlite.org/releaselog/3_35_0.html - if compareVersion(version, "3.35.0") >= 0 { - callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ - CreateClauses: []string{"INSERT", "VALUES", "ON CONFLICT", "RETURNING"}, - UpdateClauses: []string{"UPDATE", "SET", "FROM", "WHERE", "RETURNING"}, - DeleteClauses: []string{"DELETE", "FROM", "WHERE", "RETURNING"}, - LastInsertIDReversed: true, - }) - } else { - callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ - LastInsertIDReversed: true, - }) - } - - for k, v := range dialector.ClauseBuilders() { - db.ClauseBuilders[k] = v - } - return -} - -func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder { - return map[string]clause.ClauseBuilder{ - "INSERT": func(c clause.Clause, builder clause.Builder) { - if insert, ok := c.Expression.(clause.Insert); ok { - if stmt, ok := builder.(*gorm.Statement); ok { - stmt.WriteString("INSERT ") - if insert.Modifier != "" { - stmt.WriteString(insert.Modifier) - stmt.WriteByte(' ') - } - - stmt.WriteString("INTO ") - if insert.Table.Name == "" { - stmt.WriteQuoted(stmt.Table) - } else { - stmt.WriteQuoted(insert.Table) - } - return - } - } - - c.Build(builder) - }, - "LIMIT": func(c clause.Clause, builder clause.Builder) { - if limit, ok := c.Expression.(clause.Limit); ok { - var lmt = -1 - if limit.Limit != nil && *limit.Limit >= 0 { - lmt = *limit.Limit - } - if lmt >= 0 || limit.Offset > 0 { - builder.WriteString("LIMIT ") - builder.WriteString(strconv.Itoa(lmt)) - } - if limit.Offset > 0 { - builder.WriteString(" OFFSET ") - builder.WriteString(strconv.Itoa(limit.Offset)) - } - } - }, - "FOR": func(c clause.Clause, builder clause.Builder) { - if _, ok := c.Expression.(clause.Locking); ok { - // SQLite3 does not support row-level locking. - return - } - c.Build(builder) - }, - } -} - -func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression { - if field.AutoIncrement { - return clause.Expr{SQL: "NULL"} - } - - // doesn't work, will raise error - return clause.Expr{SQL: "DEFAULT"} -} - -func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator { - return Migrator{migrator.Migrator{Config: migrator.Config{ - DB: db, - Dialector: dialector, - CreateIndexAfterCreateTable: true, - }}} -} - -func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) { - writer.WriteByte('?') -} - -func (dialector Dialector) QuoteTo(writer clause.Writer, str string) { - var ( - underQuoted, selfQuoted bool - continuousBacktick int8 - shiftDelimiter int8 - ) - - for _, v := range []byte(str) { - switch v { - case '`': - continuousBacktick++ - if continuousBacktick == 2 { - writer.WriteString("``") - continuousBacktick = 0 - } - case '.': - if continuousBacktick > 0 || !selfQuoted { - shiftDelimiter = 0 - underQuoted = false - continuousBacktick = 0 - writer.WriteString("`") - } - writer.WriteByte(v) - continue - default: - if shiftDelimiter-continuousBacktick <= 0 && !underQuoted { - writer.WriteString("`") - underQuoted = true - if selfQuoted = continuousBacktick > 0; selfQuoted { - continuousBacktick -= 1 - } - } - - for ; continuousBacktick > 0; continuousBacktick -= 1 { - writer.WriteString("``") - } - - writer.WriteByte(v) - } - shiftDelimiter++ - } - - if continuousBacktick > 0 && !selfQuoted { - writer.WriteString("``") - } - writer.WriteString("`") -} - -func (dialector Dialector) Explain(sql string, vars ...interface{}) string { - return logger.ExplainSQL(sql, nil, `"`, vars...) -} - -func (dialector Dialector) DataTypeOf(field *schema.Field) string { - switch field.DataType { - case schema.Bool: - return "numeric" - case schema.Int, schema.Uint: - if field.AutoIncrement { - // doesn't check `PrimaryKey`, to keep backward compatibility - // https://www.sqlite.org/autoinc.html - return "integer PRIMARY KEY AUTOINCREMENT" - } else { - return "integer" - } - case schema.Float: - return "real" - case schema.String: - return "text" - case schema.Time: - // Distinguish between schema.Time and tag time - if val, ok := field.TagSettings["TYPE"]; ok { - return val - } else { - return "datetime" - } - case schema.Bytes: - return "blob" - } - - return string(field.DataType) -} - -func (dialectopr Dialector) SavePoint(tx *gorm.DB, name string) error { - tx.Exec("SAVEPOINT " + name) - return nil -} - -func (dialectopr Dialector) RollbackTo(tx *gorm.DB, name string) error { - tx.Exec("ROLLBACK TO SAVEPOINT " + name) - return nil -} - -func (dialector Dialector) Translate(err error) error { - switch terr := err.(type) { - case *sqlite.Error: - switch terr.Code() { - case sqlite3.SQLITE_CONSTRAINT_UNIQUE: - return gorm.ErrDuplicatedKey - case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - return gorm.ErrDuplicatedKey - case sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY: - return gorm.ErrForeignKeyViolated - } - } - return err -} - -func compareVersion(version1, version2 string) int { - n, m := len(version1), len(version2) - i, j := 0, 0 - for i < n || j < m { - x := 0 - for ; i < n && version1[i] != '.'; i++ { - x = x*10 + int(version1[i]-'0') - } - i++ - y := 0 - for ; j < m && version2[j] != '.'; j++ { - y = y*10 + int(version2[j]-'0') - } - j++ - if x > y { - return 1 - } - if x < y { - return -1 - } - } - return 0 -} diff --git a/storage/sqlite/sqlite_migrator.go b/storage/sqlite/sqlite_migrator.go deleted file mode 100644 index 120ef4050..000000000 --- a/storage/sqlite/sqlite_migrator.go +++ /dev/null @@ -1,454 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2013-NOW Jinzhu - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package sqlite - -import ( - "database/sql" - "fmt" - "strings" - - "gorm.io/gorm" - "gorm.io/gorm/clause" - "gorm.io/gorm/migrator" - "gorm.io/gorm/schema" -) - -type Migrator struct { - migrator.Migrator -} - -func (m *Migrator) RunWithoutForeignKey(fc func() error) error { - var enabled int - m.DB.Raw("PRAGMA foreign_keys").Scan(&enabled) - if enabled == 1 { - m.DB.Exec("PRAGMA foreign_keys = OFF") - defer m.DB.Exec("PRAGMA foreign_keys = ON") - } - - return fc() -} - -func (m Migrator) HasTable(value interface{}) bool { - var count int - m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { - return m.DB.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", stmt.Table).Row().Scan(&count) - }) - return count > 0 -} - -func (m Migrator) DropTable(values ...interface{}) error { - return m.RunWithoutForeignKey(func() error { - values = m.ReorderModels(values, false) - tx := m.DB.Session(&gorm.Session{}) - - for i := len(values) - 1; i >= 0; i-- { - if err := m.RunWithValue(values[i], func(stmt *gorm.Statement) error { - return tx.Exec("DROP TABLE IF EXISTS ?", clause.Table{Name: stmt.Table}).Error - }); err != nil { - return err - } - } - - return nil - }) -} - -func (m Migrator) GetTables() (tableList []string, err error) { - return tableList, m.DB.Raw("SELECT name FROM sqlite_master where type=?", "table").Scan(&tableList).Error -} - -func (m Migrator) HasColumn(value interface{}, name string) bool { - var count int - m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { - if stmt.Schema != nil { - if field := stmt.Schema.LookUpField(name); field != nil { - name = field.DBName - } - } - - if name != "" { - m.DB.Raw( - "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", - "table", stmt.Table, `%"`+name+`" %`, `%`+name+` %`, "%`"+name+"`%", "%["+name+"]%", "%\t"+name+"\t%", - ).Row().Scan(&count) - } - return nil - }) - return count > 0 -} - -func (m Migrator) AlterColumn(value interface{}, name string) error { - return m.RunWithoutForeignKey(func() error { - return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { - if field := stmt.Schema.LookUpField(name); field != nil { - var sqlArgs []interface{} - for i, f := range ddl.fields { - if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 1 && matches[1] == field.DBName { - ddl.fields[i] = fmt.Sprintf("`%v` ?", field.DBName) - sqlArgs = []interface{}{m.FullDataTypeOf(field)} - // table created by old version might look like `CREATE TABLE ? (? varchar(10) UNIQUE)`. - // FullDataTypeOf doesn't contain UNIQUE, so we need to add unique constraint. - if strings.Contains(strings.ToUpper(matches[3]), " UNIQUE") { - uniName := m.DB.NamingStrategy.UniqueName(stmt.Table, field.DBName) - uni, _ := m.GuessConstraintInterfaceAndTable(stmt, uniName) - if uni != nil { - uniSQL, uniArgs := uni.Build() - ddl.addConstraint(uniName, uniSQL) - sqlArgs = append(sqlArgs, uniArgs...) - } - } - break - } - } - return ddl, sqlArgs, nil - } - return nil, nil, fmt.Errorf("failed to alter field with name %v", name) - }) - }) -} - -// ColumnTypes return columnTypes []gorm.ColumnType and execErr error -func (m Migrator) ColumnTypes(value interface{}) ([]gorm.ColumnType, error) { - columnTypes := make([]gorm.ColumnType, 0) - execErr := m.RunWithValue(value, func(stmt *gorm.Statement) (err error) { - var ( - sqls []string - sqlDDL *ddl - ) - - if err := m.DB.Raw("SELECT sql FROM sqlite_master WHERE type IN ? AND tbl_name = ? AND sql IS NOT NULL order by type = ? desc", []string{"table", "index"}, stmt.Table, "table").Scan(&sqls).Error; err != nil { - return err - } - - if sqlDDL, err = parseDDL(sqls...); err != nil { - return err - } - - rows, err := m.DB.Session(&gorm.Session{}).Table(stmt.Table).Limit(1).Rows() - if err != nil { - return err - } - defer func() { - err = rows.Close() - }() - - var rawColumnTypes []*sql.ColumnType - rawColumnTypes, err = rows.ColumnTypes() - if err != nil { - return err - } - - for _, c := range rawColumnTypes { - columnType := migrator.ColumnType{SQLColumnType: c} - for _, column := range sqlDDL.columns { - if column.NameValue.String == c.Name() { - column.SQLColumnType = c - columnType = column - break - } - } - columnTypes = append(columnTypes, columnType) - } - - return err - }) - - return columnTypes, execErr -} - -func (m Migrator) DropColumn(value interface{}, name string) error { - return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { - if field := stmt.Schema.LookUpField(name); field != nil { - name = field.DBName - } - - ddl.removeColumn(name) - return ddl, nil, nil - }) -} - -func (m Migrator) CreateConstraint(value interface{}, name string) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) - - return m.recreateTable(value, &table, - func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { - var ( - constraintName string - constraintSql string - constraintValues []interface{} - ) - - if constraint != nil { - constraintName = constraint.GetName() - constraintSql, constraintValues = constraint.Build() - } else { - return nil, nil, nil - } - - ddl.addConstraint(constraintName, constraintSql) - return ddl, constraintValues, nil - }) - }) -} - -func (m Migrator) DropConstraint(value interface{}, name string) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) - if constraint != nil { - name = constraint.GetName() - } - - return m.recreateTable(value, &table, - func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) { - ddl.removeConstraint(name) - return ddl, nil, nil - }) - }) -} - -func (m Migrator) HasConstraint(value interface{}, name string) bool { - var count int64 - m.RunWithValue(value, func(stmt *gorm.Statement) error { - constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name) - if constraint != nil { - name = constraint.GetName() - } - - m.DB.Raw( - "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", - "table", table, `%CONSTRAINT "`+name+`" %`, `%CONSTRAINT `+name+` %`, "%CONSTRAINT `"+name+"`%", "%CONSTRAINT ["+name+"]%", "%CONSTRAINT \t"+name+"\t%", - ).Row().Scan(&count) - - return nil - }) - - return count > 0 -} - -func (m Migrator) CurrentDatabase() (name string) { - var null interface{} - m.DB.Raw("PRAGMA database_list").Row().Scan(&null, &name, &null) - return -} - -func (m Migrator) BuildIndexOptions(opts []schema.IndexOption, stmt *gorm.Statement) (results []interface{}) { - for _, opt := range opts { - str := stmt.Quote(opt.DBName) - if opt.Expression != "" { - str = opt.Expression - } - - if opt.Collate != "" { - str += " COLLATE " + opt.Collate - } - - if opt.Sort != "" { - str += " " + opt.Sort - } - results = append(results, clause.Expr{SQL: str}) - } - return -} - -func (m Migrator) CreateIndex(value interface{}, name string) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - if stmt.Schema != nil { - if idx := stmt.Schema.LookIndex(name); idx != nil { - opts := m.BuildIndexOptions(idx.Fields, stmt) - values := []interface{}{clause.Column{Name: idx.Name}, clause.Table{Name: stmt.Table}, opts} - - createIndexSQL := "CREATE " - if idx.Class != "" { - createIndexSQL += idx.Class + " " - } - createIndexSQL += "INDEX ?" - - if idx.Type != "" { - createIndexSQL += " USING " + idx.Type - } - createIndexSQL += " ON ??" - - if idx.Where != "" { - createIndexSQL += " WHERE " + idx.Where - } - - return m.DB.Exec(createIndexSQL, values...).Error - } - } - return fmt.Errorf("failed to create index with name %v", name) - }) -} - -func (m Migrator) HasIndex(value interface{}, name string) bool { - var count int - m.RunWithValue(value, func(stmt *gorm.Statement) error { - if stmt.Schema != nil { - if idx := stmt.Schema.LookIndex(name); idx != nil { - name = idx.Name - } - } - - if name != "" { - m.DB.Raw( - "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, name, - ).Row().Scan(&count) - } - return nil - }) - return count > 0 -} - -func (m Migrator) RenameIndex(value interface{}, oldName, newName string) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - var sql string - m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, oldName).Row().Scan(&sql) - if sql != "" { - if err := m.DropIndex(value, oldName); err != nil { - return err - } - return m.DB.Exec(strings.Replace(sql, oldName, newName, 1)).Error - } - return fmt.Errorf("failed to find index with name %v", oldName) - }) -} - -func (m Migrator) DropIndex(value interface{}, name string) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - if stmt.Schema != nil { - if idx := stmt.Schema.LookIndex(name); idx != nil { - name = idx.Name - } - } - - return m.DB.Exec("DROP INDEX ?", clause.Column{Name: name}).Error - }) -} - -type Index struct { - Seq int - Name string - Unique bool - Origin string - Partial bool -} - -// GetIndexes return Indexes []gorm.Index and execErr error, -// See the [doc] -// -// [doc]: https://www.sqlite.org/pragma.html#pragma_index_list -func (m Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) { - indexes := make([]gorm.Index, 0) - err := m.RunWithValue(value, func(stmt *gorm.Statement) error { - rst := make([]*Index, 0) - if err := m.DB.Debug().Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)` - return err - } - for _, index := range rst { - if index.Origin == "u" { // skip the index was created by a UNIQUE constraint - continue - } - var columns []string - if err := m.DB.Raw("SELECT name FROM PRAGMA_index_info(?)", index.Name).Scan(&columns).Error; err != nil { // alias `PRAGMA index_info(?)` - return err - } - indexes = append(indexes, &migrator.Index{ - TableName: stmt.Table, - NameValue: index.Name, - ColumnList: columns, - PrimaryKeyValue: sql.NullBool{Bool: index.Origin == "pk", Valid: true}, // The exceptions are INTEGER PRIMARY KEY - UniqueValue: sql.NullBool{Bool: index.Unique, Valid: true}, - }) - } - return nil - }) - return indexes, err -} - -func (m Migrator) getRawDDL(table string) (string, error) { - var createSQL string - m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "table", table, table).Row().Scan(&createSQL) - - if m.DB.Error != nil { - return "", m.DB.Error - } - return createSQL, nil -} - -func (m Migrator) recreateTable( - value interface{}, tablePtr *string, - getCreateSQL func(ddl *ddl, stmt *gorm.Statement) (sql *ddl, sqlArgs []interface{}, err error), -) error { - return m.RunWithValue(value, func(stmt *gorm.Statement) error { - table := stmt.Table - if tablePtr != nil { - table = *tablePtr - } - - rawDDL, err := m.getRawDDL(table) - if err != nil { - return err - } - - originDDL, err := parseDDL(rawDDL) - if err != nil { - return err - } - - createDDL, sqlArgs, err := getCreateSQL(originDDL.clone(), stmt) - if err != nil { - return err - } - if createDDL == nil { - return nil - } - - newTableName := table + "__temp" - if err := createDDL.renameTable(newTableName, table); err != nil { - return err - } - - columns := createDDL.getColumns() - createSQL := createDDL.compile() - - return m.DB.Transaction(func(tx *gorm.DB) error { - if err := tx.Exec(createSQL, sqlArgs...).Error; err != nil { - return err - } - - queries := []string{ - fmt.Sprintf("INSERT INTO `%v`(%v) SELECT %v FROM `%v`", newTableName, strings.Join(columns, ","), strings.Join(columns, ","), table), - fmt.Sprintf("DROP TABLE `%v`", table), - fmt.Sprintf("ALTER TABLE `%v` RENAME TO `%v`", newTableName, table), - } - for _, query := range queries { - if err := tx.Exec(query).Error; err != nil { - return err - } - } - return nil - }) - }) -} diff --git a/vcr/revocation/bitstring_test.go b/vcr/revocation/bitstring_test.go index 8be8bfbb3..2af0dd08a 100644 --- a/vcr/revocation/bitstring_test.go +++ b/vcr/revocation/bitstring_test.go @@ -20,7 +20,7 @@ package revocation import ( "database/sql" - "github.com/nuts-foundation/nuts-node/storage/sqlite" + "github.com/nuts-foundation/sqlite" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "math/rand"