Merge branch 'master' into gh556

.github/workflows/ci.yaml

@@ -6,14 +6,18 @@ on:
       - master
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Run unit tests
     timeout-minutes: 10
 
     strategy:
       matrix:
-        go-version: ["1.20", "1.21", "1.22.0-rc.2"]
+        go-version: [oldstable, stable]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -38,3 +42,14 @@ jobs:
           go vet ./...
           go build ./...
           make test-packages
+      - name: Install GoReleaser
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master' && matrix.go-version == 'stable'
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          install-only: true
+          distribution: goreleaser
+          version: "~> v2"
+      - name: Gorelease dry-run
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master' && matrix.go-version == 'stable'
+        run: |
+          goreleaser release --skip=publish --snapshot --fail-fast --clean

.github/workflows/e2e.yaml → .github/workflows/integration.yaml

@@ -1,20 +1,20 @@
-name: Goose end-end tests
+name: Goose integration tests
 
 on:
   push:
     branches:
       - master
   pull_request:
 
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
-    name: Run e2e tests
+    name: Run integration tests
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 2
-      matrix:
-        dialect: ["postgres", "mysql", "vertica", "clickhouse", "ydb", "turso"]
 
     steps:
       - name: Checkout code
@@ -23,6 +23,6 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "stable"
-      - name: Run e2e ${{ matrix.dialect }} tests
+      - name: Run full integration tests
         run: |
-          make test-e2e-${{ matrix.dialect }}
+          make test-integration

.github/workflows/lint.yaml

@@ -1,10 +1,16 @@
 name: golangci
+
 on:
   push:
     branches:
       - master
       - main
   pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   golangci:
     name: lint
@@ -16,7 +22,7 @@ jobs:
         with:
           go-version: "stable"
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest

.github/workflows/release.yaml

@@ -20,10 +20,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/release-notes.sh ${{github.ref_name}} > ${{runner.temp}}/release_notes.txt
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser
-          version: latest
+          version: "~> v2"
           args: release --clean --release-notes=${{runner.temp}}/release_notes.txt
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -13,3 +13,6 @@
 
 dist/
 release_notes.txt
+
+go.work
+go.work.sum

.goreleaser.yaml

@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 #
 # See https://goreleaser.com/customization/ for more information.
+version: 2
 project_name: goose
 
 before:

CHANGELOG.md

@@ -7,6 +7,79 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+- Update `WithDisableGlobalRegistry` behavior (#783). If set, this will ignore globally-registered
+  migrations instead of raising an error. Specifically, the following check is removed:
+
+```go
+if len(global) > 0 {
+	return nil, errors.New("global registry disabled, but provider has registered go migrations")
+}
+```
+
+This enables creating isolated goose provider(s) in legacy environments where global migrations may
+be registered. Without updating this behavior, it would be impossible to use
+`WithDisableGlobalRegistry` in combination with `WithGoMigrations`.
+
+## [v3.21.1]
+
+- Add `GetVersions` method to `goose.Provider`, returns the current (max db) version and the latest
+  (max filesystem) version. (#756)
+- Clarify `GetLatestVersion` method MUST return `ErrVersionNotFound` if no latest migration is
+  found. Previously it was returning a -1 and nil error, which was inconsistent with the rest of the
+  API surface.
+
+- Add `GetLatestVersion` implementations to all existing dialects. This is an optimization to avoid
+  loading all migrations when only the latest version is needed. This uses the `max` function in SQL
+  to get the latest version_id irrespective of the order of applied migrations.
+  - Refactor existing portions of the code to use the new `GetLatestVersion` method.
+
+## [v3.21.0]
+
+- Retracted. Broken release, please use v3.21.1 instead.
+
+## [v3.20.0]
+
+- Expand the `Store` interface by adding a `GetLatestVersion` method and make the interface public.
+- Add a (non-blocking) method to check if there are pending migrations to the `goose.Provider`
+  (#751):
+
+```go
+func (p *Provider) HasPending(context.Context) (bool, error) {}
+```
+
+The underlying implementation **does not respect the `SessionLocker`** (if one is enabled) and can
+be used to check for pending migrations without blocking or being blocked by other operations.
+
+- The methods `.Up`, `.UpByOne`, and `.UpTo` from `goose.Provider` will invoke `.HasPending` before
+  acquiring a lock with `SessionLocker` (if enabled). This addresses an edge case in
+  Kubernetes-style deployments where newer pods with long-running migrations prevent older pods -
+  which have all known migrations applied - from starting up due to an advisory lock. For more
+  details, refer to https://github.com/pressly/goose/pull/507#discussion_r1266498077 and #751.
+- Move integration tests to `./internal/testing` and make it a separate Go module. This will allow
+  us to have a cleaner top-level go.mod file and avoid imports unrelated to the goose project. See
+  [integration/README.md](https://github.com/pressly/goose/blob/d0641b5bfb3bd5d38d95fe7a63d7ddf2d282234d/internal/testing/integration/README.md)
+  for more details. This shouldn't affect users of the goose library.
+
+## [v3.19.2] - 2024-03-13
+
+- Remove duckdb support. The driver uses Cgo and we've decided to remove it until we can find a
+  better solution. If you were using duckdb with goose, please let us know by opening an issue.
+
+## [v3.19.1] - 2024-03-11
+
+- Fix selecting dialect for `redshift`
+- Add `GOOSE_MIGRATION_DIR` documentation
+- Bump github.com/opencontainers/runc to `v1.1.12` (security fix)
+- Update CI tests for go1.22
+- Make goose annotations case-insensitive
+  - All `-- +goose` annotations are now case-insensitive. This means that `-- +goose Up` and `--
++goose up` are now equivalent. This change was made to improve the user experience and to make the
+    annotations more consistent.
+
+## [v3.19.0] - 2024-03-11
+
+- Use [v3.19.1] instead. This was tagged but not released and does not contain release binaries.
+
 ## [v3.18.0] - 2024-01-31
 
 - Add environment variable substitution for SQL migrations. (#604)
@@ -127,7 +200,13 @@ Here's a quick summary:
 - Add new `context.Context`-aware functions and methods, for both sql and go migrations.
 - Return error when no migration files found or dir is not a directory.
 
-[Unreleased]: https://github.com/pressly/goose/compare/v3.18.0...HEAD
+[Unreleased]: https://github.com/pressly/goose/compare/v3.21.1...HEAD
+[v3.21.1]: https://github.com/pressly/goose/compare/v3.20.0...v3.21.1
+[v3.21.0]: https://github.com/pressly/goose/compare/v3.20.0...v3.21.0
+[v3.20.0]: https://github.com/pressly/goose/compare/v3.19.2...v3.20.0
+[v3.19.2]: https://github.com/pressly/goose/compare/v3.19.1...v3.19.2
+[v3.19.1]: https://github.com/pressly/goose/compare/v3.19.0...v3.19.1
+[v3.19.0]: https://github.com/pressly/goose/compare/v3.18.0...v3.19.0
 [v3.18.0]: https://github.com/pressly/goose/compare/v3.17.0...v3.18.0
 [v3.17.0]: https://github.com/pressly/goose/compare/v3.16.0...v3.17.0
 [v3.16.0]: https://github.com/pressly/goose/compare/v3.15.1...v3.16.0

Makefile

@@ -1,4 +1,4 @@
-GO_TEST_FLAGS ?= -race -count=1 -v -timeout=10m
+GO_TEST_FLAGS ?= -race -count=1 -v -timeout=5m
 
 # These are the default values for the test database. They can be overridden
 DB_USER ?= dbuser
@@ -12,7 +12,9 @@ DB_TURSO_PORT ?= 8080
 
 list-build-tags:
 	@echo "Available build tags:"
-	@echo "  $$(rg -o --trim 'no_[a-zA-Z0-9_]+' ./cmd/goose --no-line-number --no-filename | sort | uniq | tr '\n' ' ')"
+	@echo "$$(rg -o --trim 'no_[a-zA-Z0-9_]+' ./cmd/goose \
+		--no-line-number --no-filename | sort | uniq | \
+		xargs -n 4 | column -t | sed 's/^/  /')"
 
 .PHONY: dist
 dist:
@@ -42,25 +44,54 @@ test-packages:
 test-packages-short:
 	go test -test.short $(GO_TEST_FLAGS) $$(go list ./... | grep -v -e /tests -e /bin -e /cmd -e /examples)
 
-test-e2e: test-e2e-postgres test-e2e-mysql test-e2e-clickhouse test-e2e-vertica test-e2e-ydb test-e2e-turso
+coverage-short:
+	go test ./ -test.short  $(GO_TEST_FLAGS) -cover -coverprofile=coverage.out
+	go tool cover -html=coverage.out
 
-test-e2e-postgres:
-	go test $(GO_TEST_FLAGS) ./tests/e2e -dialect=postgres
+coverage:
+	go test ./ $(GO_TEST_FLAGS) -cover -coverprofile=coverage.out
+	go tool cover -html=coverage.out
 
-test-e2e-mysql:
-	go test $(GO_TEST_FLAGS) ./tests/e2e -dialect=mysql
+#
+# Integration-related targets
+#
+add-gowork:
+	@[ -f go.work ] || go work init
+	@[ -f go.work.sum ] || go work use -r .
 
-test-e2e-clickhouse:
-	go test $(GO_TEST_FLAGS) ./tests/clickhouse -test.short
+remove-gowork:
+	rm -rf go.work go.work.sum
 
-test-e2e-vertica:
-	go test $(GO_TEST_FLAGS) ./tests/vertica
+upgrade-integration-deps:
+	cd ./internal/testing && go get -u ./... && go mod tidy
 
-test-e2e-ydb:
-	go test $(GO_TEST_FLAGS) -parallel=1 ./tests/e2e -dialect=ydb
+test-postgres-long: add-gowork test-postgres
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='(TestPostgresProviderLocking|TestPostgresSessionLocker)'
 
-test-e2e-turso:
-	go test $(GO_TEST_FLAGS) -parallel=1 ./tests/e2e -dialect=turso
+test-postgres: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run="^TestPostgres$$"
+
+test-clickhouse: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='(TestClickhouse|TestClickhouseRemote)'
+
+test-mysql: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='TestMySQL'
+
+test-turso: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='TestTurso'
+
+test-vertica: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='TestVertica'
+
+test-ydb: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration -run='TestYDB'
+
+test-integration: add-gowork
+	go test $(GO_TEST_FLAGS) ./internal/testing/integration/...
+
+#
+# Docker-related targets
+#
 
 docker-cleanup:
 	docker stop -t=0 $$(docker ps --filter="label=goose_test" -aq)
@@ -73,6 +104,7 @@ docker-postgres:
 		-p $(DB_POSTGRES_PORT):5432 \
 		-l goose_test \
 		postgres:14-alpine -c log_statement=all
+	echo "postgres://$(DB_USER):$(DB_PASSWORD)@localhost:$(DB_POSTGRES_PORT)/$(DB_NAME)?sslmode=disable"
 
 docker-mysql:
 	docker run --rm -d \
@@ -83,6 +115,7 @@ docker-mysql:
 		-p $(DB_MYSQL_PORT):3306 \
 		-l goose_test \
 		mysql:8.0.31
+	echo "mysql://$(DB_USER):$(DB_PASSWORD)@localhost:$(DB_MYSQL_PORT)/$(DB_NAME)?parseTime=true"
 
 docker-clickhouse:
 	docker run --rm -d \
@@ -93,6 +126,7 @@ docker-clickhouse:
 		-p $(DB_CLICKHOUSE_PORT):9000/tcp \
 		-l goose_test \
 		clickhouse/clickhouse-server:23-alpine
+	echo "clickhouse://$(DB_USER):$(DB_PASSWORD)@localhost:$(DB_CLICKHOUSE_PORT)/$(DB_NAME)"
 
 docker-turso:
 	docker run --rm -d \