From 910de03becba0c082ccc1f33a4b6d4c75859dbcf Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 31 Aug 2023 18:31:31 +0200 Subject: [PATCH 1/6] Improve description of deployment process --- doc/design.md | 55 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/doc/design.md b/doc/design.md index 45aefb25..a63404db 100644 --- a/doc/design.md +++ b/doc/design.md @@ -426,50 +426,77 @@ Release assets: ``` postgresql-vs-2.0.2.jar postgresql-vs-2.0.2.jar.sha256 -postgres-extension-def-2.0.2.js -postgres-extension-def-2.0.2.js.sha256 +postgresql-vs-extension.js +postgresql-vs-extension.js.sha256 ``` -A crawler collects the JARs and extension definitions and copies them to BucketFS at scheduled interval. - -**This crawler is at the moment not part of this project.** +The following diagrams shows the deployment of the adapter JAR and the extension definition: ```plantuml @startuml storage "prostgres-virtual-schema release 2.0.2" { - file installerInRepo as "postgres-extension-def-2.0.2.js" + file installerInRepo as "postgresql-vs-extension.js" file jarV2InRepo as "postgresql-vs-2.0.2.jar" } storage "prostgres-virtual-schema release 1.0.0" { - file "postgres-extension-def-1.0.0.js" + file "postgresql-vs-extension.js" file jarV1InRepo as "postgresql-vs-1.0.0.jar" } -node "Exasol SaaS backend" { - storage Installers { - file installerInBucketFS as "postgres-extension-def-2.0.2.js" +node "Extension Registry" { + storage "Extension Definitions" { + file installerInBucketFS as "postgresql-vs-extension.js" } } -node "Exasol cluster (per customer)" { +node "Exasol SaaS" { storage BucketFS { file jarV2InBucketFs as "postgresql-vs-2.0.2.jar" file jarV1InBucketFS as "postgresql-vs-1.0.0.jar" } } -installerInRepo --> installerInBucketFS -jarV2InRepo --> jarV2InBucketFs -jarV1InRepo --> jarV1InBucketFS +installerInBucketFS --> installerInRepo : Update reference during deployment (1) +jarV2InRepo --> jarV2InBucketFs : Copy during deployment (2) +jarV1InRepo --> jarV1InBucketFS : Copy during deployment (2) @enduml ``` +The [extension registry](#extension-registry) contains references to all available extension definitions. + +When a new version of an extension is released, the following steps are required for deployment: + +* (1) Update extension registry to point to the new version's extension definition .js file. +* (2) Copy the adapter .jar file to the Exasol SaaS BucketFS. + +Notes: +* Both processes (1) and (2) are not yet automated. +* See details about [BucketsFS Buckets in Exasol SaaS](#bucketsfs-buckets-in-exasol-saas) in the next section. + Covers: * [`req~install-extension-database-objects~1`](system_requirements.md#update-extension) * [`req~define-configuration-parameters~1`](system_requirements.md#parameter-types) * [`req~uninstall-extension~1`](system_requirements.md#uninstalling-extensions) +#### BucketsFS Buckets in Exasol SaaS + +An Exasol SaaS instance has access to the following BucketFS buckets: + +* `/buckets/bfssaas/default/` +* `/buckets/bfssaas/advanced/` +* `/buckets/uploads/default/` + +The `bfssaas` BucketFS is managed globally for all Exasol SaaS instances by the SaaS team. All SaaS instances have read-only access to buckets `/buckets/bfssaas/default/` and `/buckets/bfssaas/advanced/` depending on the subscription. These buckets contain pre-loaded files for selected extensions. + +Bucket `/buckets/uploads/default/` is manged by end-users individually per SaaS instances. Users can upload files on their own. + +Currently Extension Manager does not have write access to BucketFS. That's why all required files (e.g. adapter JAR files, JDBC drivers etc.) must be managed with a separate manual or automated process. + +This means that new releases of adapter JARs must be uploaded manually to one of the `bfssaas` buckets before end-users can install the extension in EM. + +In the future we might have an additional bucket for each SaaS instance where EM can upload adapter JARs on demand during the installation. + ### Configure an Extension #### Validate Parameters From 33e2e0d4f26bb8528139ad78a45fc3c98103fc9c Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 31 Aug 2023 18:38:59 +0200 Subject: [PATCH 2/6] Increment version --- .github/workflows/ci-build.yml | 2 +- .github/workflows/project-keeper.sh | 2 +- .project-keeper.yml | 2 +- doc/changes/changelog.md | 1 + doc/changes/changes_0.5.1.md | 29 +++++++++++++++++++ .../pk_generated_parent.pom | 5 +++- extension-manager-client-java/pom.xml | 2 +- .../pk_generated_parent.pom | 5 +++- .../pom.xml | 2 +- .../itest/IntegrationTestCommon.java | 2 +- pkg/integrationTesting/dbTestSetup.go | 2 +- pom.xml | 2 +- 12 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 doc/changes/changes_0.5.1.md diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 6e480739..73f0da4d 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -13,7 +13,7 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }}-build cancel-in-progress: true env: - EXASOL_VERSION: 7.1.21 + EXASOL_VERSION: 7.1.22 steps: - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/project-keeper.sh b/.github/workflows/project-keeper.sh index af0f76f2..fdd34ff1 100755 --- a/.github/workflows/project-keeper.sh +++ b/.github/workflows/project-keeper.sh @@ -5,7 +5,7 @@ set -o nounset set -o pipefail readonly pk_mode="${1-verify}"; -readonly version="2.9.10" +readonly version="2.9.11" readonly pk_jar="$HOME/.m2/repository/com/exasol/project-keeper-cli/$version/project-keeper-cli-$version.jar" diff --git a/.project-keeper.yml b/.project-keeper.yml index 17e5938d..16f09d0b 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -24,7 +24,7 @@ sources: path: registry/package.json - type: npm path: pkg/parameterValidator/package.json -version: 0.5.0 +version: 0.5.1 linkReplacements: - "Unknown|https://github.com/DATA-DOG/go-sqlmock/blob/master/LICENSE" - "https://github.com/swagger-api/swagger-core/modules/swagger-annotations|https://github.com/swagger-api/swagger-core/tree/master/modules/swagger-annotations" diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index db0ae694..580b3ed3 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [0.5.1](changes_0.5.1.md) * [0.5.0](changes_0.5.0.md) * [0.4.0](changes_0.4.0.md) * [0.3.0](changes_0.3.0.md) diff --git a/doc/changes/changes_0.5.1.md b/doc/changes/changes_0.5.1.md new file mode 100644 index 00000000..bd7133ef --- /dev/null +++ b/doc/changes/changes_0.5.1.md @@ -0,0 +1,29 @@ +# Extension Manager 0.5.1, released 2023-??-?? + +Code name: + +## Summary + +## Features + +* ISSUE_NUMBER: description + +## Dependency Updates + +### Extension Manager Java Client + +#### Plugin Dependency Updates + +* Updated `com.exasol:project-keeper-maven-plugin:2.9.10` to `2.9.11` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.3.0` to `3.4.0` + +### Extension Integration Tests Library + +#### Compile Dependency Updates + +* Updated `com.exasol:extension-manager-client-java:0.5.0` to `0.5.1` + +#### Plugin Dependency Updates + +* Updated `com.exasol:project-keeper-maven-plugin:2.9.10` to `2.9.11` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.3.0` to `3.4.0` diff --git a/extension-manager-client-java/pk_generated_parent.pom b/extension-manager-client-java/pk_generated_parent.pom index 42ec6946..d57d11f0 100644 --- a/extension-manager-client-java/pk_generated_parent.pom +++ b/extension-manager-client-java/pk_generated_parent.pom @@ -68,7 +68,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.3.0 + 3.4.0 enforce-maven @@ -209,6 +209,9 @@ org.apache.maven.plugins maven-source-plugin + 3.2.1 diff --git a/extension-manager-client-java/pom.xml b/extension-manager-client-java/pom.xml index d02ef678..25dc147e 100644 --- a/extension-manager-client-java/pom.xml +++ b/extension-manager-client-java/pom.xml @@ -100,7 +100,7 @@ com.exasol project-keeper-maven-plugin - 2.9.10 + 2.9.11 diff --git a/extension-manager-integration-test-java/pk_generated_parent.pom b/extension-manager-integration-test-java/pk_generated_parent.pom index 9cd4828a..db17c4d6 100644 --- a/extension-manager-integration-test-java/pk_generated_parent.pom +++ b/extension-manager-integration-test-java/pk_generated_parent.pom @@ -68,7 +68,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.3.0 + 3.4.0 enforce-maven @@ -209,6 +209,9 @@ org.apache.maven.plugins maven-source-plugin + 3.2.1 diff --git a/extension-manager-integration-test-java/pom.xml b/extension-manager-integration-test-java/pom.xml index cd1c870d..d84bbc45 100644 --- a/extension-manager-integration-test-java/pom.xml +++ b/extension-manager-integration-test-java/pom.xml @@ -82,7 +82,7 @@ com.exasol project-keeper-maven-plugin - 2.9.10 + 2.9.11 diff --git a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java index 286d6f0e..9dac19c0 100644 --- a/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java +++ b/extension-manager-integration-test-java/src/test/java/com/exasol/extensionmanager/itest/IntegrationTestCommon.java @@ -16,7 +16,7 @@ private IntegrationTestCommon() { static ExasolTestSetup createExasolTestSetup() { if (System.getProperty("com.exasol.dockerdb.image") == null) { - System.setProperty("com.exasol.dockerdb.image", "7.1.21"); + System.setProperty("com.exasol.dockerdb.image", "7.1.22"); } return new ExasolTestSetupFactory(Path.of("dummy-config")).getTestSetup(); } diff --git a/pkg/integrationTesting/dbTestSetup.go b/pkg/integrationTesting/dbTestSetup.go index 327a1fe4..c275bcd4 100644 --- a/pkg/integrationTesting/dbTestSetup.go +++ b/pkg/integrationTesting/dbTestSetup.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/suite" ) -const defaultExasolDbVersion = "7.1.21" +const defaultExasolDbVersion = "7.1.22" type DbTestSetup struct { suite *suite.Suite diff --git a/pom.xml b/pom.xml index d5a7139e..d803d0f6 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ pom https://github.com/exasol/extension-manager/ - 0.5.0 + 0.5.1 5.10.0 From ff4e3815082fb376d7572a4f0ee9efcc9fc209ea Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 31 Aug 2023 18:42:45 +0200 Subject: [PATCH 3/6] Upgrade dependencies --- doc/changes/changes_0.5.1.md | 10 ++++++++++ extension-manager-client-java/pom.xml | 2 +- extension-manager-integration-test-java/pom.xml | 6 +++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/changes/changes_0.5.1.md b/doc/changes/changes_0.5.1.md index bd7133ef..c08e44c3 100644 --- a/doc/changes/changes_0.5.1.md +++ b/doc/changes/changes_0.5.1.md @@ -12,6 +12,10 @@ Code name: ### Extension Manager Java Client +#### Test Dependency Updates + +* Updated `nl.jqno.equalsverifier:equalsverifier:3.15` to `3.15.1` + #### Plugin Dependency Updates * Updated `com.exasol:project-keeper-maven-plugin:2.9.10` to `2.9.11` @@ -21,7 +25,13 @@ Code name: #### Compile Dependency Updates +* Updated `com.exasol:exasol-test-setup-abstraction-java:2.0.2` to `2.0.3` * Updated `com.exasol:extension-manager-client-java:0.5.0` to `0.5.1` +* Updated `com.exasol:test-db-builder-java:3.4.2` to `3.5.0` + +#### Test Dependency Updates + +* Updated `org.mockito:mockito-junit-jupiter:5.4.0` to `5.5.0` #### Plugin Dependency Updates diff --git a/extension-manager-client-java/pom.xml b/extension-manager-client-java/pom.xml index 25dc147e..474cad95 100644 --- a/extension-manager-client-java/pom.xml +++ b/extension-manager-client-java/pom.xml @@ -91,7 +91,7 @@ nl.jqno.equalsverifier equalsverifier - 3.15 + 3.15.1 test diff --git a/extension-manager-integration-test-java/pom.xml b/extension-manager-integration-test-java/pom.xml index d84bbc45..e7781400 100644 --- a/extension-manager-integration-test-java/pom.xml +++ b/extension-manager-integration-test-java/pom.xml @@ -20,7 +20,7 @@ com.exasol exasol-test-setup-abstraction-java - 2.0.2 + 2.0.3 @@ -32,7 +32,7 @@ com.exasol test-db-builder-java - 3.4.2 + 3.5.0 com.exasol @@ -54,7 +54,7 @@ org.mockito mockito-junit-jupiter - 5.4.0 + 5.5.0 test From 163433c0c700c5d423b900b6f362948e00068d0d Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 31 Aug 2023 18:47:12 +0200 Subject: [PATCH 4/6] Exclude netty-handler vulnerability --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index d803d0f6..0923c2db 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,10 @@ CVE-2020-36641 is reported to be fixed in aXMLRPC 1.12.1. We use version 1.13.0, so this is a false positive. --> CVE-2020-36641 + + CVE-2023-4586 From ebcd5ace5037de6dd428649320f20e30de65cd74 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 1 Sep 2023 12:20:00 +0200 Subject: [PATCH 5/6] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sebastian Bär --- doc/design.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design.md b/doc/design.md index a63404db..6af7623b 100644 --- a/doc/design.md +++ b/doc/design.md @@ -467,8 +467,8 @@ The [extension registry](#extension-registry) contains references to all availab When a new version of an extension is released, the following steps are required for deployment: -* (1) Update extension registry to point to the new version's extension definition .js file. -* (2) Copy the adapter .jar file to the Exasol SaaS BucketFS. +* (1) Update extension registry to point to the new version's extension definition `.js` file. +* (2) Copy the adapter `.jar` file to the Exasol SaaS BucketFS. Notes: * Both processes (1) and (2) are not yet automated. From 2ef3ec029069863e153067cc33af99afb0769bea Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 1 Sep 2023 13:10:25 +0200 Subject: [PATCH 6/6] Update changelog --- doc/changes/changes_0.5.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/changes/changes_0.5.1.md b/doc/changes/changes_0.5.1.md index c08e44c3..71a02c32 100644 --- a/doc/changes/changes_0.5.1.md +++ b/doc/changes/changes_0.5.1.md @@ -4,9 +4,9 @@ Code name: ## Summary -## Features +## Documentation -* ISSUE_NUMBER: description +* #129: Improved description of deployment process ## Dependency Updates