From 7bc2aeef7acb8bd91c808aa39b0ab9ae0847a80b Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Wed, 12 Jul 2023 10:54:11 +0200 Subject: [PATCH] #108 Workaround for GROUP BY (#109) Co-authored-by: Christoph Kuhnke --- .github/workflows/ci-build.yml | 8 +-- .vscode/settings.json | 34 ++++++----- README.md | 1 + dependencies.md | 2 +- doc/changes/changelog.md | 1 + doc/changes/changes_7.1.4.md | 34 +++++++++++ doc/developer_guide.md | 5 ++ doc/dialects/exasol.md | 2 +- pk_generated_parent.pom | 2 +- pom.xml | 36 ++---------- .../exasol/AbstractExasolSqlDialectIT.java | 58 ++++++++++++++----- .../ExasolSqlDialectExaConnectionIT.java | 6 +- .../exasol/IntegrationTestConfiguration.java | 4 +- 13 files changed, 120 insertions(+), 73 deletions(-) create mode 100644 doc/changes/changes_7.1.4.md create mode 100644 doc/developer_guide.md diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 744a81f..669595a 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -11,9 +11,9 @@ jobs: strategy: fail-fast: false matrix: - docker_db_version: [ 7.1.17, 7.0.20 ] + docker_db_version: ["7.1.21"] env: - DEFAULT_DB_VERSION: "7.1.17" + DEFAULT_DB_VERSION: "7.1.21" runs-on: ubuntu-latest concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.docker_db_version }} @@ -25,9 +25,9 @@ jobs: - name: Set up JDK 11 uses: actions/setup-java@v3 with: - distribution: 'temurin' + distribution: "temurin" java-version: 11 - cache: 'maven' + cache: "maven" - name: Cache SonarCloud packages uses: actions/cache@v3 with: diff --git a/.vscode/settings.json b/.vscode/settings.json index 21c0cc1..caf52c3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,17 +1,21 @@ { - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true, - "source.generate.finalModifiers": true, - "source.fixAll": true - }, - "java.codeGeneration.useBlocks": true, - "java.saveActions.organizeImports": true, - "java.sources.organizeImports.starThreshold": 3, - "java.sources.organizeImports.staticStarThreshold": 3, - "java.test.config": { - "vmArgs": [ - "-Djava.util.logging.config.file=src/test/resources/logging.properties" - ] - }, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true, + "source.generate.finalModifiers": true, + "source.fixAll": true + }, + "java.codeGeneration.useBlocks": true, + "java.saveActions.organizeImports": true, + "java.sources.organizeImports.starThreshold": 3, + "java.sources.organizeImports.staticStarThreshold": 3, + "java.test.config": { + "vmArgs": [ + "-Djava.util.logging.config.file=src/test/resources/logging.properties" + ] + }, + "sonarlint.connectedMode.project": { + "connectionId": "exasol", + "projectKey": "com.exasol:exasol-virtual-schema" + } } diff --git a/README.md b/README.md index b6552e5..5320af9 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Find all the documentation in the [Virtual Schemas project][vs-doc]. ## Information for Developers * [Virtual Schema API Documentation][vs-api] +* [Developer Guide](./doc/developer_guide.md) [virtual-schemas-user-guide]: https://docs.exasol.com/database_concepts/virtual_schemas.htm diff --git a/dependencies.md b/dependencies.md index 0a17a9f..f618bee 100644 --- a/dependencies.md +++ b/dependencies.md @@ -38,7 +38,7 @@ | [Versions Maven Plugin][35] | [Apache License, Version 2.0][29] | | [duplicate-finder-maven-plugin Maven Mojo][36] | [Apache License 2.0][37] | | [Project keeper maven plugin][38] | [The MIT License][39] | -| [Apache Maven Assembly Plugin][40] | [Apache License, Version 2.0][29] | +| [Apache Maven Assembly Plugin][40] | [Apache-2.0][29] | | [Apache Maven JAR Plugin][41] | [Apache License, Version 2.0][29] | | [Artifact reference checker and unifier][42] | [MIT][43] | | [Apache Maven Deploy Plugin][44] | [Apache-2.0][29] | diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index d58f1d2..bcf2613 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [7.1.4](changes_7.1.4.md) * [7.1.3](changes_7.1.3.md) * [7.1.2](changes_7.1.2.md) * [7.1.1](changes_7.1.1.md) diff --git a/doc/changes/changes_7.1.4.md b/doc/changes/changes_7.1.4.md new file mode 100644 index 0000000..56f9c1c --- /dev/null +++ b/doc/changes/changes_7.1.4.md @@ -0,0 +1,34 @@ +# Exasol Virtual Schema 7.1.4, released 2023-07-12 + +Code name: Fix Issue With Integer Constants in `GROUP BY` + +## Summary + +This release fixes an issue with queries using `DISTINCT` with integer constants. The Exasol SQL processor turns `DISTINCT ` into `GROUP BY ` before push-down as an optimization. The adapter must not feed this back as Exasol interprets integers in `GROUP BY` clauses as column numbers which could lead to invalid results or the following error: + +``` +42000:Wrong column number. Too small value 0 as select list column reference in GROUP BY (smallest possible value is 1) +``` + +To fix this, Exasol VS now replaces integer constants in `GROUP BY` clauses with a constant string. + +Please that you can still safely use `GROUP BY ` in your original query, since Exasol internally converts this to `GROUP BY ""`, so that the virtual schema adapter can tell both situations apart. + +## Bugfixes + +* #108: Fixed issue with integer constants in `GROUP BY` + +## Dependency Updates + +### Compile Dependency Updates + +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.0` to `11.0.1` + +### Test Dependency Updates + +* Updated `com.exasol:exasol-testcontainers:6.6.0` to `6.6.1` +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.0` to `11.0.1` + +### Plugin Dependency Updates + +* Updated `org.apache.maven.plugins:maven-assembly-plugin:3.3.0` to `3.6.0` diff --git a/doc/developer_guide.md b/doc/developer_guide.md new file mode 100644 index 0000000..93732c7 --- /dev/null +++ b/doc/developer_guide.md @@ -0,0 +1,5 @@ +# Developer Guide + +## Enable Debug Output + +To enable debug output for the virtual schema adapter you can set [system properties defined by test-db-builder-java](https://github.com/exasol/test-db-builder-java/blob/main/doc/user_guide/user_guide.md#debug-output). diff --git a/doc/dialects/exasol.md b/doc/dialects/exasol.md index 7e30f71..dc073e6 100644 --- a/doc/dialects/exasol.md +++ b/doc/dialects/exasol.md @@ -17,7 +17,7 @@ The SQL statement below creates the adapter script, defines the Java class that ```sql CREATE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_EXASOL AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-11.0.0-exasol-7.1.3.jar; + %jar /buckets///virtual-schema-dist-11.0.1-exasol-7.1.4.jar; / ``` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 80b9229..f95b7d4 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol exasol-virtual-schema-generated-parent - 7.1.3 + 7.1.4 pom UTF-8 diff --git a/pom.xml b/pom.xml index 7d2b47d..29fb366 100644 --- a/pom.xml +++ b/pom.xml @@ -2,23 +2,13 @@ 4.0.0 exasol-virtual-schema - 7.1.3 + 7.1.4 Exasol Virtual Schema This projects contains the Exasol dialect for Exasol's Virtual Schema https://github.com/exasol/exasol-virtual-schema/ - 11.0.0 + 11.0.1 - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - com.exasol @@ -66,7 +56,7 @@ com.exasol exasol-testcontainers - 6.6.0 + 6.6.1 test @@ -116,13 +106,8 @@ maven-assembly-plugin - 3.3.0 - - src/assembly/all-dependencies.xml - virtual-schema-dist-${vscjdbc.version}-exasol-${project.version} - false @@ -159,19 +144,6 @@ false - - org.sonatype.ossindex.maven - ossindex-maven-plugin - - - - sonatype-2020-0026 - - - org.apache.maven.plugins maven-javadoc-plugin @@ -190,7 +162,7 @@ exasol-virtual-schema-generated-parent com.exasol - 7.1.3 + 7.1.4 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/dialects/exasol/AbstractExasolSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/exasol/AbstractExasolSqlDialectIT.java index 1c1b899..09fe05f 100644 --- a/src/test/java/com/exasol/adapter/dialects/exasol/AbstractExasolSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/exasol/AbstractExasolSqlDialectIT.java @@ -77,14 +77,6 @@ private static AdapterScript installVirtualSchemaAdapter(final ExasolSchema adap .build(); } - protected static boolean exasolVersionSupportsFingerprintInAddress() { - final ExasolDockerImageReference imageReference = EXASOL.getDockerImageReference(); - if (imageReference.getMajor() >= 8) { - return true; - } - return (imageReference.getMajor() >= 7) && (imageReference.getMinor() >= 1); - } - @AfterAll static void afterAll() throws SQLException { dropAll(adapterScript, adapterSchema); @@ -108,11 +100,8 @@ private ConnectionDefinition createAdapterConnectionDefinition(final User user) private String getJdbcUrl() { final int port = EXASOL.getDefaultInternalDatabasePort(); - if (exasolVersionSupportsFingerprintInAddress()) { - final String fingerprint = EXASOL.getTlsCertificateFingerprint().orElseThrow(); - return "jdbc:exa:localhost/" + fingerprint + ":" + port; - } - return "jdbc:exa:localhost:" + port + ";validateservercertificate=0"; + final String fingerprint = EXASOL.getTlsCertificateFingerprint().orElseThrow(); + return "jdbc:exa:localhost/" + fingerprint + ":" + port; } @AfterEach @@ -171,6 +160,13 @@ protected VirtualSchema createVirtualSchema(final Schema sourceSchema) { .build(); } + /** + * Get properties for the virtual schema. Note: if you want to enable debug output, you can set system + * properties defined by test-db-builder-java. + * + * @return properties for the virtual schema + */ private Map getVirtualSchemaProperties() { return getConnectionSpecificVirtualSchemaProperties(); } @@ -855,6 +851,42 @@ void testWildcards() throws SQLException { table().row("A", "VARCHAR(20) UTF8", null, null, null).matches()); } + @Test + @DisplayName("Verify DISTINCT with integer literal") + void testDistinctWithIntegerLiteral() throws SQLException { + final Table table = createSingleColumnTable("INT") // + .insert(1).insert(1).insert(2).insert(3); + final VirtualSchema virtualSchema = createVirtualSchema(this.sourceSchema); + try { + assertThat( + query("SELECT DISTINCT c1, 0 AS attr from " + + virtualSchema.getFullyQualifiedName() + "." + table.getName()), + table("BIGINT", "SMALLINT") // + .row(1L, (short) 0).row(2L, (short) 0).row(3L, (short) 0) // + .matchesInAnyOrder()); + } finally { + virtualSchema.drop(); + } + } + + @Test + @DisplayName("Verify GROUP BY with column number reference") + void testGroupByWithColumnNumber() throws SQLException { + final Table table = createSingleColumnTable("INT") // + .insert(1).insert(1).insert(2).insert(3); + final VirtualSchema virtualSchema = createVirtualSchema(this.sourceSchema); + try { + assertThat( + query("SELECT c1, count(c1) as count from " + + virtualSchema.getFullyQualifiedName() + "." + table.getName() + " group by 1"), + table("BIGINT", "BIGINT") // + .row(1L, 2L).row(2L, 1L).row(3L, 1L) // + .matchesInAnyOrder()); + } finally { + virtualSchema.drop(); + } + } + boolean isVersionOrHigher(final int majorVersion, final int minorVersion, final int fixVersion) { final ExasolDockerImageReference version = EXASOL.getDockerImageReference(); final long comparableImageVersion = calculatedComparableVersion((version.hasMajor() ? version.getMajor() : 0), diff --git a/src/test/java/com/exasol/adapter/dialects/exasol/ExasolSqlDialectExaConnectionIT.java b/src/test/java/com/exasol/adapter/dialects/exasol/ExasolSqlDialectExaConnectionIT.java index 470d2ab..62d20ff 100644 --- a/src/test/java/com/exasol/adapter/dialects/exasol/ExasolSqlDialectExaConnectionIT.java +++ b/src/test/java/com/exasol/adapter/dialects/exasol/ExasolSqlDialectExaConnectionIT.java @@ -47,10 +47,8 @@ void beforeEach() { } private String getTargetAddress() { - final String fingerprint = exasolVersionSupportsFingerprintInAddress() - ? "/" + EXASOL.getTlsCertificateFingerprint().orElseThrow() - : ""; - return "127.0.0.1" + fingerprint + ":" + EXASOL.getDefaultInternalDatabasePort(); + return "127.0.0.1" + "/" + EXASOL.getTlsCertificateFingerprint().orElseThrow() + ":" + + EXASOL.getDefaultInternalDatabasePort(); } @Override diff --git a/src/test/java/com/exasol/adapter/dialects/exasol/IntegrationTestConfiguration.java b/src/test/java/com/exasol/adapter/dialects/exasol/IntegrationTestConfiguration.java index 8cb1fe3..b3fe9fb 100644 --- a/src/test/java/com/exasol/adapter/dialects/exasol/IntegrationTestConfiguration.java +++ b/src/test/java/com/exasol/adapter/dialects/exasol/IntegrationTestConfiguration.java @@ -3,12 +3,12 @@ import java.nio.file.Path; public final class IntegrationTestConfiguration { - private static final String DEFAULT_DOCKER_DB_REFERENCE = "7.1.17"; + private static final String DEFAULT_DOCKER_DB_REFERENCE = "7.1.21"; /** * Do not use MavenProjectVersionGetter here to enable reference checker to check if reference points to the latest * version. */ - public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.0-exasol-7.1.3.jar"; + public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.1-exasol-7.1.4.jar"; public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); private IntegrationTestConfiguration() {