Merge branch 'finos:master' into bug

.github/workflows/build.yml

@@ -17,7 +17,7 @@ name: Build CI
 env:
   CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
   CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
-  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  MAVEN_OPTS: "-Xmx6g"
 
 on: [push, pull_request]
 
@@ -39,10 +39,10 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cache Maven dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-mvn-deps
         with:
@@ -79,13 +79,12 @@ jobs:
         with:
           theme: dark
 
-      - name: Build + Test
+      - name: Build (PR)
         if: github.ref != 'refs/heads/master'
         run: |
-          mvn -B -e -DXss4m -DXX:MaxRAMPercentage=90.0 -DskipTests=true install
-          mvn -B -e -DXss4m -DXX:MaxRAMPercentage=10.0 surefire:test -DargLine="-XX:MaxRAMPercentage=70.0" -Dsurefire.reports.directory=${GITHUB_WORKSPACE}/surefire-reports-aggregate
+          mvn -B -e -DskipTests=true install
 
-      - name: Build + Test + Maven Deploy + Sonar + Docker Snapshot
+      - name: Build (with Maven Deploy + Docker Snapshot)
         if: github.ref == 'refs/heads/master'
         env:
           DOCKER_USERNAME: finos
@@ -97,8 +96,11 @@ jobs:
         # and can cause problem with some code generators
         # See https://github.com/finos/legend-engine/pull/924
         run: |
-          mvn -B -e -DXss4m -DXX:MaxRAMPercentage=90.0 -DskipTests=true deploy -P docker-snapshot,sonar
-          mvn -B -e -DXss4m -DXX:MaxRAMPercentage=10.0 surefire:test -DargLine="-XX:MaxRAMPercentage=70.0" -Dsurefire.reports.directory=${GITHUB_WORKSPACE}/surefire-reports-aggregate
+          mvn -B -e -DskipTests=true deploy -P docker-snapshot
+
+      - name: Test
+        run: |
+          mvn -B -e surefire:test -DargLine="-XX:MaxRAMPercentage=70.0" -Dsurefire.reports.directory=${GITHUB_WORKSPACE}/surefire-reports-aggregate
 
       - name: Upload Test Results
         if: always()

.github/workflows/code-quality.yml

@@ -0,0 +1,74 @@
+# Copyright 2022 Goldman Sachs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Code Quality Check
+
+on: [workflow_dispatch]
+  # NOTE: currently, this is failing, we need further investigation to fix this build, for now, we will
+  # disable this code quality check to save resource
+  # push:
+  #   branches:
+  #     - master
+
+env:
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  MAVEN_OPTS: "-Xmx6g"
+
+# Cancel running jobs from previous pipelines of the same workflow on PR to save resource when commits are pushed quickly
+# NOTE: we don't want this behavior on default branch
+# See https://stackoverflow.com/a/68422069
+concurrency:
+  group: ${{ github.ref == 'refs/heads/master' && format('ci-default-branch-{0}-{1}', github.sha, github.workflow) || format('ci-pr-{0}-{1}', github.ref, github.workflow) }}
+  cancel-in-progress: true
+
+jobs:
+  sonar-code-check:
+    name: Sonar Code Quality Check
+    # NOTE: we cannot run this action in PR anyway because secrets are not accessible from forks
+    # See https://portal.productboard.com/sonarsource/1-sonarcloud/c/50-sonarcloud-analyzes-external-pull-request
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
+    # NOTE: larger runner is required to run this build
+    runs-on: ubuntu-latest-4-cores
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Cache Maven dependencies
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-mvn-deps
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Set up JDK
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+
+      - name: Check Java version
+        run: java -version
+
+      - name: Download deps and plugins
+        run: mvn de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+
+      - name: Check Code Quality
+        run: |
+          mvn -B -e -DskipTests=true install -P sonar

CODEOWNERS

@@ -1,13 +1,14 @@
-*                                        @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs
+*                                        @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @rafaelbey @aormerod-gs
 /.github/**                              @finos/legend-engine-maintainers
 /docs/**                                 @finos/legend-engine-maintainers
 /legend-engine-application-query/**      @finos/legend-engine-maintainers
 /legend-engine-config/**                 @finos/legend-engine-maintainers
-/legend-engine-core/**                   @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs
-/legend-engine-pure/**                   @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs
+/legend-engine-core/**                   @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @rafaelbey @aormerod-gs
+/legend-engine-pure/**                   @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @rafaelbey @aormerod-gs
 /legend-engine-xts-analytics/**          @finos/legend-engine-maintainers
 /legend-engine-xts-authentication/**     @finos/legend-engine-maintainers
 /legend-engine-xts-avro/**               @finos/legend-engine-maintainers
+/legend-engine-xts-bigqueryFunction/**   @finos/legend-engine-maintainers
 /legend-engine-xts-changetoken/**        @finos/legend-engine-maintainers
 /legend-engine-xts-daml/**               @finos/legend-engine-maintainers
 /legend-engine-xts-data-push/**          @finos/legend-engine-maintainers
@@ -17,7 +18,7 @@
 /legend-engine-xts-flatdata/**           @finos/legend-engine-maintainers
 /legend-engine-xts-functionActivator/**  @finos/legend-engine-maintainers
 /legend-engine-xts-generation/**         @finos/legend-engine-maintainers
-/legend-engine-xts-graphQL/**            @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs
+/legend-engine-xts-graphQL/**            @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @rafaelbey @aormerod-gs
 /legend-engine-xts-haskell/**            @finos/legend-engine-maintainers
 /legend-engine-xts-iceberg/**            @finos/legend-engine-maintainers
 /legend-engine-xts-java/**               @finos/legend-engine-maintainers
@@ -33,6 +34,7 @@
 /legend-engine-xts-service/**            @finos/legend-engine-maintainers
 /legend-engine-xts-serviceStore/**       @finos/legend-engine-maintainers
 /legend-engine-xts-snowflakeApp/**       @finos/legend-engine-maintainers
+/legend-engine-xts-hostedService/**      @finos/legend-engine-maintainers
 /legend-engine-xts-sql/**                @finos/legend-engine-maintainers
 /legend-engine-xts-text/**               @finos/legend-engine-maintainers
 /legend-engine-xts-xml/**                @finos/legend-engine-maintainers

docs/queries/queries.md

@@ -125,7 +125,7 @@ The use of `from` is best understood as limiting the object collection.  For exa
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
 }
 ```
 can be read as query all `Person` objects *that are realizable from the specified Store(s)*.
@@ -136,14 +136,14 @@ functions to be negotiated with a Store.  For example:
 {|
    Person.all()
       ->filter(p| $p.firstName == 'Peter')
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
 }
 ```
 implies that the `filter` is available for implementation by a Store.  Whereas:
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
       ->filter(p| $p.firstName == 'Peter')
 }
 ```
@@ -543,7 +543,7 @@ _(Object stream return not currently suppored)_
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
 }
 ```
 ### Execution Plan
@@ -566,7 +566,7 @@ _(Object stream return not currently suppored)_
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
       ->filter(p| $p.firstName == 'Peter')
       ->project(p| $p.firm.legalName, 'Name')
 }
@@ -595,7 +595,7 @@ _(Object stream return not currently suppored)_
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
       ->externalize(my::Binding, #{Person {firstName, lastName)}#)
 }
 ```
@@ -635,7 +635,7 @@ then
 ``` pure
 {|
    Person.all()
-      ->from(my::Mapping, ^Runtime(connections=^DatabaseConnection(element=my::Db, ...)))
+      ->from(my::Mapping, ^Runtime(connectionStores=^ConnectionStore(element=my::Db, connection=^DatabaseConnection( ...)))
       ->filter(p| $p.firstName == 'Peter')
       ->project(p| $p.firm.legalName, 'Name')
       ->externalize(my::TdsBinding)
@@ -711,7 +711,7 @@ then
    
    LegalEntity.all
       ->graphFetch(#{...}#)
-      ->from(my::PersonToLegalEntityMapping, ^Runtime(connections=^ModelConnection(element=^ModelStore(), instances=newMop(pair(Person, list($fromJson)))))) 
+      ->from(my::PersonToLegalEntityMapping, ^Runtime(connectionStores=^ConnectionStore(element=^ModelStore(), connection=^ModelConnection(instances=newMop(pair(Person, list($fromJson))))))) 
       ->externalize(my::SecondBinding, #{...}#)
 }
 ```

legend-engine-application-query/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine</artifactId>
-        <version>4.30.3-SNAPSHOT</version>
+        <version>4.34.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-application-query</artifactId>

legend-engine-config/legend-engine-configuration/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-config</artifactId>
-        <version>4.30.3-SNAPSHOT</version>
+        <version>4.34.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-configuration</artifactId>

legend-engine-config/legend-engine-connection-integration-tests/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-config</artifactId>
-        <version>4.30.3-SNAPSHOT</version>
+        <version>4.34.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-connection-integration-tests</artifactId>
@@ -85,6 +85,11 @@
             <artifactId>junit-jupiter-engine</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.bytebuddy</groupId>
+            <artifactId>byte-buddy</artifactId>
+            <scope>test</scope>
+        </dependency>
         <!-- TEST -->
     </dependencies>
 </project>

legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/AbstractConnectionFactoryTest.java

@@ -17,20 +17,23 @@
 import org.finos.legend.authentication.vault.CredentialVault;
 import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault;
 import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault;
+import org.finos.legend.connection.AuthenticationMechanismConfiguration;
 import org.finos.legend.connection.Authenticator;
 import org.finos.legend.connection.ConnectionFactory;
 import org.finos.legend.connection.DatabaseType;
 import org.finos.legend.connection.IdentityFactory;
 import org.finos.legend.connection.IdentitySpecification;
-import org.finos.legend.connection.InstrumentedStoreInstanceProvider;
+import org.finos.legend.connection.impl.InstrumentedStoreInstanceProvider;
 import org.finos.legend.connection.LegendEnvironment;
 import org.finos.legend.connection.RelationalDatabaseStoreSupport;
 import org.finos.legend.connection.StoreInstance;
+import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration;
 import org.finos.legend.connection.impl.KerberosCredentialExtractor;
 import org.finos.legend.connection.impl.KeyPairCredentialBuilder;
 import org.finos.legend.connection.impl.SnowflakeConnectionBuilder;
+import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration;
 import org.finos.legend.connection.impl.UserPasswordCredentialBuilder;
-import org.finos.legend.connection.jdbc.StaticJDBCConnectionBuilder;
+import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder;
 import org.finos.legend.connection.protocol.AuthenticationConfiguration;
 import org.finos.legend.connection.protocol.AuthenticationMechanismType;
 import org.finos.legend.engine.shared.core.identity.Identity;
@@ -58,27 +61,22 @@ public void initialize()
                         new EnvironmentCredentialVault()
                 )
                 .withStoreSupports(
-                        new RelationalDatabaseStoreSupport.Builder()
+                        new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES)
                                 .withIdentifier("Postgres")
-                                .withDatabase(DatabaseType.POSTGRES)
-                                .withAuthenticationMechanisms(
-                                        AuthenticationMechanismType.USER_PASSWORD
+                                .withAuthenticationMechanismConfigurations(
+                                        new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes(
+                                                UserPasswordAuthenticationConfiguration.class
+                                        ).build()
                                 )
                                 .build(),
-                        new RelationalDatabaseStoreSupport.Builder()
+                        new RelationalDatabaseStoreSupport.Builder(DatabaseType.SNOWFLAKE)
                                 .withIdentifier("Snowflake")
-                                .withDatabase(DatabaseType.SNOWFLAKE)
-                                .withAuthenticationMechanisms(
-                                        AuthenticationMechanismType.KEY_PAIR
-//                                        AuthenticationMechanismType.OAUTH
+                                .withAuthenticationMechanismConfigurations(
+                                        new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).withAuthenticationConfigurationTypes(
+                                                EncryptedPrivateKeyPairAuthenticationConfiguration.class
+                                        ).build()
                                 )
                                 .build()
-                )
-                .withAuthenticationMechanisms(
-                        AuthenticationMechanismType.USER_PASSWORD,
-                        AuthenticationMechanismType.API_KEY,
-                        AuthenticationMechanismType.KEY_PAIR,
-                        AuthenticationMechanismType.KERBEROS
                 );
 
         CredentialVault credentialVault = this.getCredentialVault();