Refactoring/7 use new bfs interface (#9)

* Add documentation build folder to .gitignore

* #7 Moved to the bucketfs PathLike interface

* #7 Moved to the bucketfs PathLike interface

* #7 Fixe the problem with archive extension

* #7 Added an integration test for SaaS

* #7 Added an integration test for SaaS

* #7 Disabled all integration tests but SaaS

* Update checks.yml

* Update checks.yml

* Update checks.yml

* Update checks.yml

* Update checks.yml

* Update checks.yml

* #7 Using dummy SaaS secrets

* Update checks.yml

* Update ci.yml

* Update pr-merge.yml

* Update ci-cd.yml

* Update checks.yml

* #7 Fixed operational_saas_database_id fixture

* #7 Made a pause before the udf validation

* #7 Made a bigger pause before the udf validation

* #7 Trying a new TB feature

* #7 Trying a new TB feature

* #7 Running tests without SaaS

* Running test with Saas [run-saas-tests]

* #7 Running all checks without SaaS

* #7 Running all checks with SaaS [run-saas-tests]

* #7 Addressed the review comments [run-saas-tests]

* #7 A new connection is required [run-saas-tests]

* #7 Addressed review issues [run-saas-tests]

* #7 Updated dependencies [run-saas-tests]

---------

Co-authored-by: Christoph Kuhnke <[email protected]>

.github/workflows/checks.yml

@@ -3,12 +3,12 @@ name: Checks
 on: workflow_call
 
 jobs:
-
   version-check-job:
     name: Version Check
     runs-on: ubuntu-latest
 
     steps:
+
       - name: SCM Checkout
         uses: actions/checkout@v3
         with:
@@ -103,11 +103,26 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Calculate Test Coverage
+      - name: Calculate Test Coverage with SaaS
+        if: "contains(github.event.head_commit.message, '[run-saas-tests]') && (matrix.python-version == '3.10')"
+        env:
+          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
+          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
+          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
         run: poetry run nox -s coverage -- -- --db-version ${{ matrix.exasol-version }}
 
+      - name: Calculate Test Coverage without SaaS
+        if: "!contains(github.event.head_commit.message, '[run-saas-tests]') || (matrix.python-version != '3.10')"
+        run: poetry run nox -s coverage -- -- -m "not saas" --db-version ${{ matrix.exasol-version }}
+
       - name: Upload Artifacts
         uses: actions/upload-artifact@v3
         with:
           name: .coverage
           path: .coverage
+
+      - name: Fail, if SaaS tests are not activated
+        if: "!contains(github.event.head_commit.message, '[run-saas-tests]') && (matrix.python-version == '3.10')"
+        run: |
+          echo "Failed because the SaaS tests are not activated"
+          exit 1

.github/workflows/ci-cd.yml

@@ -15,6 +15,7 @@ jobs:
     name: Checks
     needs: [ check-tag-version-job ]
     uses: ./.github/workflows/checks.yml
+    secrets: inherit
 
   cd-job:
     name: Continues Delivery

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
   ci-job:
     name: Checks
     uses: ./.github/workflows/checks.yml
+    secrets: inherit
 
   metrics:
     needs: [ ci-job ]

.github/workflows/pr-merge.yml

@@ -11,6 +11,7 @@ jobs:
   ci-job:
     name: Checks
     uses: ./.github/workflows/checks.yml
+    secrets: inherit
 
   publish-docs:
     name: Publish Documentation

doc/changes/unreleased.md

@@ -3,3 +3,7 @@
 ## Added
     - Integration tests for the LanguageContainerDeployer class
       and the cli function - language_container_deployer_main.
+
+    - Started using the bucket-fs PathLike interface.
+
+    - Added support SaaS backend

exasol/python_extension_common/deployment/language_container_deployer.py

@@ -7,27 +7,13 @@
 import ssl
 import requests     # type: ignore
 import pyexasol     # type: ignore
-from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation             # type: ignore
-from exasol_bucketfs_utils_python.bucket_config import BucketConfig, BucketFSConfig     # type: ignore
-from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig    # type: ignore
+import exasol.bucketfs as bfs   # type: ignore
+from exasol.saas.client.api_access import get_connection_params     # type: ignore
 
-logger = logging.getLogger(__name__)
 
+logger = logging.getLogger(__name__)
 
-def create_bucketfs_location(
-        bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
-        bucketfs_use_https: bool, bucketfs_user: str, bucketfs_password: str,
-        bucket: str, path_in_bucket: str) -> BucketFSLocation:
-    _bucketfs_connection = BucketFSConnectionConfig(
-        host=bucketfs_host, port=bucketfs_port, user=bucketfs_user,
-        pwd=bucketfs_password, is_https=bucketfs_use_https)
-    _bucketfs_config = BucketFSConfig(
-        bucketfs_name=bucketfs_name, connection_config=_bucketfs_connection)
-    _bucket_config = BucketConfig(
-        bucket_name=bucket, bucketfs_config=_bucketfs_config)
-    return BucketFSLocation(
-        bucket_config=_bucket_config,
-        base_path=PurePosixPath(path_in_bucket))
+ARCHIVE_EXTENSIONS = [".tar.gz", ".tgz", ".zip", ".tar"]
 
 
 def get_websocket_sslopt(use_ssl_cert_validation: bool = True,
@@ -87,14 +73,32 @@ def get_language_settings(pyexasol_conn: pyexasol.ExaConnection, alter_type: Lan
     return result[0][0]
 
 
+def get_udf_path(bucket_base_path: bfs.path.PathLike, bucket_file: str) -> PurePosixPath:
+    """
+    Returns the path of the specified file in a bucket, as it's seen from a UDF
+    For known types of archives removes the archive extension from the file name.
+
+    bucket_base_path    - Base directory in the bucket
+    bucket_file         - File path in the bucket, relative to the base directory.
+    """
+
+    for extension in ARCHIVE_EXTENSIONS:
+        if bucket_file.endswith(extension):
+            bucket_file = bucket_file[: -len(extension)]
+            break
+
+    file_path = bucket_base_path / bucket_file
+    return PurePosixPath(file_path.as_udf_path())
+
+
 class LanguageContainerDeployer:
 
     def __init__(self,
                  pyexasol_connection: pyexasol.ExaConnection,
                  language_alias: str,
-                 bucketfs_location: BucketFSLocation) -> None:
+                 bucketfs_path: bfs.path.PathLike) -> None:
 
-        self._bucketfs_location = bucketfs_location
+        self._bucketfs_path = bucketfs_path
         self._language_alias = language_alias
         self._pyexasol_conn = pyexasol_connection
         logger.debug("Init %s", LanguageContainerDeployer.__name__)
@@ -177,8 +181,8 @@ def upload_container(self, container_file: Path,
             raise RuntimeError(f"Container file {container_file} "
                                f"is not a file.")
         with open(container_file, "br") as f:
-            self._bucketfs_location.upload_fileobj_to_bucketfs(
-                fileobj=f, bucket_file_path=bucket_file_path)
+            file_path = self._bucketfs_path / bucket_file_path
+            file_path.write(f)
         logging.debug("Container is uploaded to bucketfs")
 
     def activate_container(self, bucket_file_path: str,
@@ -209,7 +213,7 @@ def generate_activation_command(self, bucket_file_path: str,
         allow_override   - If True the activation of a language container with the same alias will be overriden,
                            otherwise a RuntimeException will be thrown.
         """
-        path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path)
+        path_in_udf = get_udf_path(self._bucketfs_path, bucket_file_path)
         new_settings = \
             self._update_previous_language_settings(alter_type, allow_override, path_in_udf)
         alter_command = \
@@ -233,7 +237,7 @@ def get_language_definition(self, bucket_file_path: str):
 
         bucket_file_path - Path within the designated bucket where the container is uploaded.
         """
-        path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path)
+        path_in_udf = get_udf_path(self._bucketfs_path, bucket_file_path)
         result = self._generate_new_language_settings(path_in_udf=path_in_udf, prev_lang_aliases=[])
         return result
 
@@ -265,27 +269,61 @@ def _check_if_requested_language_alias_already_exists(
                 raise RuntimeError(warning_message)
 
     @classmethod
-    def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
-               bucketfs_use_https: bool, bucketfs_user: str,
-               bucketfs_password: str, bucket: str, path_in_bucket: str,
-               dsn: str, db_user: str, db_password: str, language_alias: str,
+    def create(cls,
+               language_alias: str, dsn: Optional[str] = None,
+               db_user: Optional[str] = None, db_password: Optional[str] = None,
+               bucketfs_host: Optional[str] = None, bucketfs_port: Optional[int] = None,
+               bucketfs_name: Optional[str] = None, bucket: Optional[str] = None,
+               bucketfs_user: Optional[str] = None, bucketfs_password: Optional[str] = None,
+               bucketfs_use_https: bool = True,
+               saas_url: Optional[str] = None,
+               saas_account_id: Optional[str] = None, saas_database_id: Optional[str] = None,
+               saas_token: Optional[str] = None,
+               path_in_bucket: str = '',
                use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None,
                ssl_client_certificate: Optional[str] = None,
                ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer":
 
+        # Infer where the database is - on-prem or SaaS.
+        if all((dsn, db_user, db_password, bucketfs_host, bucketfs_port,
+                bucketfs_name, bucket, bucketfs_user, bucketfs_password)):
+            connection_params = {'dsn': dsn, 'user': db_user, 'password': db_password}
+            bfs_url = (f"{'https' if bucketfs_use_https else 'http'}://"
+                       f"{bucketfs_host}:{bucketfs_port}")
+            verify = ssl_trusted_ca or use_ssl_cert_validation
+            bucketfs_path = bfs.path.build_path(backend=bfs.path.StorageBackend.onprem,
+                                                url=bfs_url,
+                                                username=bucketfs_user,
+                                                password=bucketfs_password,
+                                                service_name=bucketfs_name,
+                                                bucket_name=bucket,
+                                                verify=verify,
+                                                path=path_in_bucket)
+
+        elif all((saas_url, saas_account_id, saas_database_id, saas_token)):
+            connection_params = get_connection_params(host=saas_url,
+                                                      account_id=saas_account_id,
+                                                      database_id=saas_database_id,
+                                                      pat=saas_token)
+            bucketfs_path = bfs.path.build_path(backend=bfs.path.StorageBackend.saas,
+                                                url=saas_url,
+                                                account_id=saas_account_id,
+                                                database_id=saas_database_id,
+                                                pat=saas_token,
+                                                path=path_in_bucket)
+        else:
+            raise ValueError('Incomplete parameter list. '
+                             'Please either provide the parameters [dns, db_user, '
+                             'db_password, bucketfs_host, bucketfs_port, bucketfs_name, '
+                             'bucket, bucketfs_user, bucketfs_password] for an On-Prem '
+                             'database or [saas_url, saas_account_id, saas_database_id, '
+                             'saas_token] for a SaaS database.')
+
         websocket_sslopt = get_websocket_sslopt(use_ssl_cert_validation, ssl_trusted_ca,
                                                 ssl_client_certificate, ssl_private_key)
 
-        pyexasol_conn = pyexasol.connect(
-            dsn=dsn,
-            user=db_user,
-            password=db_password,
-            encryption=True,
-            websocket_sslopt=websocket_sslopt
-        )
-
-        bucketfs_location = create_bucketfs_location(
-            bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https,
-            bucketfs_user, bucketfs_password, bucket, path_in_bucket)
+        pyexasol_conn = pyexasol.connect(**connection_params,
+                                         encryption=True,
+                                         websocket_sslopt=websocket_sslopt)
 
-        return cls(pyexasol_conn, language_alias, bucketfs_location)
+        return cls(pyexasol_conn, language_alias, bucketfs_path)

exasol/python_extension_common/deployment/language_container_deployer_cli.py

@@ -8,6 +8,9 @@
 
 DB_PASSWORD_ENVIRONMENT_VARIABLE = "DB_PASSWORD"
 BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = "BUCKETFS_PASSWORD"
+SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE = "SAAS_ACCOUNT_ID"
+SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE = "SAAS_DATABASE_ID"
+SAAS_TOKEN_ENVIRONMENT_VARIABLE = "SAAS_TOKEN"
 
 
 class CustomizableParameters(Enum):
@@ -80,24 +83,32 @@ def clear_formatters(self):
 
 
 @click.command(name="language-container")
-@click.option('--bucketfs-name', type=str, required=True)
-@click.option('--bucketfs-host', type=str, required=True)
-@click.option('--bucketfs-port', type=int, required=True)
+@click.option('--bucketfs-name', type=str)
+@click.option('--bucketfs-host', type=str)
+@click.option('--bucketfs-port', type=int)
 @click.option('--bucketfs-use-https', type=bool, default=False)
-@click.option('--bucketfs-user', type=str, required=True, default="w")
-@click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True,
-              default=lambda: os.environ.get(BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE, ""))
-@click.option('--bucket', type=str, required=True)
-@click.option('--path-in-bucket', type=str, required=True, default=None)
+@click.option('--bucketfs-user', type=str)
+@click.option('--bucketfs-password', type=str,
+              default=lambda: os.environ.get(BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE))
+@click.option('--bucket', type=str)
+@click.option('--saas-url', type=str,
+              default='https://cloud.exasol.com')
+@click.option('--saas-account-id', type=str,
+              default=lambda: os.environ.get(SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE))
+@click.option('--saas-database-id', type=str,
+              default=lambda: os.environ.get(SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE))
+@click.option('--saas-token', type=str,
+              default=lambda: os.environ.get(SAAS_TOKEN_ENVIRONMENT_VARIABLE))
+@click.option('--path-in-bucket', type=str)
 @click.option('--container-file',
-              type=click.Path(exists=True, file_okay=True), default=None)
-@click.option('--version', type=str, default=None, expose_value=False,
+              type=click.Path(exists=True, file_okay=True))
+@click.option('--version', type=str, expose_value=False,
               callback=slc_parameter_formatters)
-@click.option('--dsn', type=str, required=True)
-@click.option('--db-user', type=str, required=True)
-@click.option('--db-pass', prompt='db password', hide_input=True,
-              default=lambda: os.environ.get(DB_PASSWORD_ENVIRONMENT_VARIABLE, ""))
-@click.option('--language-alias', type=str, default="PYTHON3_TE")
+@click.option('--dsn', type=str)
+@click.option('--db-user', type=str)
+@click.option('--db-pass',
+              default=lambda: os.environ.get(DB_PASSWORD_ENVIRONMENT_VARIABLE))
+@click.option('--language-alias', type=str, default="PYTHON3_EXT")
 @click.option('--ssl-cert-path', type=str, default="")
 @click.option('--ssl-client-cert-path', type=str, default="")
 @click.option('--ssl-client-private-key', type=str, default="")
@@ -113,6 +124,10 @@ def language_container_deployer_main(
         bucketfs_user: str,
         bucketfs_password: str,
         bucket: str,
+        saas_url: str,
+        saas_account_id: str,
+        saas_database_id: str,
+        saas_token: str,
         path_in_bucket: str,
         container_file: str,
         dsn: str,
@@ -137,6 +152,10 @@ def language_container_deployer_main(
         bucketfs_user=bucketfs_user,
         bucketfs_password=bucketfs_password,
         bucket=bucket,
+        saas_url=saas_url,
+        saas_account_id=saas_account_id,
+        saas_database_id=saas_database_id,
+        saas_token=saas_token,
         path_in_bucket=path_in_bucket,
         dsn=dsn,
         db_user=db_user,