Merge remote-tracking branch 'origin/main'

exasol/python_extension_common/deployment/language_container_deployer.py

@@ -8,13 +8,11 @@
 import requests     # type: ignore
 import pyexasol     # type: ignore
 import exasol.bucketfs as bfs   # type: ignore
-from exasol.saas.client.api_access import get_connection_params     # type: ignore
+from exasol.saas.client.api_access import (get_connection_params, get_database_id)      # type: ignore
 
 
 logger = logging.getLogger(__name__)
 
-ARCHIVE_EXTENSIONS = [".tar.gz", ".tgz", ".zip", ".tar"]
-
 
 def get_websocket_sslopt(use_ssl_cert_validation: bool = True,
                          ssl_trusted_ca: Optional[str] = None,
@@ -76,17 +74,11 @@ def get_language_settings(pyexasol_conn: pyexasol.ExaConnection, alter_type: Lan
 def get_udf_path(bucket_base_path: bfs.path.PathLike, bucket_file: str) -> PurePosixPath:
     """
     Returns the path of the specified file in a bucket, as it's seen from a UDF
-    For known types of archives removes the archive extension from the file name.
 
     bucket_base_path    - Base directory in the bucket
     bucket_file         - File path in the bucket, relative to the base directory.
     """
 
-    for extension in ARCHIVE_EXTENSIONS:
-        if bucket_file.endswith(extension):
-            bucket_file = bucket_file[: -len(extension)]
-            break
-
     file_path = bucket_base_path / bucket_file
     return PurePosixPath(file_path.as_udf_path())
 
@@ -278,7 +270,7 @@ def create(cls,
                bucketfs_use_https: bool = True,
                saas_url: Optional[str] = None,
                saas_account_id: Optional[str] = None, saas_database_id: Optional[str] = None,
-               saas_token: Optional[str] = None,
+               saas_database_name: Optional[str] = None, saas_token: Optional[str] = None,
                path_in_bucket: str = '',
                use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None,
                ssl_client_certificate: Optional[str] = None,
@@ -300,11 +292,20 @@ def create(cls,
                                                 verify=verify,
                                                 path=path_in_bucket)
 
-        elif all((saas_url, saas_account_id, saas_database_id, saas_token)):
+        elif all((saas_url, saas_account_id, saas_token,
+                  any((saas_database_id, saas_database_name)))):
             connection_params = get_connection_params(host=saas_url,
                                                       account_id=saas_account_id,
                                                       database_id=saas_database_id,
+                                                      database_name=saas_database_name,
                                                       pat=saas_token)
+            saas_database_id = (saas_database_id or
+                                get_database_id(
+                                    host=saas_url,
+                                    account_id=saas_account_id,
+                                    pat=saas_token,
+                                    database_name=saas_database_name
+                                ))
             bucketfs_path = bfs.path.build_path(backend=bfs.path.StorageBackend.saas,
                                                 url=saas_url,
                                                 account_id=saas_account_id,

exasol/python_extension_common/deployment/language_container_deployer_cli.py

@@ -6,12 +6,6 @@
 import click
 from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer
 
-DB_PASSWORD_ENVIRONMENT_VARIABLE = "DB_PASSWORD"
-BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = "BUCKETFS_PASSWORD"
-SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE = "SAAS_ACCOUNT_ID"
-SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE = "SAAS_DATABASE_ID"
-SAAS_TOKEN_ENVIRONMENT_VARIABLE = "SAAS_TOKEN"
-
 
 class CustomizableParameters(Enum):
     """
@@ -82,32 +76,73 @@ def clear_formatters(self):
 slc_parameter_formatters = _ParameterFormatters()
 
 
+# This text will be displayed instead of the actual value, if found in an environment
+# variable, in a prompt.
+SECRET_DISPLAY = '***'
+
+
+class SecretParams(Enum):
+    """
+    This enum serves as a definition of confidential parameters which values should not be
+    displayed in the console, unless the user types them in the command line.
+
+    The enum name is also the name of the environment variable where the correspondent
+    secret value can be stored.
+
+    The enum value is also the name of the cli parameter.
+    """
+    DB_PASSWORD = 'db-pass'
+    BUCKETFS_PASSWORD = 'bucketfs-password'
+    SAAS_ACCOUNT_ID = 'saas-account-id'
+    SAAS_DATABASE_ID = 'saas-database-id'
+    SAAS_TOKEN = 'saas-token'
+
+
+def secret_callback(ctx: click.Context, param: click.Option, value: Any):
+    """
+    Here we try to get the secret parameter value from an environment variable.
+    The reason for doing this in the callback instead of using a callable default is
+    that we don't want the default to be displayed in the prompt. There seems to
+    be no way of altering this behaviour.
+    """
+    if value == SECRET_DISPLAY:
+        secret_param = SecretParams(param.opts[0][2:])
+        return os.environ.get(secret_param.name)
+    return value
+
+
 @click.command(name="language-container")
 @click.option('--bucketfs-name', type=str)
 @click.option('--bucketfs-host', type=str)
 @click.option('--bucketfs-port', type=int)
 @click.option('--bucketfs-use-https', type=bool, default=False)
 @click.option('--bucketfs-user', type=str)
-@click.option('--bucketfs-password', type=str,
-              default=lambda: os.environ.get(BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE))
+@click.option(f'--{SecretParams.BUCKETFS_PASSWORD.value}', type=str,
+              prompt='BucketFS password', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
 @click.option('--bucket', type=str)
 @click.option('--saas-url', type=str,
               default='https://cloud.exasol.com')
-@click.option('--saas-account-id', type=str,
-              default=lambda: os.environ.get(SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE))
-@click.option('--saas-database-id', type=str,
-              default=lambda: os.environ.get(SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE))
-@click.option('--saas-token', type=str,
-              default=lambda: os.environ.get(SAAS_TOKEN_ENVIRONMENT_VARIABLE))
+@click.option(f'--{SecretParams.SAAS_ACCOUNT_ID.value}', type=str,
+              prompt='SaaS account id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option(f'--{SecretParams.SAAS_DATABASE_ID.value}', type=str,
+              prompt='SaaS database id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--saas-database-name', type=str)
+@click.option(f'--{SecretParams.SAAS_TOKEN.value}', type=str,
+              prompt='SaaS token', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
 @click.option('--path-in-bucket', type=str)
 @click.option('--container-file',
               type=click.Path(exists=True, file_okay=True))
 @click.option('--version', type=str, expose_value=False,
               callback=slc_parameter_formatters)
 @click.option('--dsn', type=str)
 @click.option('--db-user', type=str)
-@click.option('--db-pass',
-              default=lambda: os.environ.get(DB_PASSWORD_ENVIRONMENT_VARIABLE))
+@click.option(f'--{SecretParams.DB_PASSWORD.value}', type=str,
+              prompt='DB password', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
 @click.option('--language-alias', type=str, default="PYTHON3_EXT")
 @click.option('--ssl-cert-path', type=str, default="")
 @click.option('--ssl-client-cert-path', type=str, default="")
@@ -127,6 +162,7 @@ def language_container_deployer_main(
         saas_url: str,
         saas_account_id: str,
         saas_database_id: str,
+        saas_database_name: str,
         saas_token: str,
         path_in_bucket: str,
         container_file: str,
@@ -155,6 +191,7 @@ def language_container_deployer_main(
         saas_url=saas_url,
         saas_account_id=saas_account_id,
         saas_database_id=saas_database_id,
+        saas_database_name=saas_database_name,
         saas_token=saas_token,
         path_in_bucket=path_in_bucket,
         dsn=dsn,

pyproject.toml

@@ -11,8 +11,8 @@ readme = "README.md"
 python = ">=3.8.0,<4.0"
 pyexasol = "^0.25.0"
 exasol-bucketfs = ">=0.10.0"
-click = "^8.0.4"
-exasol-saas-api = ">=0.6.0"
+click = "^8.1.7"
+exasol-saas-api = {git = 'https://github.com/exasol/saas-api-python.git', branch = 'main'}
 requests = "<2.32.0"
 
 [tool.poetry.group.dev.dependencies]

test/integration/conftest.py

@@ -11,11 +11,6 @@
     OpenApiAccess,
     get_connection_params
 )
-from exasol.saas.client.openapi.models import CreateAllowedIP
-from exasol.saas.client.openapi.api.security.add_allowed_ip import sync as add_allowed_ip
-from exasol.saas.client.openapi.api.security.delete_allowed_ip import sync_detailed as delete_allowed_ip
-from exasol.saas.client.openapi.api.clusters.list_clusters import sync as list_clusters
-from exasol.saas.client.openapi.api.clusters.get_cluster_connection import sync as get_cluster_connection
 
 from exasol.python_extension_common.deployment.language_container_deployer_cli import (
     language_container_deployer_main, slc_parameter_formatters, CustomizableParameters)
@@ -99,9 +94,13 @@ def api_access(saas_host, saas_token, saas_account_id) -> OpenApiAccess:
 
 
 @pytest.fixture(scope="session")
-def operational_saas_database_id(api_access) -> str:
-    database_name = timestamp_name('PEC')
-    with api_access.database(database_name) as db:
+def saas_database_name() -> str:
+    return timestamp_name('PEC')
+
+
+@pytest.fixture(scope="session")
+def operational_saas_database_id(api_access, saas_database_name) -> str:
+    with api_access.database(saas_database_name) as db:
         api_access.wait_until_running(db.id)
         yield db.id
 

test/integration/test_language_container_deployer_saas_cli.py

@@ -8,11 +8,7 @@
 from click.testing import CliRunner
 
 import pyexasol
-from exasol.python_extension_common.deployment.language_container_deployer_cli import (
-    SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE,
-    SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE,
-    SAAS_TOKEN_ENVIRONMENT_VARIABLE,
-)
+from exasol.python_extension_common.deployment.language_container_deployer_cli import SecretParams
 
 from test.utils.revert_language_settings import revert_language_settings
 from test.utils.db_utils import (create_schema, assert_udf_running)
@@ -26,26 +22,28 @@ def call_language_definition_deployer_cli(func,
                                           language_alias: str,
                                           url: str,
                                           account_id: str,
-                                          database_id: str,
                                           token: str,
-                                          connection_params: dict[str, Any],
+                                          database_id: Optional[str] = None,
+                                          database_name: Optional[str] = None,
                                           container_path: Optional[str] = None,
                                           version: Optional[str] = None,
                                           use_ssl_cert_validation: bool = False):
 
-    os.environ[SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE] = account_id
-    os.environ[SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE] = database_id
-    os.environ[SAAS_TOKEN_ENVIRONMENT_VARIABLE] = token
+    os.environ[SecretParams.SAAS_ACCOUNT_ID.name] = account_id
+    os.environ[SecretParams.SAAS_TOKEN.name] = token
+    if database_id:
+        os.environ[SecretParams.SAAS_DATABASE_ID.name] = database_id
 
     args_list = [
         "language-container",
         "--saas-url", url,
         "--path-in-bucket", "container",
-        "--dsn", connection_params['dsn'],
-        "--db-user", connection_params['user'],
-        "--db-pass", connection_params['password'],
         "--language-alias", language_alias
     ]
+    if database_name:
+        args_list += [
+            "--saas-database-name", database_name
+        ]
     if use_ssl_cert_validation:
         args_list += [
             "--use-ssl-cert-validation"
@@ -73,6 +71,7 @@ def test_language_container_deployer_cli_with_container_file(
         saas_token: str,
         saas_account_id: str,
         operational_saas_database_id: str,
+        saas_database_name: str,
         saas_connection_params: dict[str, Any],
         container_path: str,
         main_func
@@ -86,9 +85,8 @@ def test_language_container_deployer_cli_with_container_file(
                                                        language_alias=TEST_LANGUAGE_ALIAS,
                                                        url=saas_host,
                                                        account_id=saas_account_id,
-                                                       database_id=operational_saas_database_id,
-                                                       token=saas_token,
-                                                       connection_params=saas_connection_params)
+                                                       database_name=saas_database_name,
+                                                       token=saas_token)
         assert result.exit_code == 0
         assert result.exception is None
         assert result.stdout == ""