diff --git a/exasol/python_extension_common/deployment/language_container_deployer.py b/exasol/python_extension_common/deployment/language_container_deployer.py index abd52ab..bbe535f 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer.py +++ b/exasol/python_extension_common/deployment/language_container_deployer.py @@ -8,13 +8,11 @@ import requests # type: ignore import pyexasol # type: ignore import exasol.bucketfs as bfs # type: ignore -from exasol.saas.client.api_access import get_connection_params # type: ignore +from exasol.saas.client.api_access import (get_connection_params, get_database_id) # type: ignore logger = logging.getLogger(__name__) -ARCHIVE_EXTENSIONS = [".tar.gz", ".tgz", ".zip", ".tar"] - def get_websocket_sslopt(use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, @@ -76,17 +74,11 @@ def get_language_settings(pyexasol_conn: pyexasol.ExaConnection, alter_type: Lan def get_udf_path(bucket_base_path: bfs.path.PathLike, bucket_file: str) -> PurePosixPath: """ Returns the path of the specified file in a bucket, as it's seen from a UDF - For known types of archives removes the archive extension from the file name. bucket_base_path - Base directory in the bucket bucket_file - File path in the bucket, relative to the base directory. """ - for extension in ARCHIVE_EXTENSIONS: - if bucket_file.endswith(extension): - bucket_file = bucket_file[: -len(extension)] - break - file_path = bucket_base_path / bucket_file return PurePosixPath(file_path.as_udf_path()) @@ -278,7 +270,7 @@ def create(cls, bucketfs_use_https: bool = True, saas_url: Optional[str] = None, saas_account_id: Optional[str] = None, saas_database_id: Optional[str] = None, - saas_token: Optional[str] = None, + saas_database_name: Optional[str] = None, saas_token: Optional[str] = None, path_in_bucket: str = '', use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, ssl_client_certificate: Optional[str] = None, @@ -300,11 +292,20 @@ def create(cls, verify=verify, path=path_in_bucket) - elif all((saas_url, saas_account_id, saas_database_id, saas_token)): + elif all((saas_url, saas_account_id, saas_token, + any((saas_database_id, saas_database_name)))): connection_params = get_connection_params(host=saas_url, account_id=saas_account_id, database_id=saas_database_id, + database_name=saas_database_name, pat=saas_token) + saas_database_id = (saas_database_id or + get_database_id( + host=saas_url, + account_id=saas_account_id, + pat=saas_token, + database_name=saas_database_name + )) bucketfs_path = bfs.path.build_path(backend=bfs.path.StorageBackend.saas, url=saas_url, account_id=saas_account_id, diff --git a/exasol/python_extension_common/deployment/language_container_deployer_cli.py b/exasol/python_extension_common/deployment/language_container_deployer_cli.py index 6a91c38..f3d2e3b 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer_cli.py +++ b/exasol/python_extension_common/deployment/language_container_deployer_cli.py @@ -6,12 +6,6 @@ import click from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer -DB_PASSWORD_ENVIRONMENT_VARIABLE = "DB_PASSWORD" -BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = "BUCKETFS_PASSWORD" -SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE = "SAAS_ACCOUNT_ID" -SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE = "SAAS_DATABASE_ID" -SAAS_TOKEN_ENVIRONMENT_VARIABLE = "SAAS_TOKEN" - class CustomizableParameters(Enum): """ @@ -82,23 +76,63 @@ def clear_formatters(self): slc_parameter_formatters = _ParameterFormatters() +# This text will be displayed instead of the actual value, if found in an environment +# variable, in a prompt. +SECRET_DISPLAY = '***' + + +class SecretParams(Enum): + """ + This enum serves as a definition of confidential parameters which values should not be + displayed in the console, unless the user types them in the command line. + + The enum name is also the name of the environment variable where the correspondent + secret value can be stored. + + The enum value is also the name of the cli parameter. + """ + DB_PASSWORD = 'db-pass' + BUCKETFS_PASSWORD = 'bucketfs-password' + SAAS_ACCOUNT_ID = 'saas-account-id' + SAAS_DATABASE_ID = 'saas-database-id' + SAAS_TOKEN = 'saas-token' + + +def secret_callback(ctx: click.Context, param: click.Option, value: Any): + """ + Here we try to get the secret parameter value from an environment variable. + The reason for doing this in the callback instead of using a callable default is + that we don't want the default to be displayed in the prompt. There seems to + be no way of altering this behaviour. + """ + if value == SECRET_DISPLAY: + secret_param = SecretParams(param.opts[0][2:]) + return os.environ.get(secret_param.name) + return value + + @click.command(name="language-container") @click.option('--bucketfs-name', type=str) @click.option('--bucketfs-host', type=str) @click.option('--bucketfs-port', type=int) @click.option('--bucketfs-use-https', type=bool, default=False) @click.option('--bucketfs-user', type=str) -@click.option('--bucketfs-password', type=str, - default=lambda: os.environ.get(BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE)) +@click.option(f'--{SecretParams.BUCKETFS_PASSWORD.value}', type=str, + prompt='BucketFS password', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) @click.option('--bucket', type=str) @click.option('--saas-url', type=str, default='https://cloud.exasol.com') -@click.option('--saas-account-id', type=str, - default=lambda: os.environ.get(SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE)) -@click.option('--saas-database-id', type=str, - default=lambda: os.environ.get(SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE)) -@click.option('--saas-token', type=str, - default=lambda: os.environ.get(SAAS_TOKEN_ENVIRONMENT_VARIABLE)) +@click.option(f'--{SecretParams.SAAS_ACCOUNT_ID.value}', type=str, + prompt='SaaS account id', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option(f'--{SecretParams.SAAS_DATABASE_ID.value}', type=str, + prompt='SaaS database id', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option('--saas-database-name', type=str) +@click.option(f'--{SecretParams.SAAS_TOKEN.value}', type=str, + prompt='SaaS token', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) @click.option('--path-in-bucket', type=str) @click.option('--container-file', type=click.Path(exists=True, file_okay=True)) @@ -106,8 +140,9 @@ def clear_formatters(self): callback=slc_parameter_formatters) @click.option('--dsn', type=str) @click.option('--db-user', type=str) -@click.option('--db-pass', - default=lambda: os.environ.get(DB_PASSWORD_ENVIRONMENT_VARIABLE)) +@click.option(f'--{SecretParams.DB_PASSWORD.value}', type=str, + prompt='DB password', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) @click.option('--language-alias', type=str, default="PYTHON3_EXT") @click.option('--ssl-cert-path', type=str, default="") @click.option('--ssl-client-cert-path', type=str, default="") @@ -127,6 +162,7 @@ def language_container_deployer_main( saas_url: str, saas_account_id: str, saas_database_id: str, + saas_database_name: str, saas_token: str, path_in_bucket: str, container_file: str, @@ -155,6 +191,7 @@ def language_container_deployer_main( saas_url=saas_url, saas_account_id=saas_account_id, saas_database_id=saas_database_id, + saas_database_name=saas_database_name, saas_token=saas_token, path_in_bucket=path_in_bucket, dsn=dsn, diff --git a/poetry.lock b/poetry.lock index db3d654..5ceb7ab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -636,13 +636,13 @@ files = [ [[package]] name = "exasol-bucketfs" -version = "0.10.0" +version = "0.11.0" description = "BucketFS utilities for the Python programming language" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "exasol_bucketfs-0.10.0-py3-none-any.whl", hash = "sha256:4f5aa81c31c5e03f19daa04d8b455ed09740f9e82bc53f3f9cb47db025146625"}, - {file = "exasol_bucketfs-0.10.0.tar.gz", hash = "sha256:033ee923728037af4d7771d9c6855e9eed2389d842c98a8456f937c917c395f8"}, + {file = "exasol_bucketfs-0.11.0-py3-none-any.whl", hash = "sha256:9eb42c6df5804aa104646e141bd22f4f85c43cc9d563254f9b9ab7293a574120"}, + {file = "exasol_bucketfs-0.11.0.tar.gz", hash = "sha256:6dc2639336816dc57383095eafbfd811ade737f9290191ee5cefa55a22f581df"}, ] [package.dependencies] @@ -690,20 +690,24 @@ name = "exasol-saas-api" version = "0.6.0" description = "API enabling Python applications connecting to Exasol database SaaS instances and using their SaaS services" optional = false -python-versions = "<4.0,>=3.8.0" -files = [ - {file = "exasol_saas_api-0.6.0-py3-none-any.whl", hash = "sha256:d6ff6501e10e97352459cd853f47053ed3affcc6d7c733debbe80b3f8c709aaa"}, - {file = "exasol_saas_api-0.6.0.tar.gz", hash = "sha256:195ad5aaf15be270838e08b2e4a9fddb7981faf33ee00ed68042c5707d90612f"}, -] +python-versions = ">=3.8.0,<4.0" +files = [] +develop = false [package.dependencies] attrs = ">=21.3.0" httpx = ">=0.20.0,<0.28.0" -ifaddr = ">=0.2.0,<0.3.0" -python-dateutil = ">=2.8.0,<3.0.0" -requests = ">=2.31.0,<3.0.0" -tenacity = ">=8.2.3,<9.0.0" -types-requests = ">=2.31.0.6,<3.0.0.0" +ifaddr = "^0.2.0" +python-dateutil = "^2.8.0" +requests = "^2.31.0" +tenacity = "^8.2.3" +types-requests = "^2.31.0.6" + +[package.source] +type = "git" +url = "https://github.com/exasol/saas-api-python.git" +reference = "main" +resolved_reference = "8831dfd0b49ebde29ec5181dbf2e46e31132c1e5" [[package]] name = "exasol-script-languages-container-tool" @@ -2447,13 +2451,13 @@ typing-extensions = ">=3.7.4.3" [[package]] name = "types-requests" -version = "2.32.0.20240521" +version = "2.32.0.20240523" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.32.0.20240521.tar.gz", hash = "sha256:c5c4a0ae95aad51f1bf6dae9eed04a78f7f2575d4b171da37b622e08b93eb5d3"}, - {file = "types_requests-2.32.0.20240521-py3-none-any.whl", hash = "sha256:ab728ba43ffb073db31f21202ecb97db8753ded4a9dc49cb480d8a5350c5c421"}, + {file = "types-requests-2.32.0.20240523.tar.gz", hash = "sha256:26b8a6de32d9f561192b9942b41c0ab2d8010df5677ca8aa146289d11d505f57"}, + {file = "types_requests-2.32.0.20240523-py3-none-any.whl", hash = "sha256:f19ed0e2daa74302069bbbbf9e82902854ffa780bc790742a810a9aaa52f65ec"}, ] [package.dependencies] @@ -2461,13 +2465,13 @@ urllib3 = ">=2" [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, ] [[package]] @@ -2620,4 +2624,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.8.0,<4.0" -content-hash = "6dbd36baea42a138ca5cd8324193a426b5f8abce16ae802ab0ac9906f736f9e7" +content-hash = "8950b3d4fac07fdc12731ae14951470162800bb234aa60abea842e4430ec438f" diff --git a/pyproject.toml b/pyproject.toml index 979ed2c..5be4f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ readme = "README.md" python = ">=3.8.0,<4.0" pyexasol = "^0.25.0" exasol-bucketfs = ">=0.10.0" -click = "^8.0.4" -exasol-saas-api = ">=0.6.0" +click = "^8.1.7" +exasol-saas-api = {git = 'https://github.com/exasol/saas-api-python.git', branch = 'main'} requests = "<2.32.0" [tool.poetry.group.dev.dependencies] diff --git a/test/integration/conftest.py b/test/integration/conftest.py index d5ecfa3..2b06b0d 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -11,11 +11,6 @@ OpenApiAccess, get_connection_params ) -from exasol.saas.client.openapi.models import CreateAllowedIP -from exasol.saas.client.openapi.api.security.add_allowed_ip import sync as add_allowed_ip -from exasol.saas.client.openapi.api.security.delete_allowed_ip import sync_detailed as delete_allowed_ip -from exasol.saas.client.openapi.api.clusters.list_clusters import sync as list_clusters -from exasol.saas.client.openapi.api.clusters.get_cluster_connection import sync as get_cluster_connection from exasol.python_extension_common.deployment.language_container_deployer_cli import ( language_container_deployer_main, slc_parameter_formatters, CustomizableParameters) @@ -99,9 +94,13 @@ def api_access(saas_host, saas_token, saas_account_id) -> OpenApiAccess: @pytest.fixture(scope="session") -def operational_saas_database_id(api_access) -> str: - database_name = timestamp_name('PEC') - with api_access.database(database_name) as db: +def saas_database_name() -> str: + return timestamp_name('PEC') + + +@pytest.fixture(scope="session") +def operational_saas_database_id(api_access, saas_database_name) -> str: + with api_access.database(saas_database_name) as db: api_access.wait_until_running(db.id) yield db.id diff --git a/test/integration/test_language_container_deployer_saas_cli.py b/test/integration/test_language_container_deployer_saas_cli.py index 0adbb0c..4c36584 100644 --- a/test/integration/test_language_container_deployer_saas_cli.py +++ b/test/integration/test_language_container_deployer_saas_cli.py @@ -8,11 +8,7 @@ from click.testing import CliRunner import pyexasol -from exasol.python_extension_common.deployment.language_container_deployer_cli import ( - SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE, - SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE, - SAAS_TOKEN_ENVIRONMENT_VARIABLE, -) +from exasol.python_extension_common.deployment.language_container_deployer_cli import SecretParams from test.utils.revert_language_settings import revert_language_settings from test.utils.db_utils import (create_schema, assert_udf_running) @@ -26,26 +22,28 @@ def call_language_definition_deployer_cli(func, language_alias: str, url: str, account_id: str, - database_id: str, token: str, - connection_params: dict[str, Any], + database_id: Optional[str] = None, + database_name: Optional[str] = None, container_path: Optional[str] = None, version: Optional[str] = None, use_ssl_cert_validation: bool = False): - os.environ[SAAS_ACCOUNT_ID_ENVIRONMENT_VARIABLE] = account_id - os.environ[SAAS_DATABASE_ID_ENVIRONMENT_VARIABLE] = database_id - os.environ[SAAS_TOKEN_ENVIRONMENT_VARIABLE] = token + os.environ[SecretParams.SAAS_ACCOUNT_ID.name] = account_id + os.environ[SecretParams.SAAS_TOKEN.name] = token + if database_id: + os.environ[SecretParams.SAAS_DATABASE_ID.name] = database_id args_list = [ "language-container", "--saas-url", url, "--path-in-bucket", "container", - "--dsn", connection_params['dsn'], - "--db-user", connection_params['user'], - "--db-pass", connection_params['password'], "--language-alias", language_alias ] + if database_name: + args_list += [ + "--saas-database-name", database_name + ] if use_ssl_cert_validation: args_list += [ "--use-ssl-cert-validation" @@ -73,6 +71,7 @@ def test_language_container_deployer_cli_with_container_file( saas_token: str, saas_account_id: str, operational_saas_database_id: str, + saas_database_name: str, saas_connection_params: dict[str, Any], container_path: str, main_func @@ -86,9 +85,8 @@ def test_language_container_deployer_cli_with_container_file( language_alias=TEST_LANGUAGE_ALIAS, url=saas_host, account_id=saas_account_id, - database_id=operational_saas_database_id, - token=saas_token, - connection_params=saas_connection_params) + database_name=saas_database_name, + token=saas_token) assert result.exit_code == 0 assert result.exception is None assert result.stdout == ""