From 7e07f193e9f0e3180245ba5ae8dd8b8d8546c939 Mon Sep 17 00:00:00 2001 From: Mikhail Beck Date: Fri, 4 Oct 2024 12:10:54 +0100 Subject: [PATCH] #70 Added LanguageContainerDeployerCli (#73) * Add documentation build folder to .gitignore * #70 Added LanguageContainerDeployerCli class * #70 Fixed the slc cli integration tests * #70 Fixed the slc cli integration tests * #70 Fixed the slc cli integration tests * #70 Fixed the slc cli integration tests * #70 Fixed the slc cli integration tests * #70 Fixed the slc cli integration tests * #70 Enabled SaaS tests * #70 Fixed saas cli tests * #70 Addressed the review comments. --- doc/changes/unreleased.md | 7 +- doc/user_guide/user-guide.md | 18 +- .../cli/language_container_deployer_cli.py | 61 ++++++ .../deployment/language_container_deployer.py | 8 +- .../language_container_deployer_cli.py | 7 + .../test_language_container_deployer_cli.py | 192 ++++++++++++++++++ test/integration/conftest.py | 32 +-- test/unit/cli/test_std_options.py | 23 +++ 8 files changed, 319 insertions(+), 29 deletions(-) create mode 100644 exasol/python_extension_common/cli/language_container_deployer_cli.py create mode 100644 test/integration/cli/test_language_container_deployer_cli.py diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index d17ac6b..f7ee722 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,9 +2,10 @@ ## Features -* #66: Implement a standard CLI options builder. -* #69: Add create_bucketfs_location function. +* #66: Implemented a standard CLI options builder. +* #70: Implemented a CLI callback function for the language container deployment. +* #69: Added create_bucketfs_location function. # Refactoring -* #68: Make open_pyexasol_connection(...) using kwargs derived from the cli options. +* #68: Made open_pyexasol_connection(...) using kwargs derived from the cli options. diff --git a/doc/user_guide/user-guide.md b/doc/user_guide/user-guide.md index e054fa3..b783574 100644 --- a/doc/user_guide/user-guide.md +++ b/doc/user_guide/user-guide.md @@ -15,10 +15,10 @@ similar to the one below. python -m .deploy language-container ``` -The name of the script (```.deploy``` in the above command) can vary from one extension to another. -Please check the user guide of a particular extension. The rest of the command line will have a common format. It -will include the command - ```language-container``` - and selected options. The choice of options is primarily -determined by the storage backend being used - On-Prem or SaaS. +The name of the script (```.deploy``` in the above command) and the command name +(e.g. ```language-container```) can vary from one extension to another. Please check the user guide of a particular +extension. The rest of the command line will have a common format. It will include some of the options defined below. +The choice of options is primarily determined by the storage backend being used - On-Prem or SaaS. ### List of options @@ -55,13 +55,13 @@ another source. | version | [x] | [x] | Optional, provide for downloading SLC from GitHub | | container-file | [x] | [x] | Optional, provide for uploading SLC file | | ssl-cert-path | [x] | [x] | Optional | -| [no_]use-ssl-cert-validation | [x] | [x] | Optional boolean, defaults to True | +| [no-]use-ssl-cert-validation | [x] | [x] | Optional boolean, defaults to True | | ssl-client-cert-path | [x] | | Optional | | ssl-client-private-key | [x] | | Optional | -| [no_]upload-container | [x] | [x] | Optional boolean, defaults to True | -| [no_]alter-system | [x] | [x] | Optional boolean, defaults to True | -| [dis]allow-override | [x] | [x] | Optional boolean, defaults to False | -| [no_]wait_for_completion | [x] | [x] | Optional boolean, defaults to True | +| [no-]upload-container | [x] | [x] | Optional boolean, defaults to True | +| [no-]alter-system | [x] | [x] | Optional boolean, defaults to True | +| [no-]allow-override | [x] | [x] | Optional boolean, defaults to False | +| [no-]wait_for_completion | [x] | [x] | Optional boolean, defaults to True | ### Container selection diff --git a/exasol/python_extension_common/cli/language_container_deployer_cli.py b/exasol/python_extension_common/cli/language_container_deployer_cli.py new file mode 100644 index 0000000..847c21e --- /dev/null +++ b/exasol/python_extension_common/cli/language_container_deployer_cli.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer +from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection +from exasol.python_extension_common.connections.bucketfs_location import create_bucketfs_location +from exasol.python_extension_common.cli.std_options import StdParams + + +class LanguageContainerDeployerCli: + """ + The class provides a CLI callback function that creates and runs the + LangaugeContainerDeployer. + + At first glance, it may look a bit over-designed. The reason for wrapping a + callback function in a class is to let the user define the option names for the + container URL and container name. These options are not defined in the StdParams + but rather generated by formatters. The user can give them arbitrary names. + Hence, we don't want to assume any particular names in the callback function. + """ + + def __init__(self, + container_url_arg: str | None = None, + container_name_arg: str | None = None) -> None: + self._container_url_arg = container_url_arg + self._container_name_arg = container_name_arg + + def __call__(self, **kwargs): + + pyexasol_connection = open_pyexasol_connection(**kwargs) + bucketfs_location = create_bucketfs_location(**kwargs) + + language_alias = kwargs[StdParams.language_alias.name] + container_file = kwargs[StdParams.container_file.name] + upload_container = kwargs[StdParams.upload_container.name] + alter_system = kwargs[StdParams.alter_system.name] + allow_override = kwargs[StdParams.allow_override.name] + wait_for_completion = kwargs[StdParams.wait_for_completion.name] + + deployer = LanguageContainerDeployer(pyexasol_connection, + language_alias, + bucketfs_location) + if not upload_container: + deployer.run(alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + elif container_file: + deployer.run(container_file=Path(container_file), + alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + elif kwargs.get(self._container_url_arg) and kwargs.get(self._container_name_arg): + deployer.download_and_run(kwargs[self._container_url_arg], + kwargs[self._container_name_arg], + alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + else: + raise ValueError("To upload a language container either its release version " + f"(--{StdParams.version.name}) or a path of the already " + f"downloaded container file (--{StdParams.container_file.name}) " + "must be provided.") diff --git a/exasol/python_extension_common/deployment/language_container_deployer.py b/exasol/python_extension_common/deployment/language_container_deployer.py index f8c6724..d33de76 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer.py +++ b/exasol/python_extension_common/deployment/language_container_deployer.py @@ -7,6 +7,7 @@ import tempfile import ssl import requests # type: ignore +import warnings import pyexasol # type: ignore import exasol.bucketfs as bfs # type: ignore from exasol.saas.client.api_access import (get_connection_params, get_database_id) # type: ignore @@ -330,7 +331,12 @@ def create(cls, use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, ssl_client_certificate: Optional[str] = None, ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer": - + warnings.warn( + "create() function is deprecated and will be removed in a future version. " + "For CLI use the LanguageContainerDeployerCli class instead.", + DeprecationWarning, + stacklevel=2 + ) # Infer where the database is - on-prem or SaaS. if all((dsn, db_user, db_password, bucketfs_host, bucketfs_port, bucketfs_name, bucket, bucketfs_user, bucketfs_password)): diff --git a/exasol/python_extension_common/deployment/language_container_deployer_cli.py b/exasol/python_extension_common/deployment/language_container_deployer_cli.py index 06c8fec..e3c2154 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer_cli.py +++ b/exasol/python_extension_common/deployment/language_container_deployer_cli.py @@ -4,6 +4,7 @@ from enum import Enum from pathlib import Path import click +import warnings from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer @@ -183,6 +184,12 @@ def language_container_deployer_main( wait_for_completion: bool, container_url: Optional[str] = None, container_name: Optional[str] = None): + warnings.warn( + "language_container_deployer_main() function is deprecated and will be removed " + "in a future version. For CLI use the LanguageContainerDeployerCli class instead.", + DeprecationWarning, + stacklevel=2 + ) deployer = LanguageContainerDeployer.create( bucketfs_name=bucketfs_name, diff --git a/test/integration/cli/test_language_container_deployer_cli.py b/test/integration/cli/test_language_container_deployer_cli.py new file mode 100644 index 0000000..6c89d01 --- /dev/null +++ b/test/integration/cli/test_language_container_deployer_cli.py @@ -0,0 +1,192 @@ +from typing import Any +from contextlib import ExitStack +from urllib.parse import urlparse +import pytest +import click +from click.testing import CliRunner + +from exasol.python_extension_common.cli.std_options import ( + StdTags, StdParams, ParameterFormatters, select_std_options) +from exasol.python_extension_common.connections.pyexasol_connection import ( + open_pyexasol_connection) +from exasol.python_extension_common.cli.language_container_deployer_cli import ( + LanguageContainerDeployerCli) +from test.utils.db_utils import (assert_udf_running, create_schema) +from test.utils.revert_language_settings import revert_language_settings + +CONTAINER_URL_ARG = 'container_url' +CONTAINER_NAME_ARG = 'container_name' + + +@pytest.fixture(scope='session') +def onprem_cli_args(backend_aware_onprem_database, + exasol_config, + bucketfs_config, + language_alias) -> dict[str, Any]: + + parsed_url = urlparse(bucketfs_config.url) + host, port = parsed_url.netloc.split(":") + return { + StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}', + StdParams.db_user.name: exasol_config.username, + StdParams.db_password.name: exasol_config.password, + StdParams.bucketfs_host.name: host, + StdParams.bucketfs_port.name: port, + StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https', + StdParams.bucketfs_user.name: bucketfs_config.username, + StdParams.bucketfs_password.name: bucketfs_config.password, + StdParams.bucketfs_name.name: 'bfsdefault', + StdParams.bucket.name: 'default', + StdParams.use_ssl_cert_validation.name: False, + } + + +@pytest.fixture(scope='session') +def saas_cli_args(saas_host, + saas_pat, + saas_account_id, + backend_aware_saas_database_id, + ) -> dict[str, Any]: + return { + StdParams.saas_url.name: saas_host, + StdParams.saas_account_id.name: saas_account_id, + StdParams.saas_database_id.name: backend_aware_saas_database_id, + StdParams.saas_token.name: saas_pat + } + + +@pytest.fixture(scope='session') +def slc_cli_args(language_alias) -> dict[str, Any]: + return { + StdParams.alter_system.name: True, + StdParams.allow_override.name: True, + StdParams.wait_for_completion.name: True, + StdParams.path_in_bucket.name: 'container', + StdParams.language_alias.name: language_alias + } + + +def create_deploy_command(backend_tag: StdTags, + container_name: str | None = None, + container_url_formatter: str | None = None) -> click.Command: + """ + This is a blueprint for creating an isolated click Command + for the language container deployment. + + backend_tag should be either StdTags.ONPREM or StdTags.SAAS. + """ + if container_name and container_url_formatter: + ver_formatter = ParameterFormatters() + ver_formatter.set_formatter(CONTAINER_URL_ARG, container_url_formatter) + ver_formatter.set_formatter(CONTAINER_NAME_ARG, container_name) + formatters = {StdParams.version: ver_formatter} + else: + formatters = None + + opts = select_std_options( + [StdTags.DB | backend_tag, StdTags.BFS | backend_tag, StdTags.SLC], + formatters=formatters) + cli_callback = LanguageContainerDeployerCli( + container_url_arg=CONTAINER_URL_ARG, + container_name_arg=CONTAINER_NAME_ARG) + + return click.Command('deploy_slc', params=opts, callback=cli_callback) + + +def make_args_string(**kwargs) -> str: + def arg_string(k: str, v: Any): + k = k.replace("_", "-") + if isinstance(v, bool): + return f'--{k}' if v else f'--no-{k}' + return f'--{k} "{v}"' + + return ' '.join(arg_string(k, v) for k, v in kwargs.items()) + + +def run_deploy_command(deploy_command: click.Command, + arg_string: str, + language_alias: str, + db_schema: str, + db_params: dict[str, Any]): + + with ExitStack() as stack: + conn_before = stack.enter_context(open_pyexasol_connection(**db_params)) + stack.enter_context(revert_language_settings(conn_before)) + + runner = CliRunner() + runner.invoke(deploy_command, args=arg_string, + catch_exceptions=False, standalone_mode=False) + + # We have to open another connection because the language settings on + # the previously opened connection are unaffected by the slc deployment. + conn_after = stack.enter_context(open_pyexasol_connection(**db_params)) + create_schema(conn_after, db_schema) + assert_udf_running(conn_after, language_alias, db_schema) + + +def test_slc_deployer_cli_onprem_url(use_onprem, + container_version, + container_name, + container_url_formatter, + language_alias, + db_schema, + onprem_cli_args, + slc_cli_args): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + deploy_command = create_deploy_command(StdTags.ONPREM, + container_name=container_name, + container_url_formatter=container_url_formatter) + extra_cli_args = {StdParams.version.name: container_version} + arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args) + run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args) + + +def test_slc_deployer_cli_onprem_file(use_onprem, + container_path, + language_alias, + db_schema, + onprem_cli_args, + slc_cli_args): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + deploy_command = create_deploy_command(StdTags.ONPREM) + extra_cli_args = {StdParams.container_file.name: container_path} + arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args) + run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args) + + +def test_slc_deployer_cli_saas_url(use_saas, + container_version, + container_name, + container_url_formatter, + language_alias, + db_schema, + saas_cli_args, + slc_cli_args): + if not use_saas: + pytest.skip("The test is not configured to run in SaaS.") + + deploy_command = create_deploy_command(StdTags.SAAS, + container_name=container_name, + container_url_formatter=container_url_formatter) + extra_cli_args = {StdParams.version.name: container_version} + arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args) + run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args) + + +def test_slc_deployer_cli_saas_file(use_saas, + container_path, + language_alias, + db_schema, + saas_cli_args, + slc_cli_args): + if not use_saas: + pytest.skip("The test is not configured to run in SaaS.") + + deploy_command = create_deploy_command(StdTags.SAAS) + extra_cli_args = {StdParams.container_file.name: container_path} + arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args) + run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 0cf352f..ec4b538 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -14,27 +14,32 @@ from test.utils.revert_language_settings import revert_language_settings from test.utils.db_utils import create_schema, open_schema - -SLC_NAME = "template-Exasol-all-python-3.10_release.tar.gz" - -SLC_URL_FORMATTER = ("https://github.com/exasol/script-languages-release/releases/" - "download/{version}/") + SLC_NAME - VERSION = "8.0.0" TEST_SCHEMA = "PEC_DEPLOYER_TESTS" TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS" +@pytest.fixture(scope='session') +def container_name() -> str: + return "template-Exasol-all-python-3.10_release.tar.gz" + + +@pytest.fixture(scope='session') +def container_url_formatter(container_name) -> str: + return ("https://github.com/exasol/script-languages-release/releases/" + "download/{version}/") + container_name + + @pytest.fixture -def main_func(): +def main_func(slc_name, slc_url_formatter): @click.group() def fake_main(): pass - slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, SLC_URL_FORMATTER) - slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, SLC_NAME) + slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, container_url_formatter) + slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, container_name) fake_main.add_command(language_container_deployer_main) return fake_main @@ -46,13 +51,8 @@ def container_version() -> str: @pytest.fixture(scope='session') -def container_name() -> str: - return SLC_NAME - - -@pytest.fixture(scope='session') -def container_url(container_version) -> str: - return SLC_URL_FORMATTER.format(version=VERSION) +def container_url(container_url_formatter, container_version) -> str: + return container_url_formatter.format(version=container_version) @pytest.fixture(scope='session') diff --git a/test/unit/cli/test_std_options.py b/test/unit/cli/test_std_options.py index f68ddef..45ff517 100644 --- a/test/unit/cli/test_std_options.py +++ b/test/unit/cli/test_std_options.py @@ -102,6 +102,29 @@ def test_select_std_options_with_override(): assert not opts[StdParams.alter_system.name].default +def test_select_std_options_with_formatter(): + container_url_arg = 'container_url' + container_name_arg = 'container_name' + url_format = "https://my_service_url/{version}/page" + name_format = "my_service_name" + version = '4.5.6' + expected_url = url_format.format(version=version) + expected_name = name_format + + def func(**kwargs): + assert kwargs[container_name_arg] == expected_name + assert kwargs[container_url_arg] == expected_url + + ver_formatter = ParameterFormatters() + ver_formatter.set_formatter(container_url_arg, url_format) + ver_formatter.set_formatter(container_name_arg, name_format) + + opts = select_std_options(StdTags.SLC, formatters={StdParams.version: ver_formatter}) + cmd = click.Command('do_something', params=opts, callback=func) + runner = CliRunner() + runner.invoke(cmd, args=f'--version {version}', catch_exceptions=False, standalone_mode=False) + + def test_hidden_opt_with_envar(): """ This test checks the mechanism of providing a value of a confidential parameter