-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/main' into feature/75-bucket-fs-…
…connection-object
- Loading branch information
Showing
8 changed files
with
319 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
exasol/python_extension_common/cli/language_container_deployer_cli.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| from pathlib import Path | ||
|
|
||
| from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer | ||
| from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection | ||
| from exasol.python_extension_common.connections.bucketfs_location import create_bucketfs_location | ||
| from exasol.python_extension_common.cli.std_options import StdParams | ||
|
|
||
|
|
||
| class LanguageContainerDeployerCli: | ||
| """ | ||
| The class provides a CLI callback function that creates and runs the | ||
| LangaugeContainerDeployer. | ||
| At first glance, it may look a bit over-designed. The reason for wrapping a | ||
| callback function in a class is to let the user define the option names for the | ||
| container URL and container name. These options are not defined in the StdParams | ||
| but rather generated by formatters. The user can give them arbitrary names. | ||
| Hence, we don't want to assume any particular names in the callback function. | ||
| """ | ||
|
|
||
| def __init__(self, | ||
| container_url_arg: str | None = None, | ||
| container_name_arg: str | None = None) -> None: | ||
| self._container_url_arg = container_url_arg | ||
| self._container_name_arg = container_name_arg | ||
|
|
||
| def __call__(self, **kwargs): | ||
|
|
||
| pyexasol_connection = open_pyexasol_connection(**kwargs) | ||
| bucketfs_location = create_bucketfs_location(**kwargs) | ||
|
|
||
| language_alias = kwargs[StdParams.language_alias.name] | ||
| container_file = kwargs[StdParams.container_file.name] | ||
| upload_container = kwargs[StdParams.upload_container.name] | ||
| alter_system = kwargs[StdParams.alter_system.name] | ||
| allow_override = kwargs[StdParams.allow_override.name] | ||
| wait_for_completion = kwargs[StdParams.wait_for_completion.name] | ||
|
|
||
| deployer = LanguageContainerDeployer(pyexasol_connection, | ||
| language_alias, | ||
| bucketfs_location) | ||
| if not upload_container: | ||
| deployer.run(alter_system=alter_system, | ||
| allow_override=allow_override, | ||
| wait_for_completion=wait_for_completion) | ||
| elif container_file: | ||
| deployer.run(container_file=Path(container_file), | ||
| alter_system=alter_system, | ||
| allow_override=allow_override, | ||
| wait_for_completion=wait_for_completion) | ||
| elif kwargs.get(self._container_url_arg) and kwargs.get(self._container_name_arg): | ||
| deployer.download_and_run(kwargs[self._container_url_arg], | ||
| kwargs[self._container_name_arg], | ||
| alter_system=alter_system, | ||
| allow_override=allow_override, | ||
| wait_for_completion=wait_for_completion) | ||
| else: | ||
| raise ValueError("To upload a language container either its release version " | ||
| f"(--{StdParams.version.name}) or a path of the already " | ||
| f"downloaded container file (--{StdParams.container_file.name}) " | ||
| "must be provided.") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
192 changes: 192 additions & 0 deletions
192
test/integration/cli/test_language_container_deployer_cli.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,192 @@ | ||
| from typing import Any | ||
| from contextlib import ExitStack | ||
| from urllib.parse import urlparse | ||
| import pytest | ||
| import click | ||
| from click.testing import CliRunner | ||
|
|
||
| from exasol.python_extension_common.cli.std_options import ( | ||
| StdTags, StdParams, ParameterFormatters, select_std_options) | ||
| from exasol.python_extension_common.connections.pyexasol_connection import ( | ||
| open_pyexasol_connection) | ||
| from exasol.python_extension_common.cli.language_container_deployer_cli import ( | ||
| LanguageContainerDeployerCli) | ||
| from test.utils.db_utils import (assert_udf_running, create_schema) | ||
| from test.utils.revert_language_settings import revert_language_settings | ||
|
|
||
| CONTAINER_URL_ARG = 'container_url' | ||
| CONTAINER_NAME_ARG = 'container_name' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def onprem_cli_args(backend_aware_onprem_database, | ||
| exasol_config, | ||
| bucketfs_config, | ||
| language_alias) -> dict[str, Any]: | ||
|
|
||
| parsed_url = urlparse(bucketfs_config.url) | ||
| host, port = parsed_url.netloc.split(":") | ||
| return { | ||
| StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}', | ||
| StdParams.db_user.name: exasol_config.username, | ||
| StdParams.db_password.name: exasol_config.password, | ||
| StdParams.bucketfs_host.name: host, | ||
| StdParams.bucketfs_port.name: port, | ||
| StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https', | ||
| StdParams.bucketfs_user.name: bucketfs_config.username, | ||
| StdParams.bucketfs_password.name: bucketfs_config.password, | ||
| StdParams.bucketfs_name.name: 'bfsdefault', | ||
| StdParams.bucket.name: 'default', | ||
| StdParams.use_ssl_cert_validation.name: False, | ||
| } | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def saas_cli_args(saas_host, | ||
| saas_pat, | ||
| saas_account_id, | ||
| backend_aware_saas_database_id, | ||
| ) -> dict[str, Any]: | ||
| return { | ||
| StdParams.saas_url.name: saas_host, | ||
| StdParams.saas_account_id.name: saas_account_id, | ||
| StdParams.saas_database_id.name: backend_aware_saas_database_id, | ||
| StdParams.saas_token.name: saas_pat | ||
| } | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def slc_cli_args(language_alias) -> dict[str, Any]: | ||
| return { | ||
| StdParams.alter_system.name: True, | ||
| StdParams.allow_override.name: True, | ||
| StdParams.wait_for_completion.name: True, | ||
| StdParams.path_in_bucket.name: 'container', | ||
| StdParams.language_alias.name: language_alias | ||
| } | ||
|
|
||
|
|
||
| def create_deploy_command(backend_tag: StdTags, | ||
| container_name: str | None = None, | ||
| container_url_formatter: str | None = None) -> click.Command: | ||
| """ | ||
| This is a blueprint for creating an isolated click Command | ||
| for the language container deployment. | ||
| backend_tag should be either StdTags.ONPREM or StdTags.SAAS. | ||
| """ | ||
| if container_name and container_url_formatter: | ||
| ver_formatter = ParameterFormatters() | ||
| ver_formatter.set_formatter(CONTAINER_URL_ARG, container_url_formatter) | ||
| ver_formatter.set_formatter(CONTAINER_NAME_ARG, container_name) | ||
| formatters = {StdParams.version: ver_formatter} | ||
| else: | ||
| formatters = None | ||
|
|
||
| opts = select_std_options( | ||
| [StdTags.DB | backend_tag, StdTags.BFS | backend_tag, StdTags.SLC], | ||
| formatters=formatters) | ||
| cli_callback = LanguageContainerDeployerCli( | ||
| container_url_arg=CONTAINER_URL_ARG, | ||
| container_name_arg=CONTAINER_NAME_ARG) | ||
|
|
||
| return click.Command('deploy_slc', params=opts, callback=cli_callback) | ||
|
|
||
|
|
||
| def make_args_string(**kwargs) -> str: | ||
| def arg_string(k: str, v: Any): | ||
| k = k.replace("_", "-") | ||
| if isinstance(v, bool): | ||
| return f'--{k}' if v else f'--no-{k}' | ||
| return f'--{k} "{v}"' | ||
|
|
||
| return ' '.join(arg_string(k, v) for k, v in kwargs.items()) | ||
|
|
||
|
|
||
| def run_deploy_command(deploy_command: click.Command, | ||
| arg_string: str, | ||
| language_alias: str, | ||
| db_schema: str, | ||
| db_params: dict[str, Any]): | ||
|
|
||
| with ExitStack() as stack: | ||
| conn_before = stack.enter_context(open_pyexasol_connection(**db_params)) | ||
| stack.enter_context(revert_language_settings(conn_before)) | ||
|
|
||
| runner = CliRunner() | ||
| runner.invoke(deploy_command, args=arg_string, | ||
| catch_exceptions=False, standalone_mode=False) | ||
|
|
||
| # We have to open another connection because the language settings on | ||
| # the previously opened connection are unaffected by the slc deployment. | ||
| conn_after = stack.enter_context(open_pyexasol_connection(**db_params)) | ||
| create_schema(conn_after, db_schema) | ||
| assert_udf_running(conn_after, language_alias, db_schema) | ||
|
|
||
|
|
||
| def test_slc_deployer_cli_onprem_url(use_onprem, | ||
| container_version, | ||
| container_name, | ||
| container_url_formatter, | ||
| language_alias, | ||
| db_schema, | ||
| onprem_cli_args, | ||
| slc_cli_args): | ||
| if not use_onprem: | ||
| pytest.skip("The test is not configured to use ITDE.") | ||
|
|
||
| deploy_command = create_deploy_command(StdTags.ONPREM, | ||
| container_name=container_name, | ||
| container_url_formatter=container_url_formatter) | ||
| extra_cli_args = {StdParams.version.name: container_version} | ||
| arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args) | ||
| run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args) | ||
|
|
||
|
|
||
| def test_slc_deployer_cli_onprem_file(use_onprem, | ||
| container_path, | ||
| language_alias, | ||
| db_schema, | ||
| onprem_cli_args, | ||
| slc_cli_args): | ||
| if not use_onprem: | ||
| pytest.skip("The test is not configured to use ITDE.") | ||
|
|
||
| deploy_command = create_deploy_command(StdTags.ONPREM) | ||
| extra_cli_args = {StdParams.container_file.name: container_path} | ||
| arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args) | ||
| run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args) | ||
|
|
||
|
|
||
| def test_slc_deployer_cli_saas_url(use_saas, | ||
| container_version, | ||
| container_name, | ||
| container_url_formatter, | ||
| language_alias, | ||
| db_schema, | ||
| saas_cli_args, | ||
| slc_cli_args): | ||
| if not use_saas: | ||
| pytest.skip("The test is not configured to run in SaaS.") | ||
|
|
||
| deploy_command = create_deploy_command(StdTags.SAAS, | ||
| container_name=container_name, | ||
| container_url_formatter=container_url_formatter) | ||
| extra_cli_args = {StdParams.version.name: container_version} | ||
| arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args) | ||
| run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args) | ||
|
|
||
|
|
||
| def test_slc_deployer_cli_saas_file(use_saas, | ||
| container_path, | ||
| language_alias, | ||
| db_schema, | ||
| saas_cli_args, | ||
| slc_cli_args): | ||
| if not use_saas: | ||
| pytest.skip("The test is not configured to run in SaaS.") | ||
|
|
||
| deploy_command = create_deploy_command(StdTags.SAAS) | ||
| extra_cli_args = {StdParams.container_file.name: container_path} | ||
| arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args) | ||
| run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args) |
Oops, something went wrong.