diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index e8b5d2d..a5acecb 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,10 +2,11 @@ ## Features -* #79 Added function get_cli_arg that makes a string CLI argument from an option and its value. - Also allowed passing an option name instead of the StdParams in the following two functions: - create_std_option, check_params. +* #52: Added timeout options for SLC deployer to CLI +* #79 Added function `get_cli_arg` that makes a string CLI argument from an option and its value. + Also allowed passing an option name instead of the `StdParams` in the following two functions: + `create_std_option`, `check_params`. ## Bug fixing -* #78 Missing default value in the definition of StdParams.path_in_bucket. +* #78 Missing default value in the definition of `StdParams.path_in_bucket`. diff --git a/exasol/python_extension_common/cli/std_options.py b/exasol/python_extension_common/cli/std_options.py index b13ab8d..38534c4 100644 --- a/exasol/python_extension_common/cli/std_options.py +++ b/exasol/python_extension_common/cli/std_options.py @@ -113,6 +113,8 @@ class StdParams(Enum): alter_system = (StdTags.SLC, auto()) allow_override = (StdTags.SLC, auto()) wait_for_completion = (StdTags.SLC, auto()) + deploy_timeout_minutes = (StdTags.SLC, auto()) + display_progress = (StdTags.SLC, auto()) def __init__(self, tags: StdTags, value): self.tags = tags @@ -157,7 +159,9 @@ def _get_param_name(std_param: StdParamOrName) -> str: StdParams.upload_container: {'type': bool, 'default': True}, StdParams.alter_system: {'type': bool, 'default': True}, StdParams.allow_override: {'type': bool, 'default': False}, - StdParams.wait_for_completion: {'type': bool, 'default': True} + StdParams.wait_for_completion: {'type': bool, 'default': True}, + StdParams.deploy_timeout_minutes: {'type': int, 'default': 10}, + StdParams.display_progress: {'type': bool, 'default': True}, } diff --git a/exasol/python_extension_common/deployment/extract_validator.py b/exasol/python_extension_common/deployment/extract_validator.py index f096051..2280f41 100644 --- a/exasol/python_extension_common/deployment/extract_validator.py +++ b/exasol/python_extension_common/deployment/extract_validator.py @@ -45,7 +45,7 @@ class ExtractValidator: def __init__(self, pyexasol_connection: pyexasol.ExaConnection, timeout: timedelta, - interval: timedelta = timedelta(seconds=10), + interval: timedelta = timedelta(seconds=30), callback: Callable[[int, List[int]], None] | None = None, ) -> None: self._pyexasol_conn = pyexasol_connection diff --git a/exasol/python_extension_common/deployment/language_container_deployer.py b/exasol/python_extension_common/deployment/language_container_deployer.py index d33de76..950f4c4 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer.py +++ b/exasol/python_extension_common/deployment/language_container_deployer.py @@ -88,6 +88,10 @@ def get_udf_path(bucket_base_path: bfs.path.PathLike, bucket_file: str) -> PureP return PurePosixPath(file_path.as_udf_path()) +def display_extract_progress(n: int, pending: List[int]): + logger.info(f"Verify extraction: {len(pending)} of {n} nodes pending, IDs: {pending}") + + class LanguageContainerDeployer: def __init__(self, @@ -105,8 +109,8 @@ def __init__(self, else: self._extract_validator = ExtractValidator( pyexasol_connection, - timeout=timedelta(minutes=5), - interval=timedelta(seconds=10), + timeout=timedelta(minutes=10), + interval=timedelta(seconds=30), ) logger.debug("Init %s", LanguageContainerDeployer.__name__) @@ -330,7 +334,10 @@ def create(cls, path_in_bucket: str = '', use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, ssl_client_certificate: Optional[str] = None, - ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer": + ssl_private_key: Optional[str] = None, + deploy_timeout: timedelta = timedelta(minutes=10), + display_progress: bool = False, + ) -> "LanguageContainerDeployer": warnings.warn( "create() function is deprecated and will be removed in a future version. " "For CLI use the LanguageContainerDeployerCli class instead.", @@ -389,4 +396,6 @@ def create(cls, encryption=True, websocket_sslopt=websocket_sslopt) - return cls(pyexasol_conn, language_alias, bucketfs_path) + callback = display_extract_progress if display_progress else None + extract_validator = ExtractValidator(pyexasol_conn, deploy_timeout, callback=callback) + return cls(pyexasol_conn, language_alias, bucketfs_path, extract_validator) diff --git a/exasol/python_extension_common/deployment/language_container_deployer_cli.py b/exasol/python_extension_common/deployment/language_container_deployer_cli.py index e3c2154..588fead 100644 --- a/exasol/python_extension_common/deployment/language_container_deployer_cli.py +++ b/exasol/python_extension_common/deployment/language_container_deployer_cli.py @@ -6,6 +6,7 @@ import click import warnings from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer +from datetime import timedelta class CustomizableParameters(Enum): @@ -154,6 +155,8 @@ def secret_callback(ctx: click.Context, param: click.Option, value: Any): @click.option('--alter-system/--no-alter-system', type=bool, default=True) @click.option('--allow-override/--disallow-override', type=bool, default=False) @click.option('--wait_for_completion/--no-wait_for_completion', type=bool, default=True) +@click.option('--deploy-timeout-minutes', type=int, default=10) +@click.option('--display-progress/--no-display-progress', type=bool, default=True) def language_container_deployer_main( bucketfs_name: str, bucketfs_host: str, @@ -182,6 +185,8 @@ def language_container_deployer_main( alter_system: bool, allow_override: bool, wait_for_completion: bool, + deploy_timeout_minutes: int, + display_progress: bool, container_url: Optional[str] = None, container_name: Optional[str] = None): warnings.warn( @@ -213,7 +218,10 @@ def language_container_deployer_main( ssl_trusted_ca=ssl_cert_path, ssl_client_certificate=ssl_client_cert_path, ssl_private_key=ssl_client_private_key, - use_ssl_cert_validation=use_ssl_cert_validation) + use_ssl_cert_validation=use_ssl_cert_validation, + deploy_timeout=timedelta(minutes=deploy_timeout_minutes), + display_progress=display_progress, + ) if not upload_container: deployer.run(alter_system=alter_system, allow_override=allow_override, diff --git a/test/unit/deployment/test_cli.py b/test/unit/deployment/test_cli.py index 1fe20ec..93ba872 100644 --- a/test/unit/deployment/test_cli.py +++ b/test/unit/deployment/test_cli.py @@ -13,6 +13,7 @@ SecretParams, ) from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer +from datetime import timedelta class CliRunner: @@ -62,6 +63,7 @@ def __init__( self, api: str, value: any = None, + cli_value: any = None, cli: str = None, env: str = None, prompt: str = None, @@ -69,6 +71,7 @@ def __init__( self.api_kwarg = api self.value = f"om_value_{api}" if value is None else value self.cli = cli or "--" + api.replace("_", "-") + self.cli_value = self.value if cli_value is None else cli_value self.env = env self.prompt = prompt @@ -144,6 +147,9 @@ def test_default_values(container_file): OptionMapper("ssl_client_certificate", "", cli="--ssl-client-cert-path"), OptionMapper("ssl_private_key", "", cli="--ssl-client-private-key"), OptionMapper("use_ssl_cert_validation", True), + OptionMapper("deploy_timeout", cli="--deploy-timeout-minutes", + cli_value=5, value=timedelta(minutes=10)), + OptionMapper("display_progress", True), ] deployer = create_autospec(LanguageContainerDeployer) with CliRunner(deployer) as runner: @@ -187,11 +193,9 @@ def test_cli_options_passed_to_create(container_file): OptionMapper("ssl_client_certificate", cli="--ssl-client-cert-path"), OptionMapper("ssl_private_key", cli="--ssl-client-private-key"), OptionMapper("use_ssl_cert_validation", False), - # For the following two arguments to - # LanguageContainerDeployer.create() there are no corresponding CLI - # options defined: - # - container_url: Optional[str] = None, - # - container_name: Optional[str] = None): + OptionMapper("deploy_timeout", cli="--deploy-timeout-minutes", + cli_value=6, value=timedelta(minutes=6)), + OptionMapper("display_progress", False), ] def keys_and_values(): for o in options: @@ -199,7 +203,7 @@ def keys_and_values(): yield "--no-" + o.cli[2:] continue yield o.cli - yield str(o.value) + yield str(o.cli_value) cli_options = list(keys_and_values()) deployer = create_autospec(LanguageContainerDeployer) @@ -250,17 +254,15 @@ def test_container_file(): "Covered by test_default_values()" -@pytest.mark.skip(reason="Not implemented, yet") +@pytest.mark.skip(reason="Test case to be implemented in derived applications") def test_container_url(): """ For the following two arguments to LanguageContainerDeployer.create() - there are no corresponding CLI options defined: + the corresponding CLI options need to defined in actual CLI applications + using ``language_container_deployer_main()`` - container_url: Optional[str] = None, - container_name: Optional[str] = None): - Hence this test case currently cannot be run. + Hence there is no related test case in the current file. """ - -# Additionally there seems to be a file main.py missing that is wrapping the -# command line.