Added SLC deployment integration tests

doc/changes/changelog.md

@@ -1 +1,11 @@
+# 📝 Changes
 
+* [unreleased](unreleased.md)
+* [0.1.0](changes_0.1.0.md)
+
+```{toctree}
+---
+hidden:
+---
+unreleased
+changes_0.1.0

doc/changes/changes_0.1.0.md

@@ -1 +1,10 @@
+# Python Extension Common 0.1.0, released T.B.D.
 
+## Summary
+
+Initial commit.
+
+## Changes
+* Added LanguageContainerDeployer class
+* Added language_container_deployer_main - the SLC deployment CLI.
+* Added unit test for the SLC deployer and its cli.

doc/changes/unreleased.md

@@ -0,0 +1,5 @@
+# Unreleased
+
+## Added
+    - Integration tests for the LanguageContainerDeployer class
+      and the cli function - language_container_deployer_main.

pyproject.toml

@@ -14,12 +14,18 @@ exasol-bucketfs = "^0.9.0"
 click = "^8.0.4"
 
 [tool.poetry.group.dev.dependencies]
+pytest = "^7.2.0"
 exasol-toolbox = "^0.8.0"
+exasol-script-languages-container-tool = "^0.18.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
+testpaths = [
+    "test"
+]
+
 [tool.coverage.run]
 relative_files = true
 source = [
@@ -52,8 +58,7 @@ max-module-lines = 800
 [[tool.mypy.overrides]]
 module = [
     "exasol.toolbox.nox.tasks",
-    "test.unit.*",
-    "test.integration.*",
+    "test.*",
 ]
 ignore_errors = true
 ignore_missing_imports = true

test/__init__.py

@@ -0,0 +1,4 @@
+import pytest
+
+
+pytest.register_assert_rewrite("test.utils.db_utils")

test/integration/conftest.py

@@ -0,0 +1,55 @@
+import pytest
+import click
+import requests
+
+from exasol.python_extension_common.deployment.language_container_deployer_cli import (
+    language_container_deployer_main, slc_parameter_formatters, CustomizableParameters)
+
+
+SLC_NAME = "template-Exasol-all-python-3.10_release.tar.gz"
+
+SLC_URL_FORMATTER = ("https://github.com/exasol/script-languages-release/releases/"
+                     "download/{version}/") + SLC_NAME
+
+VERSION = "8.0.0"
+
+
+@pytest.fixture
+def main_func():
+
+    @click.group()
+    def fake_main():
+        pass
+
+    slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, SLC_URL_FORMATTER)
+    slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, SLC_NAME)
+
+    fake_main.add_command(language_container_deployer_main)
+    return fake_main
+
+
+@pytest.fixture(scope='session')
+def container_version() -> str:
+    return VERSION
+
+
+@pytest.fixture(scope='session')
+def container_name() -> str:
+    return SLC_NAME
+
+
+@pytest.fixture(scope='session')
+def container_url(container_version) -> str:
+    return SLC_URL_FORMATTER.format(version=VERSION)
+
+
+@pytest.fixture(scope='session')
+def container_path(tmpdir_factory, container_url, container_name) -> str:
+
+    response = requests.get(container_url, allow_redirects=True)
+    response.raise_for_status()
+    slc_path = tmpdir_factory.mktemp('container').join(container_name)
+    slc_path = str(slc_path)
+    with open(slc_path, 'wb') as f:
+        f.write(response.content)
+    return slc_path

test/integration/test_language_container_deployer.py

@@ -0,0 +1,100 @@
+from typing import Callable
+from contextlib import ExitStack
+from pathlib import Path
+
+import pytest
+
+from pyexasol import ExaConnection
+from pytest_itde import config
+from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory
+
+from exasol.python_extension_common.deployment.language_container_deployer import (
+    LanguageContainerDeployer, LanguageActivationLevel)
+
+from test.utils.revert_language_settings import revert_language_settings
+from test.utils.db_utils import (create_schema, assert_udf_running)
+
+TEST_SCHEMA = "PEC_DEPLOYER_TESTS"
+TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS"
+
+
+def create_container_deployer(language_alias: str,
+                              pyexasol_connection: ExaConnection,
+                              bucketfs_config: config.BucketFs) -> LanguageContainerDeployer:
+    bucket_fs_factory = BucketFSFactory()
+    bucketfs_location = bucket_fs_factory.create_bucketfs_location(
+        url=f"{bucketfs_config.url}/default/container;bfsdefault",
+        user=f"{bucketfs_config.username}",
+        pwd=f"{bucketfs_config.password}",
+        base_path=None)
+    return LanguageContainerDeployer(
+        pyexasol_connection, language_alias, bucketfs_location)
+
+
+def test_language_container_deployer(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_path: str):
+    """
+    Tests the deployment of a container in one call, including the activation at the System level.
+    """
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        deployer = create_container_deployer(language_alias=TEST_LANGUAGE_ALIAS,
+                                             pyexasol_connection=pyexasol_connection,
+                                             bucketfs_config=itde.bucketfs)
+        deployer.run(container_file=Path(container_path), alter_system=True, allow_override=True)
+        new_connection = stack.enter_context(connection_factory(itde.db))
+        assert_udf_running(new_connection, TEST_LANGUAGE_ALIAS, TEST_SCHEMA)
+
+
+def test_language_container_deployer_alter_session(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_url: str,
+        container_name: str):
+    """
+    Tests the deployment of a container in two stages - uploading the container
+    followed by activation at the Session level.
+    """
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        deployer = create_container_deployer(language_alias=TEST_LANGUAGE_ALIAS,
+                                             pyexasol_connection=pyexasol_connection,
+                                             bucketfs_config=itde.bucketfs)
+        deployer.download_and_run(container_url, container_name, alter_system=False)
+        new_connection = stack.enter_context(connection_factory(itde.db))
+        deployer = create_container_deployer(language_alias=TEST_LANGUAGE_ALIAS,
+                                             pyexasol_connection=new_connection,
+                                             bucketfs_config=itde.bucketfs)
+        deployer.activate_container(container_name, LanguageActivationLevel.Session, True)
+        assert_udf_running(new_connection, TEST_LANGUAGE_ALIAS, TEST_SCHEMA)
+
+
+def test_language_container_deployer_activation_fail(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_path: str,
+        container_name: str):
+    """
+    Tests that an attempt to activate a container using an alias that already exists
+    causes an exception if overriding is disallowed.
+    """
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        deployer = create_container_deployer(language_alias=TEST_LANGUAGE_ALIAS,
+                                             pyexasol_connection=pyexasol_connection,
+                                             bucketfs_config=itde.bucketfs)
+        deployer.run(container_file=Path(container_path), alter_system=True, allow_override=True)
+        new_connection = stack.enter_context(connection_factory(itde.db))
+        deployer = create_container_deployer(language_alias=TEST_LANGUAGE_ALIAS,
+                                             pyexasol_connection=new_connection,
+                                             bucketfs_config=itde.bucketfs)
+        with pytest.raises(RuntimeError):
+            deployer.activate_container(container_name, LanguageActivationLevel.System, False)

test/integration/test_language_container_deployer_cli.py

@@ -0,0 +1,140 @@
+from typing import Optional, Callable
+from contextlib import ExitStack
+
+from urllib.parse import urlparse
+from click.testing import CliRunner
+from pyexasol import ExaConnection, ExaConnectionFailedError
+from pytest_itde import config
+
+from test.utils.revert_language_settings import revert_language_settings
+from test.utils.db_utils import (create_schema, assert_udf_running)
+
+
+TEST_SCHEMA = "PEC_DEPLOYER_TESTS_CLI"
+TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS_CLI"
+
+
+def call_language_definition_deployer_cli(func,
+                                          exasol_config: config.Exasol,
+                                          bucketfs_config: config.BucketFs,
+                                          language_alias: str,
+                                          container_path: Optional[str] = None,
+                                          version: Optional[str] = None,
+                                          use_ssl_cert_validation: bool = False):
+    parsed_url = urlparse(bucketfs_config.url)
+    args_list = [
+        "language-container",
+        "--bucketfs-name", "bfsdefault",
+        "--bucketfs-host", parsed_url.hostname,
+        "--bucketfs-port", parsed_url.port,
+        "--bucketfs-use-https", False,
+        "--bucketfs-user", bucketfs_config.username,
+        "--bucketfs-password", bucketfs_config.password,
+        "--bucket", "default",
+        "--path-in-bucket", "container",
+        "--dsn", f"{exasol_config.host}:{exasol_config.port}",
+        "--db-user", exasol_config.username,
+        "--db-pass", exasol_config.password,
+        "--language-alias", language_alias
+    ]
+    if use_ssl_cert_validation:
+        args_list += [
+            "--use-ssl-cert-validation"
+        ]
+    else:
+        args_list += [
+            "--no-use-ssl-cert-validation"
+        ]
+    if version:
+        args_list += [
+            "--version", version,
+        ]
+    if container_path:
+        args_list += [
+            "--container-file", container_path,
+        ]
+    runner = CliRunner()
+    result = runner.invoke(func, args_list)
+    return result
+
+
+def test_language_container_deployer_cli_with_container_file(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_path: str,
+        main_func
+):
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        result = call_language_definition_deployer_cli(main_func,
+                                                       container_path=container_path,
+                                                       language_alias=TEST_LANGUAGE_ALIAS,
+                                                       exasol_config=itde.db,
+                                                       bucketfs_config=itde.bucketfs)
+        assert result.exit_code == 0
+        assert result.exception is None
+        assert result.stdout == ""
+        new_connection = stack.enter_context(connection_factory(itde.db))
+        assert_udf_running(new_connection, TEST_LANGUAGE_ALIAS, TEST_SCHEMA)
+
+
+def test_language_container_deployer_cli_by_downloading_container(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_version: str,
+        main_func
+):
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        result = call_language_definition_deployer_cli(main_func,
+                                                       version=container_version,
+                                                       language_alias=TEST_LANGUAGE_ALIAS,
+                                                       exasol_config=itde.db,
+                                                       bucketfs_config=itde.bucketfs)
+        assert result.exit_code == 0
+        assert result.exception is None
+        assert result.stdout == ""
+        new_connection = stack.enter_context(connection_factory(itde.db))
+        assert_udf_running(new_connection, TEST_LANGUAGE_ALIAS, TEST_SCHEMA)
+
+
+def test_language_container_deployer_cli_with_missing_container_option(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        main_func
+):
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        result = call_language_definition_deployer_cli(main_func,
+                                                       language_alias=TEST_LANGUAGE_ALIAS,
+                                                       bucketfs_config=itde.bucketfs,
+                                                       exasol_config=itde.db)
+        assert result.exit_code == 1
+        assert isinstance(result.exception, ValueError)
+
+
+def test_language_container_deployer_cli_with_check_cert(
+        itde: config.TestConfig,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        container_path: str,
+        main_func
+):
+    expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]'
+    with ExitStack() as stack:
+        pyexasol_connection = stack.enter_context(connection_factory(itde.db))
+        stack.enter_context(revert_language_settings(pyexasol_connection))
+        create_schema(pyexasol_connection, TEST_SCHEMA)
+        result = call_language_definition_deployer_cli(main_func,
+                                                       container_path=container_path,
+                                                       language_alias=TEST_LANGUAGE_ALIAS,
+                                                       exasol_config=itde.db,
+                                                       bucketfs_config=itde.bucketfs,
+                                                       use_ssl_cert_validation=True)
+        assert result.exit_code == 1
+        assert expected_exception_message in result.exception.args[0].message
+        assert isinstance(result.exception, ExaConnectionFailedError)

test/utils/db_utils.py

@@ -0,0 +1,20 @@
+import textwrap
+
+from pyexasol import ExaConnection
+
+
+def create_schema(pyexasol_connection: ExaConnection, schema: str):
+    pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;")
+    pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};")
+
+
+def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str, schema: str):
+    pyexasol_connection.execute(textwrap.dedent(f"""
+        CREATE OR REPLACE {language_alias} SCALAR SCRIPT {schema}."TEST_UDF"()
+        RETURNS BOOLEAN AS
+        def run(ctx):
+            return True
+        /
+        """))
+    result = pyexasol_connection.execute(f'SELECT {schema}."TEST_UDF"()').fetchall()
+    assert result[0][0] is True

test/utils/revert_language_settings.py

@@ -0,0 +1,17 @@
+import contextlib
+
+from pyexasol import ExaConnection  # type: ignore
+
+
+@contextlib.contextmanager
+def revert_language_settings(connection: ExaConnection):
+    query = f"""
+        SELECT "SYSTEM_VALUE", "SESSION_VALUE"
+        FROM SYS.EXA_PARAMETERS
+        WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'"""
+    language_settings = connection.execute(query).fetchall()[0]
+    try:
+        yield
+    finally:
+        connection.execute(f"ALTER SYSTEM SET SCRIPT_LANGUAGES='{language_settings[0]}';")
+        connection.execute(f"ALTER SESSION SET SCRIPT_LANGUAGES='{language_settings[1]}';")