From 633706c400744d72a0a2e386e44ad49b9a340c7d Mon Sep 17 00:00:00 2001 From: mibe Date: Fri, 4 Oct 2024 15:38:56 +0100 Subject: [PATCH] #75 Fixed the bucketfs_location tests --- .../connections/bucketfs_location.py | 8 +- test/integration_hold/__init__.py | 0 .../test_language_container_deployer_cli.py | 0 test/integration_hold/conftest.py | 150 ++++++++++++++++++ .../connections/test_bucketfs_location.py | 142 +++++++++++++++++ .../connections/test_pyexasol_connection.py | 0 .../test_language_container_builder.py | 0 .../test_language_container_deployer.py | 0 8 files changed, 296 insertions(+), 4 deletions(-) create mode 100644 test/integration_hold/__init__.py rename test/{integration => integration_hold}/cli/test_language_container_deployer_cli.py (100%) create mode 100644 test/integration_hold/conftest.py create mode 100644 test/integration_hold/connections/test_bucketfs_location.py rename test/{integration => integration_hold}/connections/test_pyexasol_connection.py (100%) rename test/{integration => integration_hold}/deployment/test_language_container_builder.py (100%) rename test/{integration => integration_hold}/deployment/test_language_container_deployer.py (100%) diff --git a/exasol/python_extension_common/connections/bucketfs_location.py b/exasol/python_extension_common/connections/bucketfs_location.py index 9302413..7f6fed3 100644 --- a/exasol/python_extension_common/connections/bucketfs_location.py +++ b/exasol/python_extension_common/connections/bucketfs_location.py @@ -54,7 +54,7 @@ def _convert_onprem_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]: f"{bfs_params[StdParams.bucketfs_host.name]}:" f"{bfs_params[StdParams.bucketfs_port.name]}") return { - 'backend': bfs.path.StorageBackend.onprem, + 'backend': bfs.path.StorageBackend.onprem.name, 'url': url, 'username': bfs_params[StdParams.bucketfs_user.name], 'password': bfs_params[StdParams.bucketfs_password.name], @@ -82,7 +82,7 @@ def _convert_saas_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]: database_name=bfs_params[StdParams.saas_database_name.name] )) return { - 'backend': bfs.path.StorageBackend.saas, + 'backend': bfs.path.StorageBackend.saas.name, 'url': saas_url, 'account_id': saas_account_id, 'database_id': saas_database_id, @@ -198,8 +198,8 @@ def create_bucketfs_conn_object(conn_name: str, **kwargs) -> None: create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name, _convert_onprem_bfs_params(kwargs)) else: - create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name, - _convert_saas_bfs_params(kwargs)) + create_bucketfs_conn_object_saas(pyexasol_connection, conn_name, + _convert_saas_bfs_params(kwargs)) def create_bucketfs_location_from_conn_object(conn_obj) -> bfs.path.PathLike: diff --git a/test/integration_hold/__init__.py b/test/integration_hold/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/integration/cli/test_language_container_deployer_cli.py b/test/integration_hold/cli/test_language_container_deployer_cli.py similarity index 100% rename from test/integration/cli/test_language_container_deployer_cli.py rename to test/integration_hold/cli/test_language_container_deployer_cli.py diff --git a/test/integration_hold/conftest.py b/test/integration_hold/conftest.py new file mode 100644 index 0000000..1fd1c5c --- /dev/null +++ b/test/integration_hold/conftest.py @@ -0,0 +1,150 @@ +from __future__ import annotations +from typing import Any +import pytest +import click +import requests +from urllib.parse import urlparse +from contextlib import ExitStack, contextmanager +import pyexasol +import exasol.bucketfs as bfs + +from exasol.python_extension_common.cli.std_options import StdParams +from exasol.python_extension_common.deployment.language_container_deployer import ( + LanguageContainerDeployer, +) +from exasol.python_extension_common.deployment.language_container_deployer_cli import ( + language_container_deployer_main, slc_parameter_formatters, CustomizableParameters) +from test.utils.revert_language_settings import revert_language_settings +from test.utils.db_utils import create_schema, open_schema + +VERSION = "8.0.0" + +TEST_SCHEMA = "PEC_DEPLOYER_TESTS" +TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS" + + +@pytest.fixture(scope='session') +def container_name() -> str: + return "template-Exasol-all-python-3.10_release.tar.gz" + + +@pytest.fixture(scope='session') +def container_url_formatter(container_name) -> str: + return ("https://github.com/exasol/script-languages-release/releases/" + "download/{version}/") + container_name + + +@pytest.fixture +def main_func(slc_name, slc_url_formatter): + + @click.group() + def fake_main(): + pass + + slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, container_url_formatter) + slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, container_name) + + fake_main.add_command(language_container_deployer_main) + return fake_main + + +@pytest.fixture(scope='session') +def container_version() -> str: + return VERSION + + +@pytest.fixture(scope='session') +def container_url(container_url_formatter, container_version) -> str: + return container_url_formatter.format(version=container_version) + + +@pytest.fixture(scope='session') +def container_path(tmpdir_factory, container_url, container_name) -> str: + + response = requests.get(container_url, allow_redirects=True) + response.raise_for_status() + slc_path = tmpdir_factory.mktemp('container').join(container_name) + slc_path = str(slc_path) + with open(slc_path, 'wb') as f: + f.write(response.content) + return slc_path + + +@pytest.fixture(scope='session') +def db_schema() -> str: + return TEST_SCHEMA + + +@pytest.fixture(scope='session') +def language_alias() -> str: + return TEST_LANGUAGE_ALIAS + + +@pytest.fixture(scope='session') +def deployer_factory( + backend_aware_database_params, + backend_aware_bucketfs_params, + db_schema, + language_alias): + @contextmanager + def create_deployer(create_test_schema: bool = False, open_test_schema: bool = True): + with ExitStack() as stack: + pyexasol_connection = stack.enter_context(pyexasol.connect(**backend_aware_database_params)) + bucketfs_path = bfs.path.build_path(**backend_aware_bucketfs_params) + stack.enter_context(revert_language_settings(pyexasol_connection)) + if create_test_schema: + create_schema(pyexasol_connection, db_schema, open_test_schema) + elif open_test_schema: + open_schema(pyexasol_connection, db_schema) + yield LanguageContainerDeployer(pyexasol_connection, language_alias, bucketfs_path) + return create_deployer + + +@pytest.fixture(scope='session') +def onprem_db_params(backend_aware_onprem_database, + exasol_config) -> dict[str, Any]: + return { + StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}', + StdParams.db_user.name: exasol_config.username, + StdParams.db_password.name: exasol_config.password, + StdParams.use_ssl_cert_validation.name: False + } + + +@pytest.fixture(scope='session') +def onprem_bfs_params(backend_aware_onprem_database, + bucketfs_config) -> dict[str, Any]: + parsed_url = urlparse(bucketfs_config.url) + host, port = parsed_url.netloc.split(":") + return { + StdParams.bucketfs_host.name: host, + StdParams.bucketfs_port.name: port, + StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https', + StdParams.bucketfs_user.name: bucketfs_config.username, + StdParams.bucketfs_password.name: bucketfs_config.password, + StdParams.bucketfs_name.name: 'bfsdefault', + StdParams.bucket.name: 'default', + StdParams.use_ssl_cert_validation.name: False + } + + +@pytest.fixture(scope='session') +def saas_params_id(saas_host, + saas_pat, + saas_account_id, + backend_aware_saas_database_id) -> dict[str, Any]: + return { + StdParams.saas_url.name: saas_host, + StdParams.saas_account_id.name: saas_account_id, + StdParams.saas_database_id.name: backend_aware_saas_database_id, + StdParams.saas_token.name: saas_pat, + } + + +@pytest.fixture(scope='session') +def saas_params_name(saas_params_id, + database_name) -> dict[str, Any]: + saas_params = dict(saas_params_id) + saas_params.pop(StdParams.saas_database_id.name) + saas_params[StdParams.saas_database_name.name] = database_name + return saas_params diff --git a/test/integration_hold/connections/test_bucketfs_location.py b/test/integration_hold/connections/test_bucketfs_location.py new file mode 100644 index 0000000..9d1af9f --- /dev/null +++ b/test/integration_hold/connections/test_bucketfs_location.py @@ -0,0 +1,142 @@ +from typing import Any +from unittest.mock import patch +from contextlib import contextmanager + +import pyexasol +import pytest +import exasol.bucketfs as bfs + +from exasol.python_extension_common.cli.std_options import StdParams +from exasol.python_extension_common.connections.bucketfs_location import ( + create_bucketfs_location, + create_bucketfs_conn_object, + create_bucketfs_location_from_conn_object, + ConnectionInfo) + +TEST_FILE_CONTENT = b'A rose by any other name would smell as sweet.' + + +@pytest.fixture +def rubbish_params() -> dict[str, Any]: + return { + StdParams.bucketfs_name.name: 'bfsdefault', + StdParams.bucket.name: 'default', + StdParams.saas_url.name: 'my_saas_url', + } + + +@contextmanager +def write_test_file(bfs_path: bfs.path.PathLike) -> None: + try: + bfs_path.write(TEST_FILE_CONTENT) + yield + finally: + # We cannot reuse the same path, in subsequent tests because of the + # temporary lock bucket-fs places on deleted files, but it's useful + # to delete the file anyway to avoid potential false-positives. + bfs_path.rm() + + +def validate_test_file(bfs_path: bfs.path.PathLike) -> None: + file_content = b''.join(bfs_path.read()) + assert file_content == TEST_FILE_CONTENT + + +def validate_conn_object(pyexasol_connection: pyexasol.ExaConnection, + conn_name: str, + conn_obj: ConnectionInfo): + bfs_path = create_bucketfs_location_from_conn_object(conn_obj) + validate_test_file(bfs_path) + + +def test_create_bucketfs_location_onprem(use_onprem, + onprem_bfs_params): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + extra_params = {StdParams.path_in_bucket.name: 'test_create_location'} + bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) + with write_test_file(bfs_path): + validate_test_file(bfs_path) + + +def test_create_bucketfs_location_saas_db_id(use_saas, + saas_params_id): + if not use_saas: + pytest.skip("The test is not configured to use SaaS.") + + extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_id'} + bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) + with write_test_file(bfs_path): + validate_test_file(bfs_path) + + +def test_create_bucketfs_location_saas_db_name(use_saas, + saas_params_name): + if not use_saas: + pytest.skip("The test is not configured to use SaaS.") + + extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_name'} + bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) + with write_test_file(bfs_path): + validate_test_file(bfs_path) + + +def test_create_bucketfs_location_error(rubbish_params): + with pytest.raises(ValueError): + create_bucketfs_location(**rubbish_params) + + +@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') +def test_create_bucketfs_conn_object_onprem(write_conn_object_mock, + use_onprem, + onprem_db_params, + onprem_bfs_params): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + write_conn_object_mock.side_effect = validate_conn_object + extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object'} + bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) + with write_test_file(bfs_path): + # onprem_db_params and onprem_bfs_params have one item in common - + # use_ssl_cert_validation, so we need to take a union before using them as kwargs. + onprem_params = dict(onprem_db_params) + onprem_params.update(onprem_bfs_params) + create_bucketfs_conn_object(conn_name='ONPREM_TEST_BFS', + **onprem_params, **extra_params) + + +@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') +def test_create_bucketfs_conn_object_saas_db_id(write_conn_object_mock, + use_saas, + saas_params_id): + if not use_saas: + pytest.skip("The test is not configured to use SaaS.") + + write_conn_object_mock.side_effect = validate_conn_object + extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_id'} + bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) + with write_test_file(bfs_path): + create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_ID', + **saas_params_id, **extra_params) + + +@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') +def test_create_bucketfs_conn_object_saas_db_name(write_conn_object_mock, + use_saas, + saas_params_name): + if not use_saas: + pytest.skip("The test is not configured to use SaaS.") + + write_conn_object_mock.side_effect = validate_conn_object + extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_name'} + bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) + with write_test_file(bfs_path): + create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_NAME', + **saas_params_name, **extra_params) + + +def test_create_bucketfs_conn_object_error(rubbish_params): + with pytest.raises(ValueError): + create_bucketfs_conn_object(conn_name='WHATEVER', **rubbish_params) diff --git a/test/integration/connections/test_pyexasol_connection.py b/test/integration_hold/connections/test_pyexasol_connection.py similarity index 100% rename from test/integration/connections/test_pyexasol_connection.py rename to test/integration_hold/connections/test_pyexasol_connection.py diff --git a/test/integration/deployment/test_language_container_builder.py b/test/integration_hold/deployment/test_language_container_builder.py similarity index 100% rename from test/integration/deployment/test_language_container_builder.py rename to test/integration_hold/deployment/test_language_container_builder.py diff --git a/test/integration/deployment/test_language_container_deployer.py b/test/integration_hold/deployment/test_language_container_deployer.py similarity index 100% rename from test/integration/deployment/test_language_container_deployer.py rename to test/integration_hold/deployment/test_language_container_deployer.py