From 9ea87d986fe82fa3e275c7520c05a67d01605db0 Mon Sep 17 00:00:00 2001 From: mibe Date: Fri, 4 Oct 2024 16:04:12 +0100 Subject: [PATCH] #75 Added SaaS tests --- .github/workflows/run-tests.yml | 2 +- .../test_language_container_deployer_cli.py | 0 .../connections/test_pyexasol_connection.py | 0 .../test_language_container_builder.py | 0 .../test_language_container_deployer.py | 0 test/integration_hold/__init__.py | 0 test/integration_hold/conftest.py | 150 ------------------ .../connections/test_bucketfs_location.py | 142 ----------------- 8 files changed, 1 insertion(+), 293 deletions(-) rename test/{integration_hold => integration}/cli/test_language_container_deployer_cli.py (100%) rename test/{integration_hold => integration}/connections/test_pyexasol_connection.py (100%) rename test/{integration_hold => integration}/deployment/test_language_container_builder.py (100%) rename test/{integration_hold => integration}/deployment/test_language_container_deployer.py (100%) delete mode 100644 test/integration_hold/__init__.py delete mode 100644 test/integration_hold/conftest.py delete mode 100644 test/integration_hold/connections/test_bucketfs_location.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 13edace..f75447a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -40,4 +40,4 @@ jobs: SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} run: - poetry run nox -s integration-tests -- -- --db-version ${{ inputs.exasol-version}} --backend onprem + poetry run nox -s integration-tests -- -- --db-version ${{ inputs.exasol-version}} --backend all diff --git a/test/integration_hold/cli/test_language_container_deployer_cli.py b/test/integration/cli/test_language_container_deployer_cli.py similarity index 100% rename from test/integration_hold/cli/test_language_container_deployer_cli.py rename to test/integration/cli/test_language_container_deployer_cli.py diff --git a/test/integration_hold/connections/test_pyexasol_connection.py b/test/integration/connections/test_pyexasol_connection.py similarity index 100% rename from test/integration_hold/connections/test_pyexasol_connection.py rename to test/integration/connections/test_pyexasol_connection.py diff --git a/test/integration_hold/deployment/test_language_container_builder.py b/test/integration/deployment/test_language_container_builder.py similarity index 100% rename from test/integration_hold/deployment/test_language_container_builder.py rename to test/integration/deployment/test_language_container_builder.py diff --git a/test/integration_hold/deployment/test_language_container_deployer.py b/test/integration/deployment/test_language_container_deployer.py similarity index 100% rename from test/integration_hold/deployment/test_language_container_deployer.py rename to test/integration/deployment/test_language_container_deployer.py diff --git a/test/integration_hold/__init__.py b/test/integration_hold/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/integration_hold/conftest.py b/test/integration_hold/conftest.py deleted file mode 100644 index 1fd1c5c..0000000 --- a/test/integration_hold/conftest.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations -from typing import Any -import pytest -import click -import requests -from urllib.parse import urlparse -from contextlib import ExitStack, contextmanager -import pyexasol -import exasol.bucketfs as bfs - -from exasol.python_extension_common.cli.std_options import StdParams -from exasol.python_extension_common.deployment.language_container_deployer import ( - LanguageContainerDeployer, -) -from exasol.python_extension_common.deployment.language_container_deployer_cli import ( - language_container_deployer_main, slc_parameter_formatters, CustomizableParameters) -from test.utils.revert_language_settings import revert_language_settings -from test.utils.db_utils import create_schema, open_schema - -VERSION = "8.0.0" - -TEST_SCHEMA = "PEC_DEPLOYER_TESTS" -TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS" - - -@pytest.fixture(scope='session') -def container_name() -> str: - return "template-Exasol-all-python-3.10_release.tar.gz" - - -@pytest.fixture(scope='session') -def container_url_formatter(container_name) -> str: - return ("https://github.com/exasol/script-languages-release/releases/" - "download/{version}/") + container_name - - -@pytest.fixture -def main_func(slc_name, slc_url_formatter): - - @click.group() - def fake_main(): - pass - - slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, container_url_formatter) - slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, container_name) - - fake_main.add_command(language_container_deployer_main) - return fake_main - - -@pytest.fixture(scope='session') -def container_version() -> str: - return VERSION - - -@pytest.fixture(scope='session') -def container_url(container_url_formatter, container_version) -> str: - return container_url_formatter.format(version=container_version) - - -@pytest.fixture(scope='session') -def container_path(tmpdir_factory, container_url, container_name) -> str: - - response = requests.get(container_url, allow_redirects=True) - response.raise_for_status() - slc_path = tmpdir_factory.mktemp('container').join(container_name) - slc_path = str(slc_path) - with open(slc_path, 'wb') as f: - f.write(response.content) - return slc_path - - -@pytest.fixture(scope='session') -def db_schema() -> str: - return TEST_SCHEMA - - -@pytest.fixture(scope='session') -def language_alias() -> str: - return TEST_LANGUAGE_ALIAS - - -@pytest.fixture(scope='session') -def deployer_factory( - backend_aware_database_params, - backend_aware_bucketfs_params, - db_schema, - language_alias): - @contextmanager - def create_deployer(create_test_schema: bool = False, open_test_schema: bool = True): - with ExitStack() as stack: - pyexasol_connection = stack.enter_context(pyexasol.connect(**backend_aware_database_params)) - bucketfs_path = bfs.path.build_path(**backend_aware_bucketfs_params) - stack.enter_context(revert_language_settings(pyexasol_connection)) - if create_test_schema: - create_schema(pyexasol_connection, db_schema, open_test_schema) - elif open_test_schema: - open_schema(pyexasol_connection, db_schema) - yield LanguageContainerDeployer(pyexasol_connection, language_alias, bucketfs_path) - return create_deployer - - -@pytest.fixture(scope='session') -def onprem_db_params(backend_aware_onprem_database, - exasol_config) -> dict[str, Any]: - return { - StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}', - StdParams.db_user.name: exasol_config.username, - StdParams.db_password.name: exasol_config.password, - StdParams.use_ssl_cert_validation.name: False - } - - -@pytest.fixture(scope='session') -def onprem_bfs_params(backend_aware_onprem_database, - bucketfs_config) -> dict[str, Any]: - parsed_url = urlparse(bucketfs_config.url) - host, port = parsed_url.netloc.split(":") - return { - StdParams.bucketfs_host.name: host, - StdParams.bucketfs_port.name: port, - StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https', - StdParams.bucketfs_user.name: bucketfs_config.username, - StdParams.bucketfs_password.name: bucketfs_config.password, - StdParams.bucketfs_name.name: 'bfsdefault', - StdParams.bucket.name: 'default', - StdParams.use_ssl_cert_validation.name: False - } - - -@pytest.fixture(scope='session') -def saas_params_id(saas_host, - saas_pat, - saas_account_id, - backend_aware_saas_database_id) -> dict[str, Any]: - return { - StdParams.saas_url.name: saas_host, - StdParams.saas_account_id.name: saas_account_id, - StdParams.saas_database_id.name: backend_aware_saas_database_id, - StdParams.saas_token.name: saas_pat, - } - - -@pytest.fixture(scope='session') -def saas_params_name(saas_params_id, - database_name) -> dict[str, Any]: - saas_params = dict(saas_params_id) - saas_params.pop(StdParams.saas_database_id.name) - saas_params[StdParams.saas_database_name.name] = database_name - return saas_params diff --git a/test/integration_hold/connections/test_bucketfs_location.py b/test/integration_hold/connections/test_bucketfs_location.py deleted file mode 100644 index 9d1af9f..0000000 --- a/test/integration_hold/connections/test_bucketfs_location.py +++ /dev/null @@ -1,142 +0,0 @@ -from typing import Any -from unittest.mock import patch -from contextlib import contextmanager - -import pyexasol -import pytest -import exasol.bucketfs as bfs - -from exasol.python_extension_common.cli.std_options import StdParams -from exasol.python_extension_common.connections.bucketfs_location import ( - create_bucketfs_location, - create_bucketfs_conn_object, - create_bucketfs_location_from_conn_object, - ConnectionInfo) - -TEST_FILE_CONTENT = b'A rose by any other name would smell as sweet.' - - -@pytest.fixture -def rubbish_params() -> dict[str, Any]: - return { - StdParams.bucketfs_name.name: 'bfsdefault', - StdParams.bucket.name: 'default', - StdParams.saas_url.name: 'my_saas_url', - } - - -@contextmanager -def write_test_file(bfs_path: bfs.path.PathLike) -> None: - try: - bfs_path.write(TEST_FILE_CONTENT) - yield - finally: - # We cannot reuse the same path, in subsequent tests because of the - # temporary lock bucket-fs places on deleted files, but it's useful - # to delete the file anyway to avoid potential false-positives. - bfs_path.rm() - - -def validate_test_file(bfs_path: bfs.path.PathLike) -> None: - file_content = b''.join(bfs_path.read()) - assert file_content == TEST_FILE_CONTENT - - -def validate_conn_object(pyexasol_connection: pyexasol.ExaConnection, - conn_name: str, - conn_obj: ConnectionInfo): - bfs_path = create_bucketfs_location_from_conn_object(conn_obj) - validate_test_file(bfs_path) - - -def test_create_bucketfs_location_onprem(use_onprem, - onprem_bfs_params): - if not use_onprem: - pytest.skip("The test is not configured to use ITDE.") - - extra_params = {StdParams.path_in_bucket.name: 'test_create_location'} - bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) - with write_test_file(bfs_path): - validate_test_file(bfs_path) - - -def test_create_bucketfs_location_saas_db_id(use_saas, - saas_params_id): - if not use_saas: - pytest.skip("The test is not configured to use SaaS.") - - extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_id'} - bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) - with write_test_file(bfs_path): - validate_test_file(bfs_path) - - -def test_create_bucketfs_location_saas_db_name(use_saas, - saas_params_name): - if not use_saas: - pytest.skip("The test is not configured to use SaaS.") - - extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_name'} - bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) - with write_test_file(bfs_path): - validate_test_file(bfs_path) - - -def test_create_bucketfs_location_error(rubbish_params): - with pytest.raises(ValueError): - create_bucketfs_location(**rubbish_params) - - -@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') -def test_create_bucketfs_conn_object_onprem(write_conn_object_mock, - use_onprem, - onprem_db_params, - onprem_bfs_params): - if not use_onprem: - pytest.skip("The test is not configured to use ITDE.") - - write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object'} - bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) - with write_test_file(bfs_path): - # onprem_db_params and onprem_bfs_params have one item in common - - # use_ssl_cert_validation, so we need to take a union before using them as kwargs. - onprem_params = dict(onprem_db_params) - onprem_params.update(onprem_bfs_params) - create_bucketfs_conn_object(conn_name='ONPREM_TEST_BFS', - **onprem_params, **extra_params) - - -@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') -def test_create_bucketfs_conn_object_saas_db_id(write_conn_object_mock, - use_saas, - saas_params_id): - if not use_saas: - pytest.skip("The test is not configured to use SaaS.") - - write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_id'} - bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) - with write_test_file(bfs_path): - create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_ID', - **saas_params_id, **extra_params) - - -@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object') -def test_create_bucketfs_conn_object_saas_db_name(write_conn_object_mock, - use_saas, - saas_params_name): - if not use_saas: - pytest.skip("The test is not configured to use SaaS.") - - write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_name'} - bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) - with write_test_file(bfs_path): - create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_NAME', - **saas_params_name, **extra_params) - - -def test_create_bucketfs_conn_object_error(rubbish_params): - with pytest.raises(ValueError): - create_bucketfs_conn_object(conn_name='WHATEVER', **rubbish_params)