#75 Fixed the bucketfs_location tests

exasol/python_extension_common/connections/bucketfs_location.py

@@ -54,7 +54,7 @@ def _convert_onprem_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]:
            f"{bfs_params[StdParams.bucketfs_host.name]}:"
            f"{bfs_params[StdParams.bucketfs_port.name]}")
     return {
-        'backend': bfs.path.StorageBackend.onprem,
+        'backend': bfs.path.StorageBackend.onprem.name,
         'url': url,
         'username': bfs_params[StdParams.bucketfs_user.name],
         'password': bfs_params[StdParams.bucketfs_password.name],
@@ -82,7 +82,7 @@ def _convert_saas_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]:
                             database_name=bfs_params[StdParams.saas_database_name.name]
                         ))
     return {
-        'backend': bfs.path.StorageBackend.saas,
+        'backend': bfs.path.StorageBackend.saas.name,
         'url': saas_url,
         'account_id': saas_account_id,
         'database_id': saas_database_id,
@@ -198,8 +198,8 @@ def create_bucketfs_conn_object(conn_name: str, **kwargs) -> None:
             create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name,
                                                _convert_onprem_bfs_params(kwargs))
         else:
-            create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name,
-                                               _convert_saas_bfs_params(kwargs))
+            create_bucketfs_conn_object_saas(pyexasol_connection, conn_name,
+                                             _convert_saas_bfs_params(kwargs))
 
 
 def create_bucketfs_location_from_conn_object(conn_obj) -> bfs.path.PathLike:

test/integration_hold/conftest.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+from typing import Any
+import pytest
+import click
+import requests
+from urllib.parse import urlparse
+from contextlib import ExitStack, contextmanager
+import pyexasol
+import exasol.bucketfs as bfs
+
+from exasol.python_extension_common.cli.std_options import StdParams
+from exasol.python_extension_common.deployment.language_container_deployer import (
+    LanguageContainerDeployer,
+)
+from exasol.python_extension_common.deployment.language_container_deployer_cli import (
+    language_container_deployer_main, slc_parameter_formatters, CustomizableParameters)
+from test.utils.revert_language_settings import revert_language_settings
+from test.utils.db_utils import create_schema, open_schema
+
+VERSION = "8.0.0"
+
+TEST_SCHEMA = "PEC_DEPLOYER_TESTS"
+TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS"
+
+
+@pytest.fixture(scope='session')
+def container_name() -> str:
+    return "template-Exasol-all-python-3.10_release.tar.gz"
+
+
+@pytest.fixture(scope='session')
+def container_url_formatter(container_name) -> str:
+    return ("https://github.com/exasol/script-languages-release/releases/"
+            "download/{version}/") + container_name
+
+
+@pytest.fixture
+def main_func(slc_name, slc_url_formatter):
+
+    @click.group()
+    def fake_main():
+        pass
+
+    slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, container_url_formatter)
+    slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, container_name)
+
+    fake_main.add_command(language_container_deployer_main)
+    return fake_main
+
+
+@pytest.fixture(scope='session')
+def container_version() -> str:
+    return VERSION
+
+
+@pytest.fixture(scope='session')
+def container_url(container_url_formatter, container_version) -> str:
+    return container_url_formatter.format(version=container_version)
+
+
+@pytest.fixture(scope='session')
+def container_path(tmpdir_factory, container_url, container_name) -> str:
+
+    response = requests.get(container_url, allow_redirects=True)
+    response.raise_for_status()
+    slc_path = tmpdir_factory.mktemp('container').join(container_name)
+    slc_path = str(slc_path)
+    with open(slc_path, 'wb') as f:
+        f.write(response.content)
+    return slc_path
+
+
+@pytest.fixture(scope='session')
+def db_schema() -> str:
+    return TEST_SCHEMA
+
+
+@pytest.fixture(scope='session')
+def language_alias() -> str:
+    return TEST_LANGUAGE_ALIAS
+
+
+@pytest.fixture(scope='session')
+def deployer_factory(
+        backend_aware_database_params,
+        backend_aware_bucketfs_params,
+        db_schema,
+        language_alias):
+    @contextmanager
+    def create_deployer(create_test_schema: bool = False, open_test_schema: bool = True):
+        with ExitStack() as stack:
+            pyexasol_connection = stack.enter_context(pyexasol.connect(**backend_aware_database_params))
+            bucketfs_path = bfs.path.build_path(**backend_aware_bucketfs_params)
+            stack.enter_context(revert_language_settings(pyexasol_connection))
+            if create_test_schema:
+                create_schema(pyexasol_connection, db_schema, open_test_schema)
+            elif open_test_schema:
+                open_schema(pyexasol_connection, db_schema)
+            yield LanguageContainerDeployer(pyexasol_connection, language_alias, bucketfs_path)
+    return create_deployer
+
+
+@pytest.fixture(scope='session')
+def onprem_db_params(backend_aware_onprem_database,
+                     exasol_config) -> dict[str, Any]:
+    return {
+        StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}',
+        StdParams.db_user.name: exasol_config.username,
+        StdParams.db_password.name: exasol_config.password,
+        StdParams.use_ssl_cert_validation.name: False
+    }
+
+
+@pytest.fixture(scope='session')
+def onprem_bfs_params(backend_aware_onprem_database,
+                      bucketfs_config) -> dict[str, Any]:
+    parsed_url = urlparse(bucketfs_config.url)
+    host, port = parsed_url.netloc.split(":")
+    return {
+        StdParams.bucketfs_host.name: host,
+        StdParams.bucketfs_port.name: port,
+        StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https',
+        StdParams.bucketfs_user.name: bucketfs_config.username,
+        StdParams.bucketfs_password.name: bucketfs_config.password,
+        StdParams.bucketfs_name.name: 'bfsdefault',
+        StdParams.bucket.name: 'default',
+        StdParams.use_ssl_cert_validation.name: False
+    }
+
+
+@pytest.fixture(scope='session')
+def saas_params_id(saas_host,
+                   saas_pat,
+                   saas_account_id,
+                   backend_aware_saas_database_id) -> dict[str, Any]:
+    return {
+        StdParams.saas_url.name: saas_host,
+        StdParams.saas_account_id.name: saas_account_id,
+        StdParams.saas_database_id.name: backend_aware_saas_database_id,
+        StdParams.saas_token.name: saas_pat,
+    }
+
+
+@pytest.fixture(scope='session')
+def saas_params_name(saas_params_id,
+                     database_name) -> dict[str, Any]:
+    saas_params = dict(saas_params_id)
+    saas_params.pop(StdParams.saas_database_id.name)
+    saas_params[StdParams.saas_database_name.name] = database_name
+    return saas_params

test/integration_hold/connections/test_bucketfs_location.py

@@ -0,0 +1,142 @@
+from typing import Any
+from unittest.mock import patch
+from contextlib import contextmanager
+
+import pyexasol
+import pytest
+import exasol.bucketfs as bfs
+
+from exasol.python_extension_common.cli.std_options import StdParams
+from exasol.python_extension_common.connections.bucketfs_location import (
+    create_bucketfs_location,
+    create_bucketfs_conn_object,
+    create_bucketfs_location_from_conn_object,
+    ConnectionInfo)
+
+TEST_FILE_CONTENT = b'A rose by any other name would smell as sweet.'
+
+
+@pytest.fixture
+def rubbish_params() -> dict[str, Any]:
+    return {
+        StdParams.bucketfs_name.name: 'bfsdefault',
+        StdParams.bucket.name: 'default',
+        StdParams.saas_url.name: 'my_saas_url',
+    }
+
+
+@contextmanager
+def write_test_file(bfs_path: bfs.path.PathLike) -> None:
+    try:
+        bfs_path.write(TEST_FILE_CONTENT)
+        yield
+    finally:
+        # We cannot reuse the same path, in subsequent tests because of the
+        # temporary lock bucket-fs places on deleted files, but it's useful
+        # to delete the file anyway to avoid potential false-positives.
+        bfs_path.rm()
+
+
+def validate_test_file(bfs_path: bfs.path.PathLike) -> None:
+    file_content = b''.join(bfs_path.read())
+    assert file_content == TEST_FILE_CONTENT
+
+
+def validate_conn_object(pyexasol_connection: pyexasol.ExaConnection,
+                         conn_name: str,
+                         conn_obj: ConnectionInfo):
+    bfs_path = create_bucketfs_location_from_conn_object(conn_obj)
+    validate_test_file(bfs_path)
+
+
+def test_create_bucketfs_location_onprem(use_onprem,
+                                         onprem_bfs_params):
+    if not use_onprem:
+        pytest.skip("The test is not configured to use ITDE.")
+
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_location'}
+    bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params)
+    with write_test_file(bfs_path):
+        validate_test_file(bfs_path)
+
+
+def test_create_bucketfs_location_saas_db_id(use_saas,
+                                             saas_params_id):
+    if not use_saas:
+        pytest.skip("The test is not configured to use SaaS.")
+
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_id'}
+    bfs_path = create_bucketfs_location(**saas_params_id, **extra_params)
+    with write_test_file(bfs_path):
+        validate_test_file(bfs_path)
+
+
+def test_create_bucketfs_location_saas_db_name(use_saas,
+                                               saas_params_name):
+    if not use_saas:
+        pytest.skip("The test is not configured to use SaaS.")
+
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_location_with_name'}
+    bfs_path = create_bucketfs_location(**saas_params_name, **extra_params)
+    with write_test_file(bfs_path):
+        validate_test_file(bfs_path)
+
+
+def test_create_bucketfs_location_error(rubbish_params):
+    with pytest.raises(ValueError):
+        create_bucketfs_location(**rubbish_params)
+
+
+@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object')
+def test_create_bucketfs_conn_object_onprem(write_conn_object_mock,
+                                            use_onprem,
+                                            onprem_db_params,
+                                            onprem_bfs_params):
+    if not use_onprem:
+        pytest.skip("The test is not configured to use ITDE.")
+
+    write_conn_object_mock.side_effect = validate_conn_object
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object'}
+    bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params)
+    with write_test_file(bfs_path):
+        # onprem_db_params and onprem_bfs_params have one item in common -
+        # use_ssl_cert_validation, so we need to take a union before using them as kwargs.
+        onprem_params = dict(onprem_db_params)
+        onprem_params.update(onprem_bfs_params)
+        create_bucketfs_conn_object(conn_name='ONPREM_TEST_BFS',
+                                    **onprem_params, **extra_params)
+
+
+@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object')
+def test_create_bucketfs_conn_object_saas_db_id(write_conn_object_mock,
+                                                use_saas,
+                                                saas_params_id):
+    if not use_saas:
+        pytest.skip("The test is not configured to use SaaS.")
+
+    write_conn_object_mock.side_effect = validate_conn_object
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_id'}
+    bfs_path = create_bucketfs_location(**saas_params_id, **extra_params)
+    with write_test_file(bfs_path):
+        create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_ID',
+                                    **saas_params_id, **extra_params)
+
+
+@patch('exasol.python_extension_common.connections.bucketfs_location.write_bucketfs_conn_object')
+def test_create_bucketfs_conn_object_saas_db_name(write_conn_object_mock,
+                                                  use_saas,
+                                                  saas_params_name):
+    if not use_saas:
+        pytest.skip("The test is not configured to use SaaS.")
+
+    write_conn_object_mock.side_effect = validate_conn_object
+    extra_params = {StdParams.path_in_bucket.name: 'test_create_conn_object_with_name'}
+    bfs_path = create_bucketfs_location(**saas_params_name, **extra_params)
+    with write_test_file(bfs_path):
+        create_bucketfs_conn_object(conn_name='SAAS_TEST_BFS_NAME',
+                                    **saas_params_name, **extra_params)
+
+
+def test_create_bucketfs_conn_object_error(rubbish_params):
+    with pytest.raises(ValueError):
+        create_bucketfs_conn_object(conn_name='WHATEVER', **rubbish_params)