diff --git a/exasol/ai_lab_config.py b/exasol/ai_lab_config.py index a7633d1..228f62c 100644 --- a/exasol/ai_lab_config.py +++ b/exasol/ai_lab_config.py @@ -1,7 +1,4 @@ -from enum import ( - Enum, - auto, -) +from enum import Enum, auto class AILabConfig(Enum): diff --git a/exasol/bfs_utils.py b/exasol/bfs_utils.py index 8ed4ff4..1fb8930 100644 --- a/exasol/bfs_utils.py +++ b/exasol/bfs_utils.py @@ -1,17 +1,16 @@ """ Bucketfs-related functions. """ -import logging import pathlib - +import logging import exasol.bucketfs as bfs # type: ignore + _logger = logging.getLogger(__name__) -def put_file( - bucket: bfs.Bucket, file_path: pathlib.Path, skip_if_exists: bool = True -) -> pathlib.Path: +def put_file(bucket: bfs.Bucket, file_path: pathlib.Path, + skip_if_exists: bool = True) -> pathlib.Path: """ Uploads given file into bucketfs :param bucket: bucket to use diff --git a/exasol/cloud_storage.py b/exasol/cloud_storage.py index 39d1439..050ce03 100644 --- a/exasol/cloud_storage.py +++ b/exasol/cloud_storage.py @@ -1,15 +1,16 @@ import pyexasol # type: ignore + _SETUP_SQL = [ - "OPEN SCHEMA {schema!i}", - """ + "OPEN SCHEMA {schema!i}", + """ --/ CREATE OR REPLACE JAVA SET SCRIPT IMPORT_PATH(...) EMITS (...) AS %scriptclass com.exasol.cloudetl.scriptclasses.FilesImportQueryGenerator; %jar {jar_path!r}; / """, - """ + """ --/ CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS ( @@ -22,19 +23,17 @@ %jar {jar_path!r}; / """, - """ + """ --/ CREATE OR REPLACE JAVA SET SCRIPT IMPORT_FILES(...) EMITS (...) AS %scriptclass com.exasol.cloudetl.scriptclasses.FilesDataImporter; %jar {jar_path!r}; / - """, -] + """ + ] -def setup_scripts( - db_connection: pyexasol.ExaConnection, schema_name: str, bucketfs_jar_path: str -): +def setup_scripts(db_connection: pyexasol.ExaConnection, schema_name: str, bucketfs_jar_path: str): """ Perform initialization of scripts for could-storage-extension. :param db_connection: DB connection @@ -43,10 +42,7 @@ def setup_scripts( :return: """ for sql in _SETUP_SQL: - db_connection.execute( - sql, - query_params={ - "schema": schema_name, - "jar_path": bucketfs_jar_path, - }, - ) + db_connection.execute(sql, query_params={ + "schema": schema_name, + "jar_path": bucketfs_jar_path, + }) diff --git a/exasol/connections.py b/exasol/connections.py index 4de2c74..2fcb3e5 100644 --- a/exasol/connections.py +++ b/exasol/connections.py @@ -9,14 +9,12 @@ import sqlalchemy # type: ignore import exasol.bucketfs as bfs # type: ignore -from exasol.ai_lab_config import AILabConfig as CKey from exasol.secret_store import Secrets from exasol.utils import optional_str_to_bool +from exasol.ai_lab_config import AILabConfig as CKey -def _optional_encryption( - conf: Secrets, key: CKey = CKey.db_encryption -) -> Optional[bool]: +def _optional_encryption(conf: Secrets, key: CKey = CKey.db_encryption) -> Optional[bool]: return optional_str_to_bool(conf.get(key)) @@ -127,7 +125,9 @@ def open_sqlalchemy_connection(conf: Secrets): it is possible to set the client TLS/SSL certificate. """ - websocket_url = f"exa+websocket://{conf.get(CKey.db_user)}:{conf.get(CKey.db_password)}@{get_external_host(conf)}" + websocket_url = ( + f"exa+websocket://{conf.get(CKey.db_user)}:{conf.get(CKey.db_password)}@{get_external_host(conf)}" + ) delimiter = "?" encryption = _optional_encryption(conf) diff --git a/exasol/extension_wrapper_common.py b/exasol/extension_wrapper_common.py index 9e3f5ac..1ae070e 100644 --- a/exasol/extension_wrapper_common.py +++ b/exasol/extension_wrapper_common.py @@ -1,7 +1,7 @@ -from exasol.ai_lab_config import AILabConfig as CKey from exasol.connections import open_pyexasol_connection from exasol.secret_store import Secrets from exasol.utils import optional_str_to_bool +from exasol.ai_lab_config import AILabConfig as CKey def str_to_bool(conf: Secrets, key: CKey, default_value: bool) -> bool: diff --git a/exasol/github.py b/exasol/github.py index 6793805..c18ea1c 100644 --- a/exasol/github.py +++ b/exasol/github.py @@ -3,14 +3,10 @@ project, retrieval of artefacts, etc. """ import enum -import logging -import pathlib -from typing import ( - Optional, - Tuple, -) - import requests +import pathlib +import logging +from typing import Tuple, Optional _logger = logging.getLogger(__name__) @@ -20,7 +16,6 @@ class Project(enum.Enum): Names of github projects to be retrieved. Values have to match github project names. """ - CLOUD_STORAGE_EXTENSION = "cloud-storage-extension" KAFKA_CONNECTOR_EXTENSION = "kafka-connector-extension" @@ -33,33 +28,26 @@ def get_latest_version_and_jar_url(project: Project) -> Tuple[str, str]: :param project: name of the project :return: tuple with version and url to retrieve the artefact. """ - req = requests.get( - f"https://api.github.com/repos/exasol/{project.value}" f"/releases/latest", - timeout=10, - ) + req = requests.get(f"https://api.github.com/repos/exasol/{project.value}" + f"/releases/latest", timeout=10) if req.status_code != 200: - raise RuntimeError( - "Error sending request to the github, code: %d" % req.status_code - ) + raise RuntimeError("Error sending request to the github, code: %d" % + req.status_code) data = req.json() - version = data.get("tag_name") + version = data.get('tag_name') if version is None: - raise RuntimeError( - f"The latest version of {project.value} " f"has no tag, something is wrong" - ) - for asset in data.get("assets", []): - name = asset["name"] + raise RuntimeError(f"The latest version of {project.value} " + f"has no tag, something is wrong") + for asset in data.get('assets', []): + name = asset['name'] if name.endswith(f"{version}.jar"): - dl_url = asset["browser_download_url"] + dl_url = asset['browser_download_url'] return version, dl_url raise RuntimeError("Could not find proper jar url for the latest release") -def retrieve_jar( - project: Project, - use_local_cache: bool = True, - storage_path: Optional[pathlib.Path] = None, -) -> pathlib.Path: +def retrieve_jar(project: Project, use_local_cache: bool = True, + storage_path: Optional[pathlib.Path] = None) -> pathlib.Path: """ Returns latest jar file for the project, possibly using local cache. :param project: project to be used @@ -69,7 +57,7 @@ def retrieve_jar( :return: path to the jar file on the local filesystem """ version, jar_url = get_latest_version_and_jar_url(project) - _, local_jar_name = jar_url.rsplit("/", maxsplit=1) + _, local_jar_name = jar_url.rsplit('/', maxsplit=1) local_jar_path = pathlib.Path(local_jar_name) if storage_path is not None: if not storage_path.exists(): @@ -77,11 +65,8 @@ def retrieve_jar( local_jar_path = storage_path / local_jar_path if use_local_cache and local_jar_path.exists(): - _logger.info( - "Jar for version %s already exists in %s, skip downloading", - version, - local_jar_path, - ) + _logger.info("Jar for version %s already exists in %s, skip downloading", + version, local_jar_path) else: _logger.info("Fetching jar for version %s from %s...", version, jar_url) req = requests.get(jar_url, stream=True, timeout=10) diff --git a/exasol/itde_manager.py b/exasol/itde_manager.py index c4fc085..61580ce 100644 --- a/exasol/itde_manager.py +++ b/exasol/itde_manager.py @@ -1,14 +1,14 @@ from exasol_integration_test_docker_environment.lib import api # type: ignore -from exasol_integration_test_docker_environment.lib.docker import ( # type: ignore +from exasol_integration_test_docker_environment.lib.docker import ( # type: ignore ContextDockerClient, ) -from exasol_integration_test_docker_environment.lib.docker.container.utils import ( # type: ignore +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( # type: ignore remove_docker_container, ) -from exasol_integration_test_docker_environment.lib.docker.networks.utils import ( # type: ignore +from exasol_integration_test_docker_environment.lib.docker.networks.utils import ( # type: ignore remove_docker_networks, ) -from exasol_integration_test_docker_environment.lib.docker.volumes.utils import ( # type: ignore +from exasol_integration_test_docker_environment.lib.docker.volumes.utils import ( # type: ignore remove_docker_volumes, ) diff --git a/exasol/language_container_activation.py b/exasol/language_container_activation.py index 8ba4c0d..1248a93 100644 --- a/exasol/language_container_activation.py +++ b/exasol/language_container_activation.py @@ -1,5 +1,5 @@ -import re from typing import Dict +import re from exasol.secret_store import Secrets diff --git a/exasol/sagemaker_extension_wrapper.py b/exasol/sagemaker_extension_wrapper.py index b2fc040..62c0fa5 100644 --- a/exasol/sagemaker_extension_wrapper.py +++ b/exasol/sagemaker_extension_wrapper.py @@ -1,27 +1,21 @@ -from exasol_sagemaker_extension.deployment.deploy_create_statements import ( - DeployCreateStatements, # type: ignore -) -from exasol_sagemaker_extension.deployment.language_container_deployer import ( - LanguageActivationLevel, # type: ignore -) -from exasol_sagemaker_extension.deployment.sme_language_container_deployer import ( - SmeLanguageContainerDeployer, # type: ignore -) +from exasol_sagemaker_extension.deployment.deploy_create_statements import DeployCreateStatements # type: ignore +from exasol_sagemaker_extension.deployment.language_container_deployer import LanguageActivationLevel # type: ignore +from exasol_sagemaker_extension.deployment.sme_language_container_deployer import SmeLanguageContainerDeployer # type: ignore -from exasol.ai_lab_config import AILabConfig as CKey from exasol.connections import ( get_external_host, open_pyexasol_connection, ) from exasol.extension_wrapper_common import ( encapsulate_aws_credentials, - str_to_bool, + str_to_bool ) from exasol.language_container_activation import ( ACTIVATION_KEY_PREFIX, - get_activation_sql, + get_activation_sql ) from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey # Root directory in a bucket-fs bucket where all stuff of the Sagemaker # Extension, including its language container, will be uploaded. @@ -69,7 +63,7 @@ def deploy_language_container(conf: Secrets, version: str) -> None: bucketfs_port=int(str(conf.get(CKey.bfs_port))), bucketfs_user=conf.get(CKey.bfs_user), bucketfs_password=conf.get(CKey.bfs_password), - bucketfs_use_https=str_to_bool(conf, CKey.bfs_encryption, True), + bucketfs_use_https=str_to_bool(conf, CKey.bfs_encryption, True), bucket=conf.get(CKey.bfs_bucket), path_in_bucket=PATH_IN_BUCKET, use_ssl_cert_validation=str_to_bool(conf, CKey.cert_vld, True), @@ -105,21 +99,16 @@ def deploy_scripts(conf: Secrets) -> None: conn.execute(activation_sql) scripts_deployer = DeployCreateStatements( - exasol_conn=conn, - schema=conf.get(CKey.db_schema), - to_print=False, - develop=False, + exasol_conn=conn, schema=conf.get(CKey.db_schema), to_print=False, develop=False ) scripts_deployer.run() -def initialize_sme_extension( - conf: Secrets, - version: str = LATEST_KNOWN_VERSION, - run_deploy_container: bool = True, - run_deploy_scripts: bool = True, - run_encapsulate_aws_credentials: bool = True, -) -> None: +def initialize_sme_extension(conf: Secrets, + version: str = LATEST_KNOWN_VERSION, + run_deploy_container: bool = True, + run_deploy_scripts: bool = True, + run_encapsulate_aws_credentials: bool = True) -> None: """ Performs all necessary operations to get the Sagemaker Extension up and running. See the "Getting Started" and "Setup" sections of the diff --git a/exasol/secret_store.py b/exasol/secret_store.py index ebdd537..b0d2686 100644 --- a/exasol/secret_store.py +++ b/exasol/secret_store.py @@ -6,7 +6,7 @@ Iterable, Optional, Tuple, - Union, + Union ) from sqlcipher3 import dbapi2 as sqlcipher # type: ignore @@ -123,9 +123,8 @@ def insert(cur) -> None: insert(cur) return self - def get( - self, key: Union[str, CKey], default_value: Optional[str] = None - ) -> Optional[str]: + def get(self, key: Union[str, CKey], default_value: Optional[str] = None) -> Optional[str]: + key = key.name if isinstance(key, CKey) else key with self._cursor() as cur: diff --git a/exasol/transformers_extension_wrapper.py b/exasol/transformers_extension_wrapper.py index 45a620e..13cc258 100644 --- a/exasol/transformers_extension_wrapper.py +++ b/exasol/transformers_extension_wrapper.py @@ -1,40 +1,28 @@ -from exasol_transformers_extension.deployment.language_container_deployer import ( - LanguageActivationLevel, # type: ignore -) -from exasol_transformers_extension.deployment.scripts_deployer import ( - ScriptsDeployer, # type: ignore -) -from exasol_transformers_extension.deployment.te_language_container_deployer import ( - TeLanguageContainerDeployer, # type: ignore -) -from exasol_transformers_extension.utils.bucketfs_operations import ( - create_bucketfs_location, # type: ignore -) -from exasol_transformers_extension.utils.bucketfs_operations import ( - get_model_path, # type: ignore -) -from exasol_transformers_extension.utils.bucketfs_operations import ( - upload_model_files_to_bucketfs, # type: ignore -) +from exasol_transformers_extension.utils.bucketfs_operations import get_model_path # type: ignore +from exasol_transformers_extension.utils.bucketfs_operations import upload_model_files_to_bucketfs # type: ignore +from exasol_transformers_extension.utils.bucketfs_operations import create_bucketfs_location # type: ignore + +from exasol_transformers_extension.deployment.language_container_deployer import LanguageActivationLevel # type: ignore +from exasol_transformers_extension.deployment.scripts_deployer import ScriptsDeployer # type: ignore +from exasol_transformers_extension.deployment.te_language_container_deployer import TeLanguageContainerDeployer # type: ignore + +# TODO: Disable this mypy "missing imports" nonsense. -from exasol.ai_lab_config import AILabConfig as CKey from exasol.connections import ( get_external_host, - open_pyexasol_connection, + open_pyexasol_connection ) from exasol.extension_wrapper_common import ( encapsulate_bucketfs_credentials, encapsulate_huggingface_token, - str_to_bool, + str_to_bool ) from exasol.language_container_activation import ( ACTIVATION_KEY_PREFIX, - get_activation_sql, + get_activation_sql ) from exasol.secret_store import Secrets - -# TODO: Disable this mypy "missing imports" nonsense. - +from exasol.ai_lab_config import AILabConfig as CKey # Root directory in a bucket-fs bucket where all stuff of the Transformers # Extension, including its language container, will be uploaded. @@ -53,7 +41,7 @@ BFS_CONNECTION_PREFIX = "TE_BFS" # Models will be uploaded into this directory in bucket-fs. -BFS_MODELS_DIR = "te_models" +BFS_MODELS_DIR = 'te_models' # The name of the connection object with a Huggingface token will be prefixed # with this string. @@ -64,7 +52,9 @@ MODELS_CACHE_DIR = "models_cache" -def deploy_language_container(conf: Secrets, version: str, language_alias: str) -> None: +def deploy_language_container(conf: Secrets, + version: str, + language_alias: str) -> None: """ Calls the Transformers Extension's language container deployment API. Downloads the specified released version of the extension from the GitHub @@ -104,7 +94,7 @@ def deploy_language_container(conf: Secrets, version: str, language_alias: str) use_ssl_cert_validation=str_to_bool(conf, CKey.cert_vld, True), ssl_trusted_ca=conf.get(CKey.trusted_ca), ssl_client_certificate=conf.get(CKey.client_cert), - ssl_private_key=conf.get(CKey.client_key), + ssl_private_key=conf.get(CKey.client_key) ) # Install the language container. @@ -117,7 +107,8 @@ def deploy_language_container(conf: Secrets, version: str, language_alias: str) conf.save(ACTIVATION_KEY, activation_sql) -def deploy_scripts(conf: Secrets, language_alias: str) -> None: +def deploy_scripts(conf: Secrets, + language_alias: str) -> None: """ Deploys all the extension's scripts to the database. @@ -135,21 +126,17 @@ def deploy_scripts(conf: Secrets, language_alias: str) -> None: activation_sql = get_activation_sql(conf) conn.execute(activation_sql) - scripts_deployer = ScriptsDeployer( - language_alias, conf.get(CKey.db_schema), conn - ) + scripts_deployer = ScriptsDeployer(language_alias, conf.get(CKey.db_schema), conn) scripts_deployer.deploy_scripts() -def initialize_te_extension( - conf: Secrets, - version: str = LATEST_KNOWN_VERSION, - language_alias: str = LANGUAGE_ALIAS, - run_deploy_container: bool = True, - run_deploy_scripts: bool = True, - run_encapsulate_bfs_credentials: bool = True, - run_encapsulate_hf_token: bool = True, -) -> None: +def initialize_te_extension(conf: Secrets, + version: str = LATEST_KNOWN_VERSION, + language_alias: str = LANGUAGE_ALIAS, + run_deploy_container: bool = True, + run_deploy_scripts: bool = True, + run_encapsulate_bfs_credentials: bool = True, + run_encapsulate_hf_token: bool = True) -> None: """ Performs all necessary operations to get the Transformers Extension up and running. See the "Getting Started" and "Setup" sections of the @@ -204,7 +191,10 @@ def initialize_te_extension( conf.save(CKey.te_models_cache_dir, MODELS_CACHE_DIR) -def upload_model_from_cache(conf: Secrets, model_name: str, cache_dir: str) -> None: +def upload_model_from_cache( + conf: Secrets, + model_name: str, + cache_dir: str) -> None: """ Uploads model previously downloaded and cached on a local drive. This, for instance, could have been done with the following code. @@ -226,22 +216,21 @@ def upload_model_from_cache(conf: Secrets, model_name: str, cache_dir: str) -> N # Create bucketfs location bfs_host = conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)) bucketfs_location = create_bucketfs_location( - conf.get(CKey.bfs_service), - bfs_host, - int(str(conf.get(CKey.bfs_port))), - str(conf.get(CKey.bfs_encryption)).lower() == "true", - conf.get(CKey.bfs_user), - conf.get(CKey.bfs_password), - conf.get(CKey.bfs_bucket), - PATH_IN_BUCKET, - ) + conf.get(CKey.bfs_service), bfs_host, + int(str(conf.get(CKey.bfs_port))), str(conf.get(CKey.bfs_encryption)).lower() == 'true', + conf.get(CKey.bfs_user), conf.get(CKey.bfs_password), conf.get(CKey.bfs_bucket), + PATH_IN_BUCKET) # Upload the downloaded model files into bucketfs upload_path = get_model_path(conf.get(CKey.te_models_bfs_dir), model_name) upload_model_files_to_bucketfs(cache_dir, upload_path, bucketfs_location) -def upload_model(conf: Secrets, model_name: str, cache_dir: str, **kwargs) -> None: +def upload_model( + conf: Secrets, + model_name: str, + cache_dir: str, + **kwargs) -> None: """ Uploads model from the Huggingface hub or from the local cache in case it has already been downloaded from the hub. The user token, if found in the @@ -260,15 +249,12 @@ def upload_model(conf: Secrets, model_name: str, cache_dir: str, **kwargs) -> No methods of the AutoTokenizer and AutoModel. The user token, if specified here, will be used instead of the one in the secret store. """ - from transformers import ( # type: ignore - AutoModel, - AutoTokenizer, - ) + from transformers import AutoTokenizer, AutoModel # type: ignore - if "token" not in kwargs: + if 'token' not in kwargs: token = conf.get(CKey.huggingface_token) if token: - kwargs["token"] = token + kwargs['token'] = token AutoTokenizer.from_pretrained(model_name, cache_dir=cache_dir, **kwargs) AutoModel.from_pretrained(model_name, cache_dir=cache_dir, **kwargs) diff --git a/test/integration/test_itde_manager.py b/test/integration/test_itde_manager.py index b20aad2..08f9582 100644 --- a/test/integration/test_itde_manager.py +++ b/test/integration/test_itde_manager.py @@ -37,9 +37,7 @@ def test_bring_itde_up(secrets): assert secrets.get(AILabConfig.itde_container) == DB_CONTAINER_NAME assert secrets.get(AILabConfig.itde_volume) == DB_VOLUME_NAME assert secrets.get(AILabConfig.itde_network) == DB_NETWORK_NAME - assert secrets.get(AILabConfig.db_host_name) == secrets.get( - AILabConfig.bfs_host_name - ) + assert secrets.get(AILabConfig.db_host_name) == secrets.get(AILabConfig.bfs_host_name) assert secrets.get(AILabConfig.db_user) == "sys" assert secrets.get(AILabConfig.db_password) == "exasol" assert secrets.get(AILabConfig.db_encryption) == "True" diff --git a/test/integration/test_sagemaker_extension_wrapper.py b/test/integration/test_sagemaker_extension_wrapper.py index f9ffa07..50f1477 100644 --- a/test/integration/test_sagemaker_extension_wrapper.py +++ b/test/integration/test_sagemaker_extension_wrapper.py @@ -1,18 +1,20 @@ +from exasol.sagemaker_extension_wrapper import initialize_sme_extension +from exasol.connections import open_pyexasol_connection +from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey from test.utils.integration_test_utils import ( + setup_itde, activate_languages, assert_connection_exists, assert_run_empty_udf, - get_script_counts, - setup_itde, + get_script_counts ) -from exasol.ai_lab_config import AILabConfig as CKey -from exasol.connections import open_pyexasol_connection -from exasol.sagemaker_extension_wrapper import initialize_sme_extension -from exasol.secret_store import Secrets - -def test_initialize_sme_extension(secrets: Secrets, setup_itde): +def test_initialize_sme_extension( + secrets: Secrets, + setup_itde +): # Here are fake AWS credentials. Should be fine since we are only testing # the deployment. secrets.save(CKey.aws_bucket, "NoneExistent") @@ -29,6 +31,4 @@ def test_initialize_sme_extension(secrets: Secrets, setup_itde): script_counts = get_script_counts(pyexasol_connection, secrets) assert script_counts["SCRIPTING"] >= 4 assert script_counts["UDF"] >= 5 - assert_connection_exists( - secrets.get(CKey.sme_aws_connection), pyexasol_connection - ) + assert_connection_exists(secrets.get(CKey.sme_aws_connection), pyexasol_connection) diff --git a/test/integration/test_transformers_extension_wrapper.py b/test/integration/test_transformers_extension_wrapper.py index 57b0ab9..7ccb268 100644 --- a/test/integration/test_transformers_extension_wrapper.py +++ b/test/integration/test_transformers_extension_wrapper.py @@ -1,25 +1,29 @@ -from test.utils.integration_test_utils import ( - activate_languages, - assert_connection_exists, - assert_run_empty_udf, - get_script_counts, - setup_itde, -) - from _pytest.fixtures import FixtureRequest -from exasol.ai_lab_config import AILabConfig as CKey from exasol.connections import open_pyexasol_connection from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey from exasol.transformers_extension_wrapper import initialize_te_extension +from test.utils.integration_test_utils import ( + setup_itde, + activate_languages, + assert_connection_exists, + assert_run_empty_udf, + get_script_counts +) -def test_initialize_te_extension(request: FixtureRequest, secrets: Secrets, setup_itde): +def test_initialize_te_extension( + request: FixtureRequest, + secrets: Secrets, + setup_itde +): test_name: str = request.node.name language_alias = f"PYTHON3_TE_{test_name.upper()}" secrets.save(CKey.huggingface_token, "abc") with open_pyexasol_connection(secrets) as pyexasol_connection: + # Run the extension deployment. initialize_te_extension(secrets, language_alias=language_alias) @@ -27,9 +31,5 @@ def test_initialize_te_extension(request: FixtureRequest, secrets: Secrets, setu assert_run_empty_udf(language_alias, pyexasol_connection, secrets) script_counts = get_script_counts(pyexasol_connection, secrets) assert script_counts["UDF"] > 5 - assert_connection_exists( - secrets.get(CKey.te_bfs_connection), pyexasol_connection - ) - assert_connection_exists( - secrets.get(CKey.te_hf_connection), pyexasol_connection - ) + assert_connection_exists(secrets.get(CKey.te_bfs_connection), pyexasol_connection) + assert_connection_exists(secrets.get(CKey.te_hf_connection), pyexasol_connection) diff --git a/test/unit/test_bfs_utils.py b/test/unit/test_bfs_utils.py index 8096a17..0be5e28 100644 --- a/test/unit/test_bfs_utils.py +++ b/test/unit/test_bfs_utils.py @@ -1,11 +1,10 @@ -import pathlib -from typing import Generator -from unittest import mock - import pytest - +from unittest import mock +from typing import Generator +import pathlib from exasol import bfs_utils + MOCKED_BUCKET = "bucket" MOCKED_FILE_NAME = "bfs.file" diff --git a/test/unit/test_connections.py b/test/unit/test_connections.py index 8c47224..0023efe 100644 --- a/test/unit/test_connections.py +++ b/test/unit/test_connections.py @@ -8,7 +8,6 @@ import pytest -from exasol.ai_lab_config import AILabConfig as CKey from exasol.connections import ( get_external_host, open_bucketfs_connection, @@ -16,6 +15,7 @@ open_sqlalchemy_connection, ) from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey @pytest.fixture @@ -44,19 +44,14 @@ def mock_get(self, key: str, default_value: Optional[str] = None) -> Optional[st def test_get_external_host(conf): - assert ( - get_external_host(conf) - == f"{conf.get(CKey.db_host_name)}:{conf.get(CKey.db_port)}" - ) + assert get_external_host(conf) == f"{conf.get(CKey.db_host_name)}:{conf.get(CKey.db_port)}" @unittest.mock.patch("pyexasol.connect") def test_open_pyexasol_connection(mock_connect, conf): open_pyexasol_connection(conf) mock_connect.assert_called_once_with( - dsn=get_external_host(conf), - user=conf.get(CKey.db_user), - password=conf.get(CKey.db_password), + dsn=get_external_host(conf), user=conf.get(CKey.db_user), password=conf.get(CKey.db_password) ) diff --git a/test/unit/test_github.py b/test/unit/test_github.py index 2087a13..29e1858 100644 --- a/test/unit/test_github.py +++ b/test/unit/test_github.py @@ -1,10 +1,8 @@ import os -import pathlib -from unittest import mock - import pytest +import pathlib import requests - +from unittest import mock from exasol import github CSE_MOCK_URL = "https://github.com/some_path/exasol-cloud-storage-extension-2.7.8.jar" @@ -19,8 +17,8 @@ { "name": "exasol-cloud-storage-extension-2.7.8.jar", "browser_download_url": CSE_MOCK_URL, - }, - ], + } + ] } @@ -37,7 +35,7 @@ def mocked_requests_get(*args, **_): res.status_code = 500 elif url == CSE_MOCK_URL: res.status_code = 200 - res.content = b"binary data" + res.content = b'binary data' return res @@ -58,13 +56,11 @@ def test_retrieve_jar(_, tmpdir, caplog): # fetch for the first time, local dir jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION) assert jar_path.exists() - assert jar_path.read_bytes() == b"binary data" + assert jar_path.read_bytes() == b'binary data' # ensure file is recreated without cache old_ts = jar_path.lstat().st_ctime - jar_path = github.retrieve_jar( - github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=False - ) + jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=False) assert jar_path.exists() assert old_ts < jar_path.lstat().st_ctime @@ -72,20 +68,15 @@ def test_retrieve_jar(_, tmpdir, caplog): caplog.set_level("INFO") caplog.clear() old_ts = jar_path.lstat().st_ctime_ns - jar_path = github.retrieve_jar( - github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=True - ) + jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=True) assert jar_path.lstat().st_ctime_ns == old_ts assert "skip downloading" in caplog.text # test storage path specification caplog.clear() stg_path = pathlib.Path(tmpdir.mkdir("sub")) - jar_path_sub = github.retrieve_jar( - github.Project.CLOUD_STORAGE_EXTENSION, - use_local_cache=True, - storage_path=stg_path, - ) + jar_path_sub = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, + use_local_cache=True, storage_path=stg_path) assert jar_path_sub.exists() assert jar_path != jar_path_sub assert "Fetching jar" in caplog.text diff --git a/test/unit/test_itde_manager.py b/test/unit/test_itde_manager.py index 66f5708..51c60df 100644 --- a/test/unit/test_itde_manager.py +++ b/test/unit/test_itde_manager.py @@ -15,6 +15,7 @@ ) from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports + from exasol.ai_lab_config import AILabConfig as CKey from exasol.itde_manager import ( ENVIRONMENT_NAME, diff --git a/test/unit/test_secret_store.py b/test/unit/test_secret_store.py index a0002a5..67deb37 100644 --- a/test/unit/test_secret_store.py +++ b/test/unit/test_secret_store.py @@ -2,11 +2,11 @@ import pytest -from exasol.ai_lab_config import AILabConfig as CKey from exasol.secret_store import ( InvalidPassword, Secrets, ) +from exasol.ai_lab_config import AILabConfig as CKey def test_no_database_file(secrets): diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index fbb35f8..5bbf270 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -3,10 +3,7 @@ import pytest -from exasol.utils import ( - optional_str_to_bool, - upward_file_search, -) +from exasol.utils import upward_file_search, optional_str_to_bool def test_upward_file_search(): @@ -37,23 +34,20 @@ def test_upward_file_search_failure(): os.chdir(current_dir) -@pytest.mark.parametrize( - "v, expected", - [ - (None, None), - ("y", True), - ("yes", True), - ("true", True), - ("Y", True), - ("YES", True), - ("TRUE", True), - ("n", False), - ("no", False), - ("false", False), - ("N", False), - ("NO", False), - ("FALSE", False), - ], -) +@pytest.mark.parametrize("v, expected", [ + (None, None), + ('y', True), + ('yes', True), + ('true', True), + ('Y', True), + ('YES', True), + ('TRUE', True), + ('n', False), + ('no', False), + ('false', False), + ('N', False), + ('NO', False), + ('FALSE', False) +]) def test_optional_str_to_bool(v, expected): assert optional_str_to_bool(v) == expected diff --git a/test/utils/integration_test_utils.py b/test/utils/integration_test_utils.py index dafbdb0..1e2c37a 100644 --- a/test/utils/integration_test_utils.py +++ b/test/utils/integration_test_utils.py @@ -1,17 +1,17 @@ import textwrap from typing import Dict - import pytest + from pyexasol import ExaConnection -from exasol.ai_lab_config import AILabConfig -from exasol.connections import open_pyexasol_connection +from exasol.language_container_activation import get_activation_sql +from exasol.secret_store import Secrets from exasol.itde_manager import ( bring_itde_up, - take_itde_down, + take_itde_down ) -from exasol.language_container_activation import get_activation_sql -from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig +from exasol.connections import open_pyexasol_connection @pytest.fixture @@ -23,7 +23,7 @@ def setup_itde(secrets) -> None: bring_itde_up(secrets) - schema = "INTEGRATION_TEST" + schema = 'INTEGRATION_TEST' secrets.save(AILabConfig.db_schema, schema) with open_pyexasol_connection(secrets) as pyexasol_connection: pyexasol_connection.execute(f"CREATE SCHEMA {schema};")