diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6cc907772..2c873f7ab 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -88,7 +88,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run type-check - run: poetry run nox -s lint:typing || echo ignoring... + run: poetry run nox -s lint:typing Security: name: Security Checks (Python-${{ matrix.python-version }}) @@ -109,7 +109,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run security linter - run: poetry run nox -s lint:security || echo ignoring... + run: poetry run nox -s lint:security - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 01bae738c..252ba4848 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -9,4 +9,5 @@ Code name: * #119: Refactored `pkg_resources` usage to `importlib.resources` * #420: Added file `py.typed` to enable mypy to find project specific types * #418: Use exasol/python-toolbox -* #411: Removed usage of exasol-bucketfs +* #411: Removed usage of exasol-bucketfs +* #425: Fixed type checks found by MyPy diff --git a/exasol_integration_test_docker_environment/cli/termination_handler.py b/exasol_integration_test_docker_environment/cli/termination_handler.py index a2330492e..89cef1e3d 100644 --- a/exasol_integration_test_docker_environment/cli/termination_handler.py +++ b/exasol_integration_test_docker_environment/cli/termination_handler.py @@ -48,7 +48,7 @@ def _handle_failure(self, task_error: TaskRuntimeError): def _print_task_failures(task_error: TaskRuntimeError): print_err() print_err("Task failure message: %s" % task_error.msg) - print_err(task_error.__cause__.args[0]) + print_err(task_error.__cause__.args[0]) # type: ignore print_err() def _handle_success(self): diff --git a/exasol_integration_test_docker_environment/doctor.py b/exasol_integration_test_docker_environment/doctor.py index 57e489610..4fe8bc21b 100644 --- a/exasol_integration_test_docker_environment/doctor.py +++ b/exasol_integration_test_docker_environment/doctor.py @@ -3,7 +3,8 @@ package and also provide help to find potential fixes. """ import sys -from typing import Iterable +from collections.abc import Callable +from typing import Iterable, List, Tuple from exasol import error from enum import Enum @@ -78,9 +79,11 @@ def health_checkup() -> Iterable[error.ExaError]: return an iterator of error codes specifying which problems have been identified. """ - examinations = [ + check_function = Callable[[],bool] + diagnosis_function = Callable[[],Iterable[error.ExaError]] + examinations : List[Tuple[check_function, diagnosis_function]] = [ (is_docker_daemon_available, diagnose_docker_daemon_not_available), - (is_supported_platform, lambda: Error.TargetPlatformNotSupported), + (is_supported_platform, lambda: [Error.TargetPlatformNotSupported]), ] for is_fine, diagnosis in examinations: if not is_fine(): diff --git a/exasol_integration_test_docker_environment/lib/api/api_errors.py b/exasol_integration_test_docker_environment/lib/api/api_errors.py index 40c51e152..d1d30475d 100644 --- a/exasol_integration_test_docker_environment/lib/api/api_errors.py +++ b/exasol_integration_test_docker_environment/lib/api/api_errors.py @@ -12,13 +12,16 @@ class HealthProblem(RuntimeError): class TaskFailures(Exception): """Represents a potential cause of a TaskRuntimeError""" - def __init__(self, inner: List[str] = None): + def __init__(self, inner: Optional[List[str]] = None): super().__init__(self._construct_exception_message(inner)) self.inner = inner - def _construct_exception_message(self, failures: Iterable[str]) -> str: - formatted_task_failures = "\n".join(failures) - return f"Following task failures were caught during the execution:\n{formatted_task_failures}" + def _construct_exception_message(self, failures: Optional[Iterable[str]]) -> str: + if failures is not None: + formatted_task_failures = "\n".join(failures) + return f"Following task failures were caught during the execution:\n{formatted_task_failures}" + else: + return f"No task failures were caught during the execution:" class TaskRuntimeError(RuntimeError): diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index 9603ac2a6..37921ca32 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -28,7 +28,7 @@ class JobCounterSingleton(object): """ - We use here a Singleton to avoid a unprotected global variable. + We use here a Singleton to avoid an unprotected global variable. However, this counter needs to be global counter to guarantee unique job ids. This is needed in case of task that finish in less than a second, to avoid duplicated job ids """ @@ -42,8 +42,9 @@ def __new__(cls): return cls._instance def get_next_value(self) -> int: - self._counter += 1 - return self._counter + # self._counter is a class variable and because of this we need to suppress type checks + self._counter += 1 # type: ignore + return self._counter # type: ignore def set_build_config(force_rebuild: bool, @@ -51,9 +52,9 @@ def set_build_config(force_rebuild: bool, force_pull: bool, log_build_context_content: bool, output_directory: str, - temporary_base_directory: str, - cache_directory: str, - build_name: str, ): + temporary_base_directory: Optional[str], + cache_directory: Optional[str], + build_name: Optional[str], ): luigi.configuration.get_config().set('build_config', 'force_rebuild', str(force_rebuild)) luigi.configuration.get_config().set('build_config', 'force_rebuild_from', json.dumps(force_rebuild_from)) luigi.configuration.get_config().set('build_config', 'force_pull', str(force_pull)) @@ -72,9 +73,8 @@ def set_output_directory(output_directory): luigi.configuration.get_config().set('build_config', 'output_directory', output_directory) -def set_docker_repository_config(docker_password: str, docker_repository_name: str, docker_username: str, - tag_prefix: str, - kind: str): +def set_docker_repository_config(docker_password: Optional[str], docker_repository_name: Optional[str], + docker_username: Optional[str], tag_prefix: str, kind: str): config_class = f'{kind}_docker_repository_config' luigi.configuration.get_config().set(config_class, 'tag_prefix', tag_prefix) if docker_repository_name is not None: @@ -105,6 +105,7 @@ def import_build_steps(flavor_path: Tuple[str, ...]): path_to_build_steps = Path(path).joinpath("flavor_base/build_steps.py") module_name_for_build_steps = extract_modulename_for_build_steps(path) spec = importlib.util.spec_from_file_location(module_name_for_build_steps, path_to_build_steps) + assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -119,7 +120,7 @@ def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int = 2, task_dependencies_dot_file: Optional[str] = None, - log_level: str = None, use_job_specific_log_file: bool = False) \ + log_level: Optional[str] = None, use_job_specific_log_file: bool = False) \ -> Any: setup_worker() task = task_creator() @@ -141,7 +142,6 @@ def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int f"The detailed log of the integration-test-docker-environment can be found at: {log_file_path}") task.cleanup(success) - def _run_task_with_logging_config( task: DependencyLoggerBaseTask, log_file_path: Path, @@ -153,7 +153,6 @@ def _run_task_with_logging_config( local_scheduler=True, **run_kwargs) return no_scheduling_errors - def _handle_task_result( no_scheduling_errors: bool, success: bool, @@ -199,7 +198,7 @@ def _configure_logging( yield run_kwargs -def _configure_logging_parameter(log_level: str, luigi_config: Path, use_job_specific_log_file: bool) \ +def _configure_logging_parameter(log_level: Optional[str], luigi_config: Path, use_job_specific_log_file: bool) \ -> Tuple[bool, Dict[str, str]]: if use_job_specific_log_file: no_configure_logging = False @@ -226,13 +225,13 @@ def generate_graph_from_task_dependencies(task: DependencyLoggerBaseTask, task_d def collect_dependencies(task: DependencyLoggerBaseTask) -> Set[TaskDependency]: - dependencies = set() + dependencies : Set[TaskDependency] = set() for root, directories, files in os.walk(task._get_dependencies_path_for_job()): for file in files: file_path = Path(root).joinpath(file) with open(file_path) as f: for line in f.readlines(): - task_dependency = TaskDependency.from_json(line) # type: TaskDependency + task_dependency : TaskDependency = TaskDependency.from_json(line) # type: ignore if task_dependency.state == DependencyState.requested.name: dependencies.add(task_dependency) return dependencies diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py index d141473a4..28e57a41f 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py @@ -1,5 +1,5 @@ import functools -from typing import Tuple, Optional, Callable +from typing import Tuple, Optional, Callable, Any import humanfriendly from exasol_integration_test_docker_environment.lib.api.common import ( @@ -26,8 +26,10 @@ def _cleanup(environment_info: EnvironmentInfo) -> None: - remove_docker_container([environment_info.database_info.container_info.container_name]) - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) + if environment_info.database_info.container_info is not None: + remove_docker_container([environment_info.database_info.container_info.container_name]) + if environment_info.database_info.container_info.volume_name is not None: + remove_docker_volumes([environment_info.database_info.container_info.volume_name]) remove_docker_networks([environment_info.network_info.network_name]) @@ -69,7 +71,7 @@ def spawn_test_environment( raises: TaskRuntimeError if spawning the test environment fails """ - def str_or_none(x: any) -> str: + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) @@ -78,6 +80,8 @@ def str_or_none(x: any) -> str: parsed_db_disk_size = humanfriendly.parse_size(db_disk_size) if parsed_db_disk_size < humanfriendly.parse_size("100 MiB"): raise ArgumentConstraintError("db_disk_size", "needs to be at least 100 MiB") + db_os_access_value = DbOsAccess[db_os_access] if db_os_access else DbOsAccess.DOCKER_EXEC + set_build_config(False, tuple(), False, @@ -101,7 +105,7 @@ def str_or_none(x: any) -> str: docker_runtime=docker_runtime, docker_db_image_version=docker_db_image_version, docker_db_image_name=docker_db_image_name, - db_os_access=DbOsAccess[db_os_access], + db_os_access=db_os_access_value, db_user="sys", db_password="exasol", bucketfs_write_password="write", diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py index 41da7bd81..c75a87ada 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py @@ -1,5 +1,5 @@ import functools -from typing import Tuple, Optional, Callable +from typing import Tuple, Optional, Callable, Any import humanfriendly from exasol_integration_test_docker_environment.lib.api.common import set_build_config, set_docker_repository_config, \ @@ -22,14 +22,15 @@ .docker_db_test_environment_parameter import DbOsAccess def _cleanup(environment_info: EnvironmentInfo) -> None: - if environment_info.test_container_info is None: - remove_docker_container([environment_info.database_info.container_info.container_name]) - else: - remove_docker_container([environment_info.test_container_info.container_name, - environment_info.database_info.container_info.container_name]) - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) - remove_docker_networks([environment_info.network_info.network_name]) + if test_container_info := environment_info.test_container_info: + remove_docker_container([test_container_info.container_name]) + + if db_container_info := environment_info.database_info.container_info: + remove_docker_container([db_container_info.container_name]) + if name := db_container_info.volume_name: + remove_docker_volumes([name]) + remove_docker_networks([environment_info.network_info.network_name]) @no_cli_function def spawn_test_environment_with_test_container( @@ -71,7 +72,7 @@ def spawn_test_environment_with_test_container( raises: TaskRuntimeError if spawning the test environment fails """ - def str_or_none(x: any) -> str: + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"): diff --git a/exasol_integration_test_docker_environment/lib/base/base_task.py b/exasol_integration_test_docker_environment/lib/base/base_task.py index f559b8dfd..ed840cd01 100644 --- a/exasol_integration_test_docker_environment/lib/base/base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/base_task.py @@ -3,7 +3,7 @@ import logging import shutil from pathlib import Path -from typing import Dict, List, Generator, Any, Union, Set, Optional +from typing import Dict, List, Generator, Any, Union, Set import luigi import six @@ -85,8 +85,8 @@ def get_output(self) -> Any: class BaseTask(Task): - caller_output_path = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) - job_id = luigi.Parameter() + caller_output_path : List[str] = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) # type: ignore + job_id : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): self._registered_tasks = [] @@ -184,7 +184,7 @@ def get_log_path(self) -> Path: return path def get_cache_path(self) -> Path: - path = Path(build_config().output_directory, "cache") + path = Path(str(build_config().output_directory), "cache") path.mkdir(parents=True, exist_ok=True) return path diff --git a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py index 46c1c67b6..50d8463a9 100644 --- a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py +++ b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py @@ -3,7 +3,7 @@ import docker import time from docker import DockerClient -from typing import Protocol, runtime_checkable +from typing import Protocol, runtime_checkable, Optional from docker.models.containers import Container, ExecResult from exasol_integration_test_docker_environment \ .lib.base.ssh_access import SshKey @@ -37,19 +37,18 @@ class DbOsExecutor(Protocol): concrete implementations in sub-classes ``DockerExecutor`` and ``SshExecutor``. """ - @abstractmethod def exec(self, cmd: str) -> ExecResult: ... def prepare(self): - pass + ... class DockerExecutor(DbOsExecutor): def __init__(self, docker_client: DockerClient, container_name: str): self._client = docker_client self._container_name = container_name - self._container = None + self._container : Optional[Container] = None def __enter__(self): self._container = self._client.containers.get(self._container_name) @@ -62,8 +61,12 @@ def __del__(self): self.close() def exec(self, cmd: str) -> ExecResult: + assert self._container return self._container.exec_run(cmd) + def prepare(self): + pass + def close(self): self._container = None if self._client is not None: @@ -75,7 +78,7 @@ class SshExecutor(DbOsExecutor): def __init__(self, connect_string: str, key_file: str): self._connect_string = connect_string self._key_file = key_file - self._connection = None + self._connection : Optional[fabric.Connection] = None def __enter__(self): key = SshKey.read_from(self._key_file) @@ -92,6 +95,7 @@ def __del__(self): self.close() def exec(self, cmd: str) -> ExecResult: + assert self._connection result = self._connection.run(cmd, warn=True, hide=True) output = result.stdout.encode("utf-8") return ExecResult(result.exited, output) @@ -143,9 +147,11 @@ def executor(self) -> DbOsExecutor: return DockerExecutor(client, self._container_name) + class SshExecFactory(DbOsExecFactory): @classmethod def from_database_info(cls, info: DatabaseInfo): + assert info.ssh_info return SshExecFactory( f"{info.ssh_info.user}@{info.host}:{info.ports.ssh}", info.ssh_info.key_file, diff --git a/exasol_integration_test_docker_environment/lib/base/flavor_task.py b/exasol_integration_test_docker_environment/lib/base/flavor_task.py index 9c7ac71d3..940f00b44 100644 --- a/exasol_integration_test_docker_environment/lib/base/flavor_task.py +++ b/exasol_integration_test_docker_environment/lib/base/flavor_task.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, List import luigi @@ -8,7 +8,7 @@ class FlavorsBaseTask(DependencyLoggerBaseTask): - flavor_paths = luigi.ListParameter() + flavor_paths : List[str] = luigi.ListParameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py index b259e6f83..067d7e662 100644 --- a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py +++ b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any +from typing import Any, Optional import jsonpickle from luigi import LocalTarget @@ -10,7 +10,7 @@ class JsonPickleTarget(LocalTarget): def __init__(self, path: Path, is_tmp: bool = False): super().__init__(path=str(path), is_tmp=is_tmp) - def write(self, obj:Any, indent:int=None): + def write(self, obj:Any, indent: Optional[int] = None): jsonpickle.set_preferred_backend('simplejson') jsonpickle.set_encoder_options('simplejson', indent=indent) json_str=jsonpickle.encode(obj) diff --git a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py index bb7cd830c..359d1fbdd 100644 --- a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py +++ b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py @@ -2,10 +2,12 @@ import os import tempfile from pathlib import Path -from typing import Optional, Callable +from typing import Optional, Callable, Generator, List, Any +from dataclasses import dataclass import jinja2 import logging + from exasol_integration_test_docker_environment.lib import PACKAGE_NAME from exasol_integration_test_docker_environment.lib.config.build_config import build_config @@ -40,28 +42,30 @@ def get_log_path(job_id: str) -> Path: log_path_dir.mkdir(parents=True, exist_ok=True) return log_path +@dataclass +class LogInfoStorage: + level : int + handlers: List[logging.Handler] + filters: List[Any] + propagate: bool @contextlib.contextmanager def restore_logger(logger_creator: Callable[[], logging.Logger]): before_logger = logger_creator() - logger_info = { - LOG_LEVEL: before_logger.level, - HANDLERS: list(before_logger.handlers), - FILTERS: list(before_logger.filters), - PROPAGATE: before_logger.propagate - } + logger_info_before = LogInfoStorage(level=before_logger.level, handlers=list(before_logger.handlers), + filters=list(before_logger.filters), propagate=before_logger.propagate) yield after_logger = logger_creator() - after_logger.level = logger_info[LOG_LEVEL] - after_logger.handlers = logger_info[HANDLERS] - after_logger.filters = logger_info[FILTERS] - after_logger.propagate = logger_info[PROPAGATE] + after_logger.level = logger_info_before.level + after_logger.handlers = logger_info_before.handlers + after_logger.filters = logger_info_before.filters + after_logger.propagate = logger_info_before.propagate @contextlib.contextmanager def get_luigi_log_config(log_file_target: Path, use_job_specific_log_file: bool, - log_level: Optional[str] = None) -> Path: + log_level: Optional[str] = None) -> Generator[Path, None, None]: """ Yields a context manager containing the path of the log-config file. log_file_target contains the location of the log-file. diff --git a/exasol_integration_test_docker_environment/lib/base/ssh_access.py b/exasol_integration_test_docker_environment/lib/base/ssh_access.py index 8da3a585d..0a792cab9 100644 --- a/exasol_integration_test_docker_environment/lib/base/ssh_access.py +++ b/exasol_integration_test_docker_environment/lib/base/ssh_access.py @@ -6,8 +6,7 @@ import portalocker from pathlib import Path from string import Template -from typing import Optional - +from typing import Optional, Union _LOCK_FILE = "$TMP/$MODULE-ssh-access.lock" _DEFAULT_CACHE_DIR = "$HOME/.cache/exasol/$MODULE" @@ -84,7 +83,7 @@ def opener(path, flags): return self @classmethod - def read_from(cls, private_key_file: Path) -> 'SshKey': + def read_from(cls, private_key_file: Union[Path, str]) -> 'SshKey': with open(private_key_file, "r") as file: rsa_key = paramiko.RSAKey.from_private_key(file) return SshKey(rsa_key) @@ -95,7 +94,7 @@ def generate(cls) -> 'SshKey': return SshKey(rsa_key) @classmethod - def from_cache(cls, cache_directory: Path = None) -> 'SshKey': + def from_cache(cls, cache_directory: Optional[Path] = None) -> 'SshKey': cache = SshKeyCache(cache_directory) priv = cache.private_key with portalocker.Lock(_path(_LOCK_FILE), 'wb', timeout=10) as fh: @@ -106,6 +105,6 @@ def from_cache(cls, cache_directory: Path = None) -> 'SshKey': os.makedirs(cache.directory, mode=0o700, exist_ok=True) return ( cls.generate() - .write_private_key(priv) - .write_public_key(cache.public_key) + .write_private_key(str(priv)) + .write_public_key(str(cache.public_key)) ) diff --git a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py index ce204ee07..e7d61407a 100644 --- a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py @@ -46,7 +46,7 @@ def handle_failure(self, exception, exception_tb): f.write("%s" % self.task_id) def collect_failures(self) -> Dict[str,None]: - failures = OrderedDict() + failures : Dict[str, None] = OrderedDict() failures.update(self.collect_failures_of_child_tasks()) if self.get_failure_log_path().exists(): with self.get_failure_log_path().open("r") as f: @@ -59,7 +59,7 @@ def collect_failures(self) -> Dict[str,None]: return failures def collect_failures_of_child_tasks(self) -> Dict[str,None]: - failures_of_child_tasks = OrderedDict() + failures_of_child_tasks : Dict[str, None] = OrderedDict() if self._run_dependencies_target.exists(): _run_dependencies_tasks_from_target = self._run_dependencies_target.read() else: diff --git a/exasol_integration_test_docker_environment/lib/config/build_config.py b/exasol_integration_test_docker_environment/lib/config/build_config.py index 832edcbae..d1076f324 100644 --- a/exasol_integration_test_docker_environment/lib/config/build_config.py +++ b/exasol_integration_test_docker_environment/lib/config/build_config.py @@ -1,16 +1,17 @@ import luigi +from typing import List, Optional from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY class build_config(luigi.Config): - force_pull = luigi.BoolParameter(False) - force_load = luigi.BoolParameter(False) - force_rebuild = luigi.BoolParameter(False) - force_rebuild_from = luigi.ListParameter([]) - log_build_context_content = luigi.BoolParameter(False) + force_pull : bool = luigi.BoolParameter(False) # type: ignore + force_load : bool = luigi.BoolParameter(False) # type: ignore + force_rebuild : bool = luigi.BoolParameter(False) # type: ignore + force_rebuild_from : List[str] = luigi.ListParameter([]) # type: ignore + log_build_context_content : bool = luigi.BoolParameter(False) # type: ignore # keep_build_context = luigi.BoolParameter(False) - temporary_base_directory = luigi.OptionalParameter(None) - output_directory = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) - cache_directory = luigi.OptionalParameter("") - build_name = luigi.OptionalParameter("") + temporary_base_directory : Optional[str] = luigi.OptionalParameter(None) # type: ignore + output_directory : str = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) # type: ignore + cache_directory : Optional[str] = luigi.OptionalParameter("") # type: ignore + build_name : Optional[str] = luigi.OptionalParameter("") # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/data/container_info.py b/exasol_integration_test_docker_environment/lib/data/container_info.py index ed0afcb54..8e19a82e9 100644 --- a/exasol_integration_test_docker_environment/lib/data/container_info.py +++ b/exasol_integration_test_docker_environment/lib/data/container_info.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from exasol_integration_test_docker_environment.lib.base.info import Info from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo @@ -10,7 +10,7 @@ def __init__(self, container_name: str, ip_address: str, network_aliases: List[str], network_info: DockerNetworkInfo, - volume_name: str = None): + volume_name: Optional[str] = None): self.network_aliases = network_aliases self.ip_address = ip_address self.network_info = network_info diff --git a/exasol_integration_test_docker_environment/lib/data/database_info.py b/exasol_integration_test_docker_environment/lib/data/database_info.py index 5d08f903d..941adcb19 100644 --- a/exasol_integration_test_docker_environment/lib/data/database_info.py +++ b/exasol_integration_test_docker_environment/lib/data/database_info.py @@ -1,3 +1,5 @@ +from typing import Optional + from exasol_integration_test_docker_environment.lib.base.info import Info from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports @@ -10,9 +12,9 @@ def __init__( host: str, ports: Ports, reused: bool, - container_info: ContainerInfo = None, - ssh_info: SshInfo = None, - forwarded_ports: Ports = None, + container_info: Optional[ContainerInfo] = None, + ssh_info: Optional[SshInfo] = None, + forwarded_ports: Optional[Ports] = None, ): self.container_info = container_info self.ports = ports diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py index ca790270e..7daa9a2fa 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py @@ -85,12 +85,13 @@ def _create_build_task_for_image_info( def _build_with_depenencies_is_requested(self, image_info: ImageInfo, shortcut_build: bool): needs_to_be_build = image_info.image_state == ImageState.NEEDS_TO_BE_BUILD.name result = (not shortcut_build or needs_to_be_build) and \ - len(image_info.depends_on_images) > 0 + image_info.depends_on_images and len(image_info.depends_on_images) > 0 return result def _create_build_task_with_dependencies( self, image_info: ImageInfo, shortcut_build: bool = True) -> DockerCreateImageTask: + assert image_info.depends_on_images required_tasks = \ self._create_build_tasks_for_image_infos( image_info.depends_on_images, shortcut_build) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index 28ebf193d..376222075 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -1,5 +1,6 @@ +import abc from pathlib import Path -from typing import Dict, Type +from typing import Dict, Type, Optional import docker import git @@ -38,6 +39,7 @@ def __init__(self, *args, **kwargs): ) self._dockerfile = self.get_dockerfile() + @abc.abstractmethod def get_source_repository_name(self) -> str: """ Called by the constructor to get the image name for pulls. Sub classes need to implement this method. @@ -45,6 +47,7 @@ def get_source_repository_name(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_target_repository_name(self) -> str: """ Called by the constructor to get the image name for pushs. Sub classes need to implement this method. @@ -52,6 +55,7 @@ def get_target_repository_name(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_source_image_tag(self) -> str: """ Called by the constructor to get the image tag for pulls. Sub classes need to implement this method. @@ -59,6 +63,7 @@ def get_source_image_tag(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_target_image_tag(self) -> str: """ Called by the constructor to get the image tag for pushs. Sub classes need to implement this method. @@ -76,6 +81,7 @@ def get_mapping_of_build_files_and_directories(self) -> Dict[str, str]: """ raise AbstractMethodException() + @abc.abstractmethod def get_dockerfile(self) -> str: """ Called by the constructor to get the path to the dockerfile. @@ -106,6 +112,7 @@ def get_transparent_build_arguments(self) -> Dict[str, str]: """ return dict() + @abc.abstractmethod def is_rebuild_requested(self) -> bool: raise AbstractMethodException() @@ -122,16 +129,16 @@ def register_required(self): tasks = None self.dependencies_futures = self.register_dependencies(tasks) - def keys_are_string(self, task_classes): + def keys_are_string(self, task_classes) -> bool: return all(isinstance(key, str) for key in task_classes.keys()) - def values_are_subclass_of_baseclass(self, task_classes): + def values_are_subclass_of_baseclass(self, task_classes) -> bool: return all(issubclass(value, DockerAnalyzeImageTask) for value in task_classes.values()) def requires_tasks(self) -> Dict[str, Type["DockerAnalyzeImageTask"]]: - pass + return dict() def run_task(self): image_info_of_dependencies = self.get_values_from_futures(self.dependencies_futures) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py index 55fe41af2..71c313238 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py @@ -17,11 +17,11 @@ class DockerCreateImageTask(DockerBaseTask): - image_name = luigi.Parameter() + image_name : str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info = JsonPickleParameter(ImageInfo, + image_info : ImageInfo = JsonPickleParameter(ImageInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ImageInfo + significant=True) # type: ignore def run_task(self): new_image_info = yield from self.build(self.image_info) @@ -70,9 +70,9 @@ def rename_source_image_to_target_image(self, image_info): class DockerCreateImageTaskWithDeps(DockerCreateImageTask): # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - required_task_infos = JsonPickleParameter(RequiredTaskInfoDict, + required_task_infos : RequiredTaskInfoDict = JsonPickleParameter(RequiredTaskInfoDict, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: RequiredTaskInfoDict + significant=True) # type: ignore def register_required(self): self.required_tasks = {key: self.create_required_task(required_task_info) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py index bf3214acc..071f8c0c1 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py @@ -6,8 +6,8 @@ class DockerImageCreatorBaseTask(DockerBaseTask): - image_name = luigi.Parameter() + image_name : str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info = JsonPickleParameter(ImageInfo, + image_info : ImageInfo= JsonPickleParameter(ImageInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type:ImageInfo + significant=True) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py index 0f2da542e..3353a1317 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py @@ -35,7 +35,7 @@ def map(self, image: str, queue: mp.Queue): def check(self, image: str): log_handler = DockerRegistryImageCheckerPullLogHandler() - queue = mp.Queue() + queue : mp.Queue = mp.Queue() process = mp.Process(target=self.map, args=(image, queue)) process.start() try: diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py index 43b8f7e38..57a8569b0 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py @@ -233,6 +233,7 @@ def __init__(self, hashfunc: str = 'md5', def hash(self, path_mapping: DestinationMapping): src_path = path_mapping.source_path dest_path = path_mapping.destination_path + assert self.hash_func hasher = self.hash_func() hasher.update(str(dest_path).encode('utf-8')) if self.hash_permissions: @@ -252,6 +253,7 @@ def __init__(self, hashfunc: str = 'md5', blocksize: str = "64kb"): raise NotImplementedError('{} not implemented.'.format(hashfunc)) def hash(self, filepath: Path): + assert self.hash_func hasher = self.hash_func() with open(filepath, 'rb') as fp: while True: diff --git a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py index 63685d510..28203129b 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Dict, Any +from typing import Dict, Any, Optional from exasol_integration_test_docker_environment.lib.base.info import Info @@ -48,7 +48,7 @@ def __init__(self, build_name: str = "", build_date_time: datetime = datetime.utcnow(), image_state: ImageState = ImageState.NOT_EXISTING, - depends_on_images: Dict[str, "ImageInfo"] = None): + depends_on_images: Optional[Dict[str, "ImageInfo"]] = None): self.build_name = build_name self.date_time = str(build_date_time) self.commit = commit diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py index c574b86f7..fc6d38ea7 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py @@ -14,9 +14,9 @@ class DockerPushImageTask(DockerPushImageBaseTask): # don't want to wait for the push finishing before starting to build depended images, # but we also need to create a DockerPushImageTask for each DockerCreateImageTask of a goal - required_task_info = JsonPickleParameter(RequiredTaskInfo, + required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type:RequiredTaskInfo + significant=True) # type:ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py index 52c78da05..9accd37d4 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py @@ -14,9 +14,9 @@ class DockerSaveImageTask(DockerSaveImageBaseTask): # don't want to wait for the save finishing before starting to build depended images, # but we also need to create a DockerSaveImageTask for each DockerCreateImageTask of a goal - required_task_info = JsonPickleParameter(RequiredTaskInfo, + required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: RequiredTaskInfo + significant=True) # type: ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py index b1822570b..03d4bea7c 100644 --- a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py @@ -1,10 +1,10 @@ import logging -from typing import Iterator +from typing import Iterable from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -def remove_docker_networks(networks: Iterator[str]): +def remove_docker_networks(networks: Iterable[str]): """ Removes the given networks using docker API. """ diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index e50db2b1b..4a88daa70 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Generator, Tuple, Optional +from typing import Generator, Tuple, Optional, Any from exasol_integration_test_docker_environment.lib.docker.container.utils import default_bridge_ip_address import luigi @@ -28,7 +28,7 @@ class AbstractSpawnTestEnvironment(DockerBaseTask, GeneralSpawnTestEnvironmentParameter, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() # type: str + environment_name : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -68,6 +68,7 @@ def create_test_environment_info_in_test_container( shell_variables: ShellVariables, json: str, ): + assert test_environment_info.test_container_info test_container_name = test_environment_info.test_container_info.container_name with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(test_container_name) @@ -102,36 +103,34 @@ def create_test_environment_info_in_test_container_and_on_host( json, ) - def _default_bridge_ip_address(self, test_environment_info) -> str: - if test_environment_info.database_info.container_info is None: - return None - container_name = test_environment_info.database_info.container_info.container_name - with self._get_docker_client() as docker_client: - db_container = docker_client.containers.get(container_name) - return default_bridge_ip_address(db_container) + def _default_bridge_ip_address(self, test_environment_info) -> Optional[str]: + if test_environment_info.database_info.container_info is not None: + container_name = test_environment_info.database_info.container_info.container_name + with self._get_docker_client() as docker_client: + db_container = docker_client.containers.get(container_name) + return default_bridge_ip_address(db_container) + return None def collect_shell_variables(self, test_environment_info) -> ShellVariables: - default_bridge_ip_address = self._default_bridge_ip_address(test_environment_info) return ShellVariables.from_test_environment_info( - default_bridge_ip_address, + self._default_bridge_ip_address(test_environment_info), test_environment_info, ) def _start_database(self, attempt) \ - -> Generator[BaseTask, BaseTask, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: + -> Generator[Any, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: network_info = yield from self._create_network(attempt) ssl_volume_info = None if self.create_certificates: ssl_volume_info = yield from self._create_ssl_certificates() - database_info, test_container_info = \ - yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) + database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) is_database_ready = yield from self._wait_for_database(database_info, attempt) return network_info, database_info, is_database_ready, test_container_info - def _create_ssl_certificates(self) -> DockerVolumeInfo: - ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) - ssl_info = self.get_values_from_future(ssl_info_future) - return ssl_info + def _create_ssl_certificates(self) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: + ssl_volume_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) + ssl_volume_info : Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_volume_info_future) # type: ignore + return ssl_volume_info def create_ssl_certificates(self): raise AbstractMethodException() @@ -149,7 +148,7 @@ def _spawn_database_and_test_container( network_info: DockerNetworkInfo, certificate_volume_info: Optional[DockerVolumeInfo], attempt: int, - ) -> Tuple[DatabaseInfo, Optional[ContainerInfo]]: + ) -> Generator[BaseTask, None, Tuple[DatabaseInfo, Optional[ContainerInfo]]]: def volume_name(info): return None if info is None else info.volume_name @@ -169,8 +168,8 @@ def volume_name(info): ) futures = yield from self.run_dependencies(child_tasks) results = self.get_values_from_futures(futures) - database_info = results[DATABASE] - test_container_info = results[TEST_CONTAINER] if self.test_container_content is not None else None + database_info : DatabaseInfo = results[DATABASE] # type: ignore + test_container_info : Optional[ContainerInfo] = results[TEST_CONTAINER] if self.test_container_content is not None else None # type: ignore return database_info, test_container_info def create_spawn_database_task(self, diff --git a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py index 5c570bf04..df4291662 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py @@ -42,7 +42,7 @@ def get_dockerfile(self): def is_rebuild_requested(self) -> bool: config = build_config() - return config.force_rebuild + return bool(config.force_rebuild) class DockerTestContainerBuildBase(DockerBuildBase, TestContainerParameter): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py index 032691a9c..ae5a6f6ec 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py @@ -1,8 +1,11 @@ -from typing import Dict +from typing import Dict, Optional, Generator, Set import docker import luigi + +from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask +from exasol_integration_test_docker_environment.lib.base.pickle_target import PickleTarget from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import \ @@ -14,14 +17,14 @@ class CreateSSLCertificatesTask(DockerBaseTask): - environment_name = luigi.Parameter() - docker_runtime = luigi.OptionalParameter(None, significant=False) - db_container_name = luigi.Parameter(significant=False) - network_name = luigi.Parameter() - reuse = luigi.BoolParameter(False, significant=False) - no_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - volume_name = luigi.Parameter() + environment_name : str = luigi.Parameter() # type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + db_container_name : str = luigi.Parameter(significant=False) # type: ignore + network_name : str = luigi.Parameter() # type: ignore + reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + volume_name : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -53,11 +56,11 @@ def run_task(self): self.return_object(self.volume_info) - def build_image(self) -> Dict[str, ImageInfo]: + def build_image(self) -> Generator[BaseTask, None, Set[ImageInfo]]: task = self.create_child_task(task_class=DockerCertificateContainerBuild, certificate_container_root_directory=self._temp_resource_directory.tmp_directory) image_infos_future = yield from self.run_dependencies(task) - image_infos = self.get_values_from_future(image_infos_future) + image_infos: Set[ImageInfo] = self.get_values_from_future(image_infos_future) # type: ignore return image_infos def get_volume_info(self, reused: bool) -> DockerVolumeInfo: @@ -66,6 +69,7 @@ def get_volume_info(self, reused: bool) -> DockerVolumeInfo: if volume_properties['Name'] == self.volume_name: return DockerVolumeInfo(volume_name=str(self.volume_name), mount_point=volume_properties['Mountpoint'], reused=reused) + raise RuntimeError(f"Volume Info not found for created volume {self.volume_name}") def create_docker_volume(self) -> DockerVolumeInfo: self.remove_volume(self.volume_name) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py index 9a6ebe446..84af03192 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py @@ -1,4 +1,5 @@ from pathlib import PurePath +from typing import List import docker from exasol_integration_test_docker_environment.lib.base.db_os_executor \ @@ -21,7 +22,7 @@ def find_exaplus( exit, output = os_executor.exec("find /opt -name 'exaplus' -type f") if exit != 0: raise RuntimeError(f"Exaplus not found on docker db! Output is {output}") - found_paths = list(filter(None, output.decode("UTF-8").split("\n"))) + found_paths : List[str] = list(filter(None, output.decode("UTF-8").split("\n"))) if len(found_paths) != 1: raise RuntimeError(f"Error determining exaplus path! Output is {output}") exaplus_path = PurePath(found_paths[0]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py index f0ef9a025..1a472d941 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py @@ -13,9 +13,9 @@ class PopulateTestDataToDatabase(DockerBaseTask, DatabaseCredentialsParameter): logger = logging.getLogger('luigi-interface') - environment_name = luigi.Parameter() - test_environment_info = JsonPickleParameter( - EnvironmentInfo, significant=False) # type: EnvironmentInfo + environment_name : str = luigi.Parameter() # type: ignore + test_environment_info : EnvironmentInfo = JsonPickleParameter( + EnvironmentInfo, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py index cdf4b6a2d..61c08bca5 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py @@ -2,7 +2,7 @@ import time from pathlib import Path from threading import Thread -from typing import Callable, Optional +from typing import Callable, Optional, List from docker.models.containers import Container @@ -13,15 +13,15 @@ class DBContainerLogThread(Thread): def __init__(self, container: Container, logger, log_file: Path, description: str): super().__init__() - self.complete_log = [] + self.complete_log : List[str] = list() self.description = description self.logger = logger self.log_file = log_file self.container = container self.finish = False - self.previous_timestamp = None - self.current_timestamp = None - self.error_message = None + self.previous_timestamp : Optional[float] = None + self.current_timestamp : Optional[float] = None + self.error_message : Optional[str] = None self.ignore_error_return_codes = ( "(membership) returned with state 1", # exclude webui not found in 7.0.0 "rsyslogd) returned with state 1" # exclude rsyslogd which might crash when running itde under lima diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py index e15b05b0f..b39ce02bc 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py @@ -8,10 +8,10 @@ class WaitForTestExternalDatabase(DockerBaseTask, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - database_info = JsonPickleParameter(DatabaseInfo, significant=False) # type: DatabaseInfo - db_startup_timeout_in_seconds = luigi.IntParameter(1 * 60, significant=False) - attempt = luigi.IntParameter(1) + environment_name : str = luigi.Parameter() # type: ignore + database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds : int = luigi.IntParameter(1 * 60, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore def run_task(self): # Since we can't assume that the test container exists, we cannot connect easily here diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py index 268726dd2..9f6bc900d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py @@ -19,12 +19,12 @@ class WaitForTestDockerDatabase(DockerBaseTask, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - database_info = JsonPickleParameter(DatabaseInfo, significant=False) # type: DatabaseInfo - db_startup_timeout_in_seconds = luigi.IntParameter(10 * 60, significant=False) - attempt = luigi.IntParameter(1) - docker_db_image_version = luigi.Parameter() - executor_factory = JsonPickleParameter(DbOsExecFactory, significant=False) + environment_name : str = luigi.Parameter() # type: ignore + database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds : int = luigi.IntParameter(10 * 60, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore + docker_db_image_version : str = luigi.Parameter() # type: ignore + executor_factory : DbOsExecFactory = JsonPickleParameter(DbOsExecFactory, significant=False) # type: ignore def run_task(self): with self._get_docker_client() as docker_client: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py index e8f42199d..fe4efcaf4 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Tuple from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION @@ -13,14 +13,14 @@ def __init__(self, major, minor, stable): @classmethod def from_db_version_str(cls, db_version_str: Optional[str]): - db_version = db_version_str - if db_version_str in (None, DEFAULT_VERSION): + db_version : str = LATEST_DB_VERSION if db_version_str is None else db_version_str + if db_version_str == DEFAULT_VERSION: db_version = LATEST_DB_VERSION if db_version.endswith("-d1"): db_version = "-".join(db_version.split("-")[0:-1]) if db_version.startswith("prerelease-"): db_version = "-".join(db_version.split("-")[1:]) - version = tuple([int(v) for v in db_version.split(".")]) + version : Tuple[int,...] = tuple([int(v) for v in db_version.split(".")]) if len(version) != 3: raise ValueError(f"Invalid db version given: {db_version}") return DbVersion(version[0], version[1], version[2]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py index 9b15bce22..ab28d9a37 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py @@ -1,3 +1,5 @@ +from typing import Optional + from docker.models.containers import Container import io import tarfile @@ -8,8 +10,8 @@ class DockerContainerCopy: def __init__(self, container:Container): super().__init__() self.open = True - self.file_like_object = io.BytesIO() - self.tar = tarfile.open(fileobj=self.file_like_object, mode="x") + self.file_like_object : Optional[io.BytesIO] = io.BytesIO() + self.tar : Optional[tarfile.TarFile] = tarfile.open(fileobj=self.file_like_object, mode="x") self.container = container def __del__(self): @@ -31,17 +33,21 @@ def add_string_to_file(self, name: str, string: str): tar_info = tarfile.TarInfo(name=name) tar_info.mtime = time.time() tar_info.size = len(encoded) + assert self.tar self.tar.addfile(tarinfo=tar_info, fileobj=bytes_io) def add_file(self, host_path:str, path_in_tar:str): self.is_open_or_raise() + assert self.tar self.tar.add(host_path, path_in_tar) def copy(self, path_in_container:str): self.is_open_or_raise() + assert self.tar self.tar.close() self.open = False self.tar = None + assert self.file_like_object self.container.put_archive(path_in_container, self.file_like_object.getbuffer().tobytes()) self.file_like_object = None diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py index 1e72e8c67..738c7a281 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py @@ -1,6 +1,7 @@ import luigi from enum import Enum, auto from luigi import Config +from typing import List, Optional class DbOsAccess(Enum): """ @@ -19,15 +20,15 @@ class DbOsAccess(Enum): class DockerDBTestEnvironmentParameter(Config): - docker_db_image_name = luigi.OptionalParameter(None) - docker_db_image_version = luigi.OptionalParameter(None) - reuse_database = luigi.BoolParameter(False, significant=False) + docker_db_image_name : Optional[str] = luigi.OptionalParameter(None) # type: ignore + docker_db_image_version : Optional[str] = luigi.OptionalParameter(None) # type: ignore + reuse_database : bool = luigi.BoolParameter(False, significant=False) # type: ignore db_os_access = luigi.EnumParameter(DbOsAccess.DOCKER_EXEC, enum=DbOsAccess, significant=False) - no_database_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_database_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - database_port_forward = luigi.OptionalParameter(None, significant=False) - bucketfs_port_forward = luigi.OptionalParameter(None, significant=False) - ssh_port_forward = luigi.OptionalParameter(None, significant=False) - mem_size = luigi.OptionalParameter("2 GiB", significant=False) - disk_size = luigi.OptionalParameter("2 GiB", significant=False) - nameservers = luigi.ListParameter([], significant=False) + no_database_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_database_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + database_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + bucketfs_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + ssh_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + mem_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + disk_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + nameservers : List[str] = luigi.ListParameter([], significant=False) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py index 539a62fe7..2b89f121a 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py @@ -1,3 +1,5 @@ +from typing import Optional + import luigi from luigi import Config @@ -5,34 +7,34 @@ class ExternalDatabaseXMLRPCParameter(Config): - external_exasol_xmlrpc_host = luigi.OptionalParameter() - external_exasol_xmlrpc_port = luigi.IntParameter(443) - external_exasol_xmlrpc_user = luigi.OptionalParameter() - external_exasol_xmlrpc_cluster_name = luigi.OptionalParameter() - external_exasol_xmlrpc_password = luigi.OptionalParameter( + external_exasol_xmlrpc_host : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_port : int = luigi.IntParameter(443) #type: ignore + external_exasol_xmlrpc_user : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_cluster_name : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) + ) #type: ignore # See ticket https://github.com/exasol/integration-test-docker-environment/issues/341 class ExternalDatabaseHostParameter(Config): - external_exasol_db_host = luigi.OptionalParameter() - external_exasol_db_port = luigi.IntParameter() - external_exasol_bucketfs_port = luigi.IntParameter() - external_exasol_ssh_port = luigi.IntParameter() + external_exasol_db_host : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_db_port : int = luigi.IntParameter() #type: ignore + external_exasol_bucketfs_port : int = luigi.IntParameter() #type: ignore + external_exasol_ssh_port : int = luigi.IntParameter() #type: ignore class ExternalDatabaseCredentialsParameter( ExternalDatabaseHostParameter, ExternalDatabaseXMLRPCParameter, ): - external_exasol_db_user = luigi.OptionalParameter() - external_exasol_db_password = luigi.OptionalParameter( + external_exasol_db_user : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_db_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) - external_exasol_bucketfs_write_password = luigi.OptionalParameter( + ) #type: ignore + external_exasol_bucketfs_write_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) + ) #type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py index 8473b869b..f4917a3ff 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py @@ -1,16 +1,17 @@ import luigi from luigi import Config +from typing import List, Optional from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ OptionalTestContainerParameter class GeneralSpawnTestEnvironmentParameter(OptionalTestContainerParameter): - reuse_database_setup = luigi.BoolParameter(False, significant=False) - reuse_test_container = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - max_start_attempts = luigi.IntParameter(2, significant=False) - docker_runtime = luigi.OptionalParameter(None, significant=False) - create_certificates = luigi.BoolParameter() - additional_db_parameter = luigi.ListParameter() + reuse_database_setup : bool = luigi.BoolParameter(False, significant=False) #type: ignore + reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore + max_start_attempts : int = luigi.IntParameter(2, significant=False) #type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore + create_certificates : bool = luigi.BoolParameter() #type: ignore + additional_db_parameter : List[str] = luigi.ListParameter() #type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py index 52675552d..445f908b3 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py @@ -6,11 +6,11 @@ class TestContainerParameter: - test_container_content = JsonPickleParameter(TestContainerContentDescription, - visibility=ParameterVisibility.HIDDEN) + test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, + visibility=ParameterVisibility.HIDDEN) # type: ignore class OptionalTestContainerParameter: - test_container_content = JsonPickleParameter(TestContainerContentDescription, + test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, visibility=ParameterVisibility.HIDDEN, - is_optional=True) + is_optional=True) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/ports.py b/exasol_integration_test_docker_environment/lib/test_environment/ports.py index a7506fc17..92afa5665 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/ports.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/ports.py @@ -1,6 +1,6 @@ import socket from contextlib import ExitStack -from typing import List, Optional +from typing import List, Optional, Generator def find_free_ports(num_ports: int) -> List[int]: @@ -9,7 +9,7 @@ def new_socket(): def bind(sock: socket.socket, port: int): sock.bind(('', port)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - def acquire_port_numbers(num_ports: int) -> List[int]: + def acquire_port_numbers(num_ports: int) -> Generator[int, None, None]: with ExitStack() as stack: sockets = [stack.enter_context(new_socket()) for dummy in range(num_ports)] for sock in sockets: @@ -55,5 +55,6 @@ def __init__(self, database: int, bucketfs: int, ssh: Optional[int] = None): @classmethod def random_free(cls, ssh: bool = True) -> 'Ports': - ports = find_free_ports(3 if ssh else 2) + [None] - return Ports(*ports[:3]) + count = 3 if ssh else 2 + ports = find_free_ports(count) + return Ports(ports[0], ports[1], None) if not ssh else Ports(ports[0], ports[1], ports[2]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py index 06e25f8f1..a906573d3 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py @@ -1,3 +1,4 @@ +from typing import Optional import docker import luigi @@ -6,14 +7,14 @@ class PrepareDockerNetworkForTestEnvironment(DockerBaseTask): - environment_name = luigi.Parameter() - network_name = luigi.Parameter() - test_container_name = luigi.Parameter(significant=False) - db_container_name = luigi.OptionalParameter(None, significant=False) - reuse = luigi.BoolParameter(False, significant=False) - no_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - attempt = luigi.IntParameter(-1) + environment_name : str = luigi.Parameter() # type: ignore + network_name : str = luigi.Parameter() # type: ignore + test_container_name : str = luigi.Parameter(significant=False) # type: ignore + db_container_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + attempt : int = luigi.IntParameter(-1) # type: ignore def run_task(self): self.network_info = None diff --git a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py index c9f65af8d..0140ffe02 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py @@ -21,9 +21,9 @@ class SetupExternalDatabaseHost(DependencyLoggerBaseTask, ExternalDatabaseXMLRPCParameter, ExternalDatabaseHostParameter, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - network_info = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - attempt = luigi.IntParameter(1) + environment_name : str = luigi.Parameter() # type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore def run_task(self): database_host = self.external_exasol_db_host @@ -59,6 +59,7 @@ def setup_database(self): self.logger.info(e) def get_xml_rpc_object(self, object_name: str = ""): + assert self.external_exasol_xmlrpc_user and self.external_exasol_xmlrpc_password uri = 'https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}'.format( user=quote_plus(self.external_exasol_xmlrpc_user), password=quote_plus(self.external_exasol_xmlrpc_password), @@ -77,8 +78,8 @@ def start_database(self, cluster: ServerProxy): all_nodes_online = False while not all_nodes_online: all_nodes_online = True - for nodeName in cluster.getNodeList(): - node_state = self.get_xml_rpc_object(nodeName).getNodeState() + for nodeName in cluster.getNodeList(): # type: ignore + node_state = self.get_xml_rpc_object(nodeName).getNodeState() # type: ignore if node_state['status'] != 'Running': all_nodes_online = False break @@ -95,10 +96,10 @@ def start_database(self, cluster: ServerProxy): self.logger.info('EXAStorage already online; continuing startup process') # triggering database startup - for databaseName in cluster.getDatabaseList(): - database = self.get_xml_rpc_object('/db_' + quote_plus(databaseName)) + for databaseName in cluster.getDatabaseList(): # type: ignore + database = self.get_xml_rpc_object('/db_' + quote_plus(str(databaseName))) if not database.runningDatabase(): - self.logger.info('Starting database instance %s' % databaseName) + self.logger.info('Starting database instance %s' % str(databaseName)) database.startDatabase() else: - self.logger.info('Database instance %s already running' % databaseName) + self.logger.info('Database instance %s already running' % str(databaseName)) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index 174bcdf31..a94db5563 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Optional from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo @@ -12,7 +12,7 @@ def __init__(self, env: Dict[str, str]): @classmethod def from_test_environment_info( cls, - default_bridge_ip_address: str, + default_bridge_ip_address: Optional[str], test_environment_info: EnvironmentInfo, ) -> 'ShellVariables': """ @@ -20,15 +20,20 @@ def from_test_environment_info( default_bridge_ip_address and EnvironmentInfo. """ info = test_environment_info - env = { + assert info.database_info.ports.database is not None + assert info.database_info.ports.bucketfs is not None + env : Dict[str, str] = { "NAME": info.name, "TYPE": info.type, "DATABASE_HOST": info.database_info.host, - "DATABASE_DB_PORT": info.database_info.ports.database, - "DATABASE_BUCKETFS_PORT": info.database_info.ports.bucketfs, - "DATABASE_SSH_PORT": info.database_info.ports.ssh, + "DATABASE_DB_PORT": str(info.database_info.ports.database), + "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs), + "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh is not None else "", } + if info.database_info.container_info is not None: + assert info.database_info.container_info.volume_name + assert default_bridge_ip_address network_aliases = " ".join(info.database_info.container_info.network_aliases) env.update({ "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index 3a6598e35..217c362eb 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -1,4 +1,5 @@ -from typing import List +from pathlib import Path +from typing import List, Optional, Dict, Union import luigi import netaddr @@ -22,17 +23,17 @@ class SpawnTestContainer(DockerBaseTask, TestContainerParameter): - environment_name = luigi.Parameter() - test_container_name = luigi.Parameter() - network_info = JsonPickleParameter( - DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - ip_address_index_in_subnet = luigi.IntParameter(significant=False) - attempt = luigi.IntParameter(1) - reuse_test_container = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - docker_runtime = luigi.OptionalParameter(None, significant=False) - certificate_volume_name = luigi.OptionalParameter(None, significant=False) + environment_name : str = luigi.Parameter() #type: ignore + test_container_name : str = luigi.Parameter() #type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter( + DockerNetworkInfo, significant=False) # type: ignore + ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) #type: ignore + attempt : int = luigi.IntParameter(1) #type: ignore + reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore + certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -47,18 +48,18 @@ def register_required(self): test_container_content=self.test_container_content)) def is_reuse_possible(self) -> bool: - test_container_image_info = \ - self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ImageInfo + test_container_image_info : ImageInfo = \ + self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ignore test_container = None with self._get_docker_client() as docker_client: try: test_container = docker_client.containers.get(self.test_container_name) except Exception as e: pass - ret_val = self.network_info.reused and self.reuse_test_container and \ + ret_val : bool = bool(self.network_info.reused and self.reuse_test_container and \ test_container is not None and \ test_container_image_info.get_target_complete_name() in test_container.image.tags and \ - test_container_image_info.image_state == ImageState.USED_LOCAL.name + test_container_image_info.image_state == ImageState.USED_LOCAL.name) return ret_val @@ -91,7 +92,7 @@ def _copy_runtime_targets(self): test_container.exec_run(cmd=f"cp -r {runtime_mapping.target} {runtime_mapping.deployment_target}") def _try_to_reuse_test_container(self, ip_address: str, - network_info: DockerNetworkInfo) -> ContainerInfo: + network_info: DockerNetworkInfo) -> Optional[ContainerInfo]: self.logger.info("Try to reuse test container %s", self.test_container_name) container_info = None @@ -110,7 +111,7 @@ def _create_test_container(self, ip_address, test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] - volumes = dict() + volumes : Dict[Union[str, Path], Dict[str, str]] = dict() for runtime_mapping in self.test_container_content.runtime_mappings: volumes[runtime_mapping.source.absolute()] = { "bind": runtime_mapping.target, @@ -167,7 +168,7 @@ def create_container_info(self, ip_address: str, network_aliases: List[str], return container_info def _get_export_directory(self): - return self.get_values_from_future(self.export_directory_future) + return self.get_values_from_future(self.export_directory_future) # type: ignore def _remove_container(self, container_name: str): try: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 4799134ff..a54dfcce7 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -1,5 +1,6 @@ import math -from typing import Optional, Tuple +from typing import Optional, Tuple, List, Union +from pathlib import Path import docker import humanfriendly @@ -10,6 +11,7 @@ from docker.models.containers import Container from docker.models.volumes import Volume from docker.client import DockerClient +from importlib_resources.abc import Traversable from jinja2 import Template from exasol_integration_test_docker_environment.lib import PACKAGE_NAME @@ -49,16 +51,16 @@ def int_or_none(value: str) -> Optional[int]: class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): - environment_name = luigi.Parameter() # type: str - db_container_name = luigi.Parameter() # type: str - attempt = luigi.IntParameter(1) # type: int - network_info = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - ip_address_index_in_subnet = luigi.IntParameter(significant=False) # type: int - docker_runtime = luigi.OptionalParameter(None, significant=False) # type: str - certificate_volume_name = luigi.OptionalParameter(None, significant=False) - additional_db_parameter = luigi.ListParameter() - ssh_user = luigi.Parameter("root") - ssh_key_file = luigi.OptionalParameter(None, significant=False) + environment_name : str = luigi.Parameter() # type: ignore + db_container_name : str = luigi.Parameter() # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) # type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + additional_db_parameter : List[str] = luigi.ListParameter() # type: ignore + ssh_user : str = luigi.Parameter("root") # type: ignore + ssh_key_file : Union[str, Path, None] = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -89,17 +91,16 @@ def run_task(self): database_info = self._create_database_container(db_ip_address, db_private_network) self.return_object(database_info) - def _try_to_reuse_database(self, db_ip_address: str) -> DatabaseInfo: + def _try_to_reuse_database(self, db_ip_address: str) -> Optional[DatabaseInfo]: self.logger.info("Try to reuse database container %s", self.db_container_name) - database_info = None - ssh_key = self._get_ssh_key() try: database_info = self._create_database_info(db_ip_address=db_ip_address, reused=True) + return database_info except Exception as e: self.logger.warning("Tried to reuse database container %s, but got Exeception %s. " "Fallback to create new database.", self.db_container_name, e) - return database_info + return None def _get_ssh_key(self) -> SshKey: if self.ssh_key_file: @@ -201,7 +202,7 @@ def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInf network_info=self.network_info, volume_name=self._get_db_volume_name(), ) - ssh_info = SshInfo(self.ssh_user, self.ssh_key_file) + ssh_info = SshInfo(self.ssh_user, str(self.ssh_key_file or "")) database_info = DatabaseInfo( host=db_ip_address, ports=self.internal_ports, @@ -305,7 +306,7 @@ def _prepare_volume( ) return volume, container - def _db_file(self, filename: str) -> str: + def _db_file(self, filename: str) -> Traversable: return ( importlib_resources.files(PACKAGE_NAME) / self.docker_db_config_resource_name diff --git a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py index cfe2b35f7..54333b504 100644 --- a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py +++ b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from types import ModuleType +from typing import Optional import importlib_resources as ir @@ -60,7 +61,7 @@ def __init__(self, resource_package: ModuleType): # We need to transform the module to a string and later back to a module # because this class will be pickled by luigi and modules are not supported for serialization self._resource_package_str = resource_package.__name__ - self._tmp_directory = None + self._tmp_directory : Optional[tempfile.TemporaryDirectory] = None @property def tmp_directory(self): @@ -71,14 +72,16 @@ def tmp_directory(self): def create(self) -> str: self._tmp_directory = tempfile.TemporaryDirectory() + assert self._tmp_directory source_path = ir.files(self._resource_package_str) LOG.debug(f"Copying resource package: '{self._resource_package_str}' to '{self._tmp_directory.name}'") _copy_importlib_resources_dir_tree(source_path, Path(self._tmp_directory.name)) return self._tmp_directory.name def cleanup(self): - self._tmp_directory.cleanup() - self._tmp_directory = None + if self._tmp_directory is not None: + self._tmp_directory.cleanup() + self._tmp_directory = None def __enter__(self): return self.create() diff --git a/exasol_integration_test_docker_environment/test/get_test_container_content.py b/exasol_integration_test_docker_environment/test/get_test_container_content.py index 85c0ef561..0f1cd4b72 100644 --- a/exasol_integration_test_docker_environment/test/get_test_container_content.py +++ b/exasol_integration_test_docker_environment/test/get_test_container_content.py @@ -11,7 +11,7 @@ def get_test_container_content(test_container_path: Path = FULL_TEST_CONTAINER_PATH, - runtime_mapping: Tuple[TestContainerRuntimeMapping] = tuple()) \ + runtime_mapping: Tuple[TestContainerRuntimeMapping, ...] = tuple()) \ -> TestContainerContentDescription: return TestContainerContentDescription( docker_file=str(test_container_path / "Dockerfile"), diff --git a/exasol_integration_test_docker_environment/test/test_api_logging.py b/exasol_integration_test_docker_environment/test/test_api_logging.py index 8e8ce6927..47064064a 100644 --- a/exasol_integration_test_docker_environment/test/test_api_logging.py +++ b/exasol_integration_test_docker_environment/test/test_api_logging.py @@ -239,7 +239,7 @@ def create_logger_infos(self) -> Dict[str, Dict[str, Any]]: return logger_infos def get_logger_info(self, logger: logging.Logger) -> Dict[str, Any]: - logger_info = {} + logger_info : Dict[str, Any] = dict() logger_info[LOGGER_STR] = str(logger) logger_info[LEVEL] = logger.level logger_info[LEVEL_NAME] = logging.getLevelName(logger.level) diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_test_environment.py index c07f73dc2..7043ec82b 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment.py @@ -106,6 +106,7 @@ def tearDownClass(cls): def _assert_deployment_available(self, environment_info: EnvironmentInfo) -> None: with ContextDockerClient() as docker_client: + assert environment_info.test_container_info test_container = docker_client.containers.get(environment_info.test_container_info.container_name) exit_code, output = test_container.exec_run("cat /test/test.txt") self.assertEqual(exit_code, 0) @@ -113,6 +114,7 @@ def _assert_deployment_available(self, environment_info: EnvironmentInfo) -> Non def _assert_deployment_not_shared(self, environment_info: EnvironmentInfo, temp_path: Path): with ContextDockerClient() as docker_client: + assert environment_info.test_container_info test_container = docker_client.containers.get(environment_info.test_container_info.container_name) exit_code, output = test_container.exec_run("touch /test_target/test_new.txt") self.assertEqual(exit_code, 0) diff --git a/exasol_integration_test_docker_environment/test/test_common_run_task.py b/exasol_integration_test_docker_environment/test/test_common_run_task.py index 4f599aaaf..04422dd23 100644 --- a/exasol_integration_test_docker_environment/test/test_common_run_task.py +++ b/exasol_integration_test_docker_environment/test/test_common_run_task.py @@ -1,6 +1,4 @@ -import subprocess import unittest -from pathlib import Path import luigi diff --git a/exasol_integration_test_docker_environment/test/test_docker_access_method.py b/exasol_integration_test_docker_environment/test/test_docker_access_method.py index b6adb7a89..259c8e7c9 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_access_method.py +++ b/exasol_integration_test_docker_environment/test/test_docker_access_method.py @@ -1,3 +1,5 @@ +from typing import Dict, Any + import luigi import pytest @@ -14,10 +16,10 @@ def run_task(self): @classmethod def make(cls, method: str) -> 'Testee': - kwargs={"task_class": Testee} + kwargs : Dict[str, Any] = {"task_class": Testee} if method: kwargs["db_os_access"] = method - task = generate_root_task(**kwargs) + task : Testee = generate_root_task(**kwargs) # type: ignore luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") return task diff --git a/exasol_integration_test_docker_environment/test/test_docker_build_base.py b/exasol_integration_test_docker_environment/test/test_docker_build_base.py index 9fe45918b..6935eda24 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_build_base.py +++ b/exasol_integration_test_docker_environment/test/test_docker_build_base.py @@ -5,7 +5,7 @@ import luigi from luigi import Parameter -from typing import Set, Dict +from typing import Set, Dict, List from exasol_integration_test_docker_environment.lib.api.common import generate_root_task from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient @@ -44,7 +44,7 @@ def is_rebuild_requested(self) -> bool: class TestDockerBuildBase(DockerBuildBase): - goals = luigi.ListParameter([]) + goals : List[str] = luigi.ListParameter([]) # type: ignore def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: goal_class_map = { diff --git a/exasol_integration_test_docker_environment/test/test_doctor.py b/exasol_integration_test_docker_environment/test/test_doctor.py index add497c86..7ed1767a4 100644 --- a/exasol_integration_test_docker_environment/test/test_doctor.py +++ b/exasol_integration_test_docker_environment/test/test_doctor.py @@ -1,7 +1,8 @@ import os import unittest from contextlib import contextmanager -from typing import ContextManager, Mapping + +from typing import Generator from unittest.mock import patch from exasol_integration_test_docker_environment.doctor import ( @@ -14,7 +15,7 @@ @contextmanager -def temporary_env(env_vars) -> ContextManager[Mapping[str, str]]: +def temporary_env(env_vars) -> Generator[os._Environ, None, None]: """ Creates a temporary environment, containing the current environment variables. diff --git a/exasol_integration_test_docker_environment/test/test_populate_data.py b/exasol_integration_test_docker_environment/test/test_populate_data.py index e66e4b341..9aeb79c7e 100644 --- a/exasol_integration_test_docker_environment/test/test_populate_data.py +++ b/exasol_integration_test_docker_environment/test/test_populate_data.py @@ -66,11 +66,13 @@ def setUp(self) -> None: def _execute_sql_on_db(self, sql: str) -> str: with ContextDockerClient() as docker_client: print(f"Executing sql on db: '{sql}'") - test_container = docker_client.containers.get(self.environment.environment_info. + #environment is a class variable, need to suppress type check + environment = self.environment # type: ignore + test_container = docker_client.containers.get(environment.environment_info. test_container_info.container_name) - db_info = self.environment.environment_info.database_info - db_user_name = self.environment.db_username - db_password = self.environment.db_password + db_info = environment.environment_info.database_info + db_user_name = environment.db_username + db_password = environment.db_password cmd = f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.ports.database}' " \ f"-u '{db_user_name}' -p '{db_password}' -sql '{sql}' " \ f"-jdbcparam 'validateservercertificate=0'" diff --git a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py index c1a65c92d..d2f850824 100644 --- a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py +++ b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py @@ -50,7 +50,7 @@ def run_task(self): test_container_content=self.test_container_content ) test_container_future_1 = yield from self.run_dependencies(test_container_task_1) - container_info = test_container_future_1.get_output() # type: ContainerInfo + container_info : ContainerInfo = test_container_future_1.get_output() # type: ignore with ContextDockerClient() as docker_client: container = docker_client.containers.get(container_info.container_name) self.return_object({"container_id": container.id, "image_id": container.image.id}) diff --git a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py index 52a98fa3a..b6c809fa0 100644 --- a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py +++ b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, List, Tuple +from typing import Any, List, Tuple, Optional import click @@ -29,14 +29,14 @@ def is_click_command(obj: Any) -> bool: return isinstance(obj, click.Command) -def defaults_of_click_call(click_call: click.Command) -> List[Tuple[str, Any]]: +def defaults_of_click_call(click_call: click.Command) -> List[Tuple[Optional[str] , Any]]: """ Returns the default values of all None-required parameters of a click-command. """ return [(o.name, adjust_default_value_for_multiple(o)) for o in click_call.params if not o.required] -def param_names_of_click_call(click_call: click.Command) -> List[str]: +def param_names_of_click_call(click_call: click.Command) -> List[Optional[str]]: """ Returns names of all parameters of a click call """ @@ -48,7 +48,7 @@ def get_click_and_api_functions(click_module, api_module) -> Tuple[List[Any], Li click_commands = [c[1] for c in inspect.getmembers(click_module, is_click_command)] # Get all functions in module 'api_module' api_functions = [f[1] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] + if f[1].__cli_function__] # type: ignore return click_commands, api_functions @@ -57,5 +57,5 @@ def get_click_and_api_function_names(click_module, api_module) -> Tuple[List[Any click_command_names = [c[0] for c in inspect.getmembers(click_module, is_click_command)] # Get all function names in module 'api_module' api_function_names = [f[0] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] + if f[1].__cli_function__] # type: ignore return click_command_names, api_function_names diff --git a/exasol_integration_test_docker_environment/testing/api_test_environment.py b/exasol_integration_test_docker_environment/testing/api_test_environment.py index ac2eb95b4..fd4acd4df 100644 --- a/exasol_integration_test_docker_environment/testing/api_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/api_test_environment.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path from sys import stderr -from typing import Dict, Any +from typing import Dict, Any, Optional from exasol_integration_test_docker_environment.lib.api import spawn_test_environment from exasol_integration_test_docker_environment.lib.api import spawn_test_environment_with_test_container @@ -62,7 +62,7 @@ def spawn_docker_test_environment_with_test_container( self, name: str, test_container_content: TestContainerContentDescription, - additional_parameter: Dict[str, Any] = None, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() @@ -84,7 +84,7 @@ def spawn_docker_test_environment_with_test_container( def spawn_docker_test_environment( self, name: str, - additional_parameter: Dict[str, Any] = None, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index 732c892f9..3cbcab4a1 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -7,7 +7,7 @@ from pathlib import Path import shlex from sys import stderr -from typing import List +from typing import List, Optional from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient @@ -130,7 +130,7 @@ def close(self): except Exception as e: print(e, file=stderr) - def spawn_docker_test_environments(self, name: str, additional_parameter: List[str] = None) \ + def spawn_docker_test_environments(self, name: str, additional_parameter: Optional[List[str]] = None) \ -> SpawnedTestEnvironments: ports = Ports.random_free() on_host_parameter = ExaslctDockerTestEnvironment( @@ -143,7 +143,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s ports=ports, ) - arguments = [ + arguments : List[str] = [ f"--environment-name {on_host_parameter.name}", f"--database-port-forward {on_host_parameter.ports.database}", f"--bucketfs-port-forward {on_host_parameter.ports.bucketfs}", @@ -154,9 +154,9 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s arguments.append(f'--docker-db-image-version "{db_version}"') if additional_parameter: arguments += additional_parameter - arguments = " ".join(arguments) + arguments_str = " ".join(arguments) - command = f"{self.executable} spawn-test-environment {arguments}" + command = f"{self.executable} spawn-test-environment {arguments_str}" completed_process = self.run_command(command, use_flavor_path=False, use_docker_repository=False, capture_output=True) on_host_parameter.completed_process = completed_process @@ -168,7 +168,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s with environment_info_json_path.open() as f: environment_info = EnvironmentInfo.from_json(f.read()) on_host_parameter.environment_info = environment_info - on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) + on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) #type: ignore if "RUN_SLC_TESTS_WITHIN_CONTAINER" in os.environ: slc_test_run_parameter = ExaslctDockerTestEnvironment( name=on_host_parameter.name, @@ -178,7 +178,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s bucketfs_username=on_host_parameter.bucketfs_username, bucketfs_password=on_host_parameter.bucketfs_password, ports=Ports.default_ports, - environment_info=on_host_parameter.completed_process, + environment_info=on_host_parameter.environment_info, completed_process=on_host_parameter.completed_process ) diff --git a/noxfile.py b/noxfile.py index dcdebe0dd..29c59e176 100644 --- a/noxfile.py +++ b/noxfile.py @@ -66,9 +66,9 @@ def run_minimal_tests(session: nox.Session, db_version: str): "test_termination_handler.py", ], "new-itest": ["test_cli_environment.py", "test_db_container_log_thread.py"], - "unit": "./test/unit", + "unit": ["./test/unit"], } - session.run("pytest", minimal_tests["unit"]) + session.run("pytest", *minimal_tests["unit"]) for test in minimal_tests["new-itest"]: session.run( "pytest", @@ -104,7 +104,7 @@ def starter_scripts_checksums(session: nox.Session): with session.chdir(start_script_dir): for start_script_entry in start_script_dir.iterdir(): if start_script_entry.is_file(): - sha512 = session.run("sha512sum", start_script_entry.name, silent=True) + sha512 : str = session.run("sha512sum", start_script_entry.name, silent=True) # type: ignore with open( start_script_dir /"checksums" / f"{start_script_entry.name}.sha512sum", "w") as f: f.write(sha512) session.run("git", "add", "starter_scripts/checksums") diff --git a/pyproject.toml b/pyproject.toml index f2e3f34bf..c2b170b6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,5 +57,11 @@ joblib = "^1.4.2" itde = 'exasol_integration_test_docker_environment.main:main' +[[tool.mypy.overrides]] +module = [ "luigi.*", "docker.*", "humanfriendly", "configobj", "toml", "netaddr", "joblib.testing", "networkx", + "fabric", "requests", "pyexasol", "paramiko.ssh_exception", "six", "jsonpickle", "exasol.toolbox.*", + "paramiko", "exasol"] +ignore_missing_imports = true + [tool.pylint.master] fail-under = 6.50 diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 32fa8fdaf..3994bb89b 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,10 +1,10 @@ import contextlib import io + import pytest -from typing import Any, Callable, Dict, Iterator, List, NewType, Optional +from typing import Any, Callable, Dict, Iterator, List, NewType, Optional, Generator, ContextManager -from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from test.integration.helpers import normalize_request_name from exasol_integration_test_docker_environment.testing import utils from exasol_integration_test_docker_environment \ @@ -40,7 +40,7 @@ def api_isolation(request) -> Iterator[ApiTestEnvironment]: utils.close_environments(environment) -CliContextProvider = NewType( +CliContextProvider = NewType( # type: ignore "CliContextProvider", Callable[ [Optional[str], Optional[List[str]]], SpawnedTestEnvironments @@ -49,7 +49,7 @@ def api_isolation(request) -> Iterator[ApiTestEnvironment]: @pytest.fixture -def cli_database(cli_isolation) -> CliContextProvider: +def cli_database(cli_isolation) -> Callable[[Optional[str], Optional[list[str]]], ContextManager[SpawnedTestEnvironments]]: """ Returns a method that test case implementations can use to create a context with a database. @@ -66,7 +66,7 @@ def test_case(database): def create_context( name: Optional[str] = None, additional_parameters: Optional[List[str]] = None, - ) -> SpawnedTestEnvironments: + ) -> Iterator[SpawnedTestEnvironments]: name = name if name else cli_isolation.name spawned = cli_isolation.spawn_docker_test_environments( name=name, @@ -78,7 +78,7 @@ def create_context( return create_context -ApiContextProvider = NewType( +ApiContextProvider = NewType( # type: ignore "ApiContextProvider", Callable[ [Optional[str], Optional[Dict[str, Any]]], @@ -93,7 +93,7 @@ def api_database(api_isolation: ApiTestEnvironment) -> ApiContextProvider: def create_context( name: Optional[str] = None, additional_parameters: Optional[Dict[str, Any]] = None, - ) -> ExaslctDockerTestEnvironment: + ) -> Generator[ExaslctDockerTestEnvironment, None, None]: name = name if name else api_isolation.name spawned = api_isolation.spawn_docker_test_environment( name=name, @@ -102,7 +102,7 @@ def create_context( yield spawned utils.close_environments(spawned) - return create_context + return create_context # type: ignore @pytest.fixture def fabric_stdin(monkeypatch): diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 506f4eec3..2baae5c4c 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -44,6 +44,7 @@ def get_executor_factory( if db_os_access == DbOsAccess.SSH: return SshExecFactory.from_database_info(dbinfo) client_factory = DockerClientFactory(timeout=100000) + assert dbinfo.container_info return DockerExecFactory(dbinfo.container_info.container_name, client_factory) diff --git a/test/integration/test_cli_environment.py b/test/integration/test_cli_environment.py index 1eb16caa9..29f28a067 100644 --- a/test/integration/test_cli_environment.py +++ b/test/integration/test_cli_environment.py @@ -27,6 +27,7 @@ def count(self, selected: Optional[List[str]] = None): @property def log(self) -> str: + assert self.db.on_host_docker_environment.completed_process return ( self .db @@ -50,8 +51,9 @@ def smoke_test_sql(exaplus_path: str, env: ExaslctDockerTestEnvironment) -> str: def quote(s): return f"'{s}'" + assert env.environment_info db_info = env.environment_info.database_info - command = [ + command : List[str] = [ str(exaplus_path), "-c", quote(f"{db_info.host}:{db_info.ports.database}"), "-u", quote(env.db_username), @@ -63,8 +65,8 @@ def quote(s): "-jdbcparam", "validateservercertificate=0", ] - command = " ".join(command) - return f'bash -c "{command}" ' + command_str = " ".join(command) + return f'bash -c "{command_str}" ' def test_db_container_started(cli_database): diff --git a/test/integration/test_db_container_log_thread.py b/test/integration/test_db_container_log_thread.py index 32093f75c..f398b9fb7 100644 --- a/test/integration/test_db_container_log_thread.py +++ b/test/integration/test_db_container_log_thread.py @@ -2,7 +2,7 @@ import tempfile import time from pathlib import Path -from typing import List +from typing import List, Optional import pytest @@ -24,7 +24,7 @@ def _build_docker_command(logs: List[str]): bash_command = f"while true; do {echo_commdands_str}; sleep 0.1; done" return ["bash", "-c", bash_command] -def _run_container_log_thread(logger, logs: List[str]) -> str: +def _run_container_log_thread(logger, logs: List[str]) -> Optional[str]: """ Starts a dummy docker container which prints logs in an endless loop, and calls DBContainerLogThread on that container. @@ -60,7 +60,7 @@ def test_container_log_thread_error(test_logger) -> None: Integration test which verifies that the DBContainerLogThread returns error message if error is logged. """ error_message = _run_container_log_thread(test_logger, ["confd returned with state 1"]) - assert "confd returned with state 1\n" in error_message + assert error_message and "confd returned with state 1\n" in error_message def test_container_log_thread_ignore_rsyslogd(test_logger) -> None: """ @@ -96,4 +96,4 @@ def test_container_log_thread_exception(test_logger) -> None: 'Exception: bad thing happend' ] error_message = _run_container_log_thread(test_logger, sshd_logs) - assert "exception: bad thing happend\n" in error_message + assert error_message and "exception: bad thing happend\n" in error_message diff --git a/test/unit/test_env_variable.py b/test/unit/test_env_variable.py index da33a3799..d536123dc 100644 --- a/test/unit/test_env_variable.py +++ b/test/unit/test_env_variable.py @@ -41,7 +41,7 @@ class LogPathCorrectnessMatcher: def __init__(self, expected_log_path: Path): self.expected_log_path = expected_log_path - def __eq__(self, used_log_path: UsedLogPath): + def __eq__(self, used_log_path: object): if not isinstance(used_log_path, UsedLogPath): return False log_path = used_log_path.log_path