From 505c9f2add32f056c1e33dd676d83b7fc4665fec Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:48:21 -0300 Subject: [PATCH 01/60] #425: Fixed type errors found by MyPy --- .../cli/termination_handler.py | 2 +- .../doctor.py | 2 +- .../lib/api/api_errors.py | 11 ++-- .../lib/api/common.py | 55 +++++++++---------- .../lib/api/spawn_test_environment.py | 14 +++-- ...wn_test_environment_with_test_container.py | 14 ++--- .../lib/base/base_task.py | 8 +-- .../lib/base/db_os_executor.py | 9 ++- .../lib/base/flavor_task.py | 2 +- .../lib/base/json_pickle_target.py | 4 +- .../lib/base/luigi_log_config.py | 12 ++-- .../lib/base/ssh_access.py | 6 +- .../lib/base/stoppable_base_task.py | 4 +- .../lib/config/build_config.py | 18 +++--- .../lib/data/container_info.py | 4 +- .../lib/data/database_info.py | 8 ++- .../docker/images/create/docker_build_base.py | 3 +- .../create/docker_image_analyze_task.py | 12 +++- .../images/create/docker_image_create_task.py | 10 ++-- .../create/docker_image_creator_base_task.py | 6 +- .../utils/docker_registry_image_checker.py | 2 +- .../utils/file_directory_list_hasher.py | 2 + .../lib/docker/images/image_info.py | 4 +- .../images/push/docker_image_push_task.py | 4 +- .../images/save/docker_image_save_task.py | 4 +- .../lib/docker/networks/utils.py | 4 +- .../abstract_spawn_test_environment.py | 34 ++++++------ .../analyze_test_container.py | 2 +- .../create_ssl_certificates_task.py | 24 ++++---- .../find_exaplus_in_db_container.py | 3 +- .../database_setup/populate_data.py | 6 +- .../db_container_log_thread.py | 10 ++-- .../wait_for_external_database.py | 8 +-- .../wait_for_test_docker_database.py | 12 ++-- .../lib/test_environment/db_version.py | 8 +-- .../test_environment/docker_container_copy.py | 10 +++- .../external_test_environment_parameter.py | 32 ++++++----- .../lib/test_environment/ports.py | 11 ++-- .../prepare_network_for_test_environment.py | 16 +++--- .../setup_external_database_host.py | 22 ++++---- .../lib/test_environment/shell_variables.py | 10 ++-- .../test_environment/spawn_test_container.py | 46 ++++++++-------- .../test_environment/spawn_test_database.py | 34 ++++++------ .../lib/utils/resource_directory.py | 9 ++- .../test/get_test_container_content.py | 2 +- .../test/test_api_logging.py | 12 ++-- .../test/test_api_test_environment.py | 2 + .../test/test_docker_access_method.py | 6 +- .../test/test_doctor.py | 5 +- .../test/test_populate_data.py | 9 +-- .../test/test_test_container_reuse.py | 2 +- .../testing/api_consistency_utils.py | 10 ++-- .../testing/api_test_environment.py | 6 +- .../testing/exaslct_test_environment.py | 10 ++-- noxfile.py | 4 +- pyproject.toml | 6 ++ test/integration/conftest.py | 22 +++++--- test/integration/helpers.py | 1 + test/integration/test_cli_environment.py | 4 +- .../test_db_container_log_thread.py | 8 +-- test/unit/test_env_variable.py | 2 +- 61 files changed, 344 insertions(+), 288 deletions(-) diff --git a/exasol_integration_test_docker_environment/cli/termination_handler.py b/exasol_integration_test_docker_environment/cli/termination_handler.py index a2330492e..89cef1e3d 100644 --- a/exasol_integration_test_docker_environment/cli/termination_handler.py +++ b/exasol_integration_test_docker_environment/cli/termination_handler.py @@ -48,7 +48,7 @@ def _handle_failure(self, task_error: TaskRuntimeError): def _print_task_failures(task_error: TaskRuntimeError): print_err() print_err("Task failure message: %s" % task_error.msg) - print_err(task_error.__cause__.args[0]) + print_err(task_error.__cause__.args[0]) # type: ignore print_err() def _handle_success(self): diff --git a/exasol_integration_test_docker_environment/doctor.py b/exasol_integration_test_docker_environment/doctor.py index 57e489610..5640a8006 100644 --- a/exasol_integration_test_docker_environment/doctor.py +++ b/exasol_integration_test_docker_environment/doctor.py @@ -84,5 +84,5 @@ def health_checkup() -> Iterable[error.ExaError]: ] for is_fine, diagnosis in examinations: if not is_fine(): - for error_code in diagnosis(): + for error_code in diagnosis(): # type: ignore yield error_code diff --git a/exasol_integration_test_docker_environment/lib/api/api_errors.py b/exasol_integration_test_docker_environment/lib/api/api_errors.py index 40c51e152..d1d30475d 100644 --- a/exasol_integration_test_docker_environment/lib/api/api_errors.py +++ b/exasol_integration_test_docker_environment/lib/api/api_errors.py @@ -12,13 +12,16 @@ class HealthProblem(RuntimeError): class TaskFailures(Exception): """Represents a potential cause of a TaskRuntimeError""" - def __init__(self, inner: List[str] = None): + def __init__(self, inner: Optional[List[str]] = None): super().__init__(self._construct_exception_message(inner)) self.inner = inner - def _construct_exception_message(self, failures: Iterable[str]) -> str: - formatted_task_failures = "\n".join(failures) - return f"Following task failures were caught during the execution:\n{formatted_task_failures}" + def _construct_exception_message(self, failures: Optional[Iterable[str]]) -> str: + if failures is not None: + formatted_task_failures = "\n".join(failures) + return f"Following task failures were caught during the execution:\n{formatted_task_failures}" + else: + return f"No task failures were caught during the execution:" class TaskRuntimeError(RuntimeError): diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index 9603ac2a6..97e170fc7 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -28,7 +28,7 @@ class JobCounterSingleton(object): """ - We use here a Singleton to avoid a unprotected global variable. + We use here a Singleton to avoid an unprotected global variable. However, this counter needs to be global counter to guarantee unique job ids. This is needed in case of task that finish in less than a second, to avoid duplicated job ids """ @@ -42,8 +42,8 @@ def __new__(cls): return cls._instance def get_next_value(self) -> int: - self._counter += 1 - return self._counter + self._counter += 1 # type: ignore + return self._counter # type: ignore def set_build_config(force_rebuild: bool, @@ -51,9 +51,9 @@ def set_build_config(force_rebuild: bool, force_pull: bool, log_build_context_content: bool, output_directory: str, - temporary_base_directory: str, - cache_directory: str, - build_name: str, ): + temporary_base_directory: Optional[str], + cache_directory: Optional[str], + build_name: Optional[str], ): luigi.configuration.get_config().set('build_config', 'force_rebuild', str(force_rebuild)) luigi.configuration.get_config().set('build_config', 'force_rebuild_from', json.dumps(force_rebuild_from)) luigi.configuration.get_config().set('build_config', 'force_pull', str(force_pull)) @@ -72,9 +72,8 @@ def set_output_directory(output_directory): luigi.configuration.get_config().set('build_config', 'output_directory', output_directory) -def set_docker_repository_config(docker_password: str, docker_repository_name: str, docker_username: str, - tag_prefix: str, - kind: str): +def set_docker_repository_config(docker_password: Optional[str], docker_repository_name: Optional[str], + docker_username: Optional[str], tag_prefix: str, kind: str): config_class = f'{kind}_docker_repository_config' luigi.configuration.get_config().set(config_class, 'tag_prefix', tag_prefix) if docker_repository_name is not None: @@ -105,8 +104,8 @@ def import_build_steps(flavor_path: Tuple[str, ...]): path_to_build_steps = Path(path).joinpath("flavor_base/build_steps.py") module_name_for_build_steps = extract_modulename_for_build_steps(path) spec = importlib.util.spec_from_file_location(module_name_for_build_steps, path_to_build_steps) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + module = importlib.util.module_from_spec(spec) # type: ignore + spec.loader.exec_module(module) # type: ignore def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: @@ -117,9 +116,21 @@ def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: return task_class(**params) +def _run_task_with_logging_config( + task: DependencyLoggerBaseTask, + log_file_path: Path, + log_level: Optional[str], + use_job_specific_log_file: bool, + workers: int) -> bool: + with _configure_logging(log_file_path, log_level, use_job_specific_log_file) as run_kwargs: + no_scheduling_errors = luigi.build([task], workers=workers, + local_scheduler=True, **run_kwargs) + return no_scheduling_errors + + def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int = 2, task_dependencies_dot_file: Optional[str] = None, - log_level: str = None, use_job_specific_log_file: bool = False) \ + log_level: Optional[str] = None, use_job_specific_log_file: bool = False) \ -> Any: setup_worker() task = task_creator() @@ -142,18 +153,6 @@ def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int task.cleanup(success) -def _run_task_with_logging_config( - task: DependencyLoggerBaseTask, - log_file_path: Path, - log_level: Optional[str], - use_job_specific_log_file: bool, - workers: int) -> bool: - with _configure_logging(log_file_path, log_level, use_job_specific_log_file) as run_kwargs: - no_scheduling_errors = luigi.build([task], workers=workers, - local_scheduler=True, **run_kwargs) - return no_scheduling_errors - - def _handle_task_result( no_scheduling_errors: bool, success: bool, @@ -181,7 +180,7 @@ def _configure_logging( use_job_specific_log_file: bool) -> Iterator[Dict[str, str]]: with get_luigi_log_config(log_file_target=log_file_path, log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) as luigi_config: + use_job_specific_log_file=use_job_specific_log_file) as luigi_config: # type: Path no_configure_logging, run_kwargs = _configure_logging_parameter( log_level=log_level, luigi_config=luigi_config, @@ -199,7 +198,7 @@ def _configure_logging( yield run_kwargs -def _configure_logging_parameter(log_level: str, luigi_config: Path, use_job_specific_log_file: bool) \ +def _configure_logging_parameter(log_level: Optional[str], luigi_config: Path, use_job_specific_log_file: bool) \ -> Tuple[bool, Dict[str, str]]: if use_job_specific_log_file: no_configure_logging = False @@ -226,13 +225,13 @@ def generate_graph_from_task_dependencies(task: DependencyLoggerBaseTask, task_d def collect_dependencies(task: DependencyLoggerBaseTask) -> Set[TaskDependency]: - dependencies = set() + dependencies : Set[TaskDependency] = set() for root, directories, files in os.walk(task._get_dependencies_path_for_job()): for file in files: file_path = Path(root).joinpath(file) with open(file_path) as f: for line in f.readlines(): - task_dependency = TaskDependency.from_json(line) # type: TaskDependency + task_dependency : TaskDependency = TaskDependency.from_json(line) # type: ignore if task_dependency.state == DependencyState.requested.name: dependencies.add(task_dependency) return dependencies diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py index d141473a4..28e57a41f 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py @@ -1,5 +1,5 @@ import functools -from typing import Tuple, Optional, Callable +from typing import Tuple, Optional, Callable, Any import humanfriendly from exasol_integration_test_docker_environment.lib.api.common import ( @@ -26,8 +26,10 @@ def _cleanup(environment_info: EnvironmentInfo) -> None: - remove_docker_container([environment_info.database_info.container_info.container_name]) - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) + if environment_info.database_info.container_info is not None: + remove_docker_container([environment_info.database_info.container_info.container_name]) + if environment_info.database_info.container_info.volume_name is not None: + remove_docker_volumes([environment_info.database_info.container_info.volume_name]) remove_docker_networks([environment_info.network_info.network_name]) @@ -69,7 +71,7 @@ def spawn_test_environment( raises: TaskRuntimeError if spawning the test environment fails """ - def str_or_none(x: any) -> str: + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) @@ -78,6 +80,8 @@ def str_or_none(x: any) -> str: parsed_db_disk_size = humanfriendly.parse_size(db_disk_size) if parsed_db_disk_size < humanfriendly.parse_size("100 MiB"): raise ArgumentConstraintError("db_disk_size", "needs to be at least 100 MiB") + db_os_access_value = DbOsAccess[db_os_access] if db_os_access else DbOsAccess.DOCKER_EXEC + set_build_config(False, tuple(), False, @@ -101,7 +105,7 @@ def str_or_none(x: any) -> str: docker_runtime=docker_runtime, docker_db_image_version=docker_db_image_version, docker_db_image_name=docker_db_image_name, - db_os_access=DbOsAccess[db_os_access], + db_os_access=db_os_access_value, db_user="sys", db_password="exasol", bucketfs_write_password="write", diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py index 41da7bd81..9a9057b98 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py @@ -1,5 +1,5 @@ import functools -from typing import Tuple, Optional, Callable +from typing import Tuple, Optional, Callable, Any import humanfriendly from exasol_integration_test_docker_environment.lib.api.common import set_build_config, set_docker_repository_config, \ @@ -22,12 +22,12 @@ .docker_db_test_environment_parameter import DbOsAccess def _cleanup(environment_info: EnvironmentInfo) -> None: - if environment_info.test_container_info is None: + if environment_info.test_container_info is not None: + remove_docker_container([environment_info.test_container_info.container_name]) + if environment_info.database_info.container_info is not None: remove_docker_container([environment_info.database_info.container_info.container_name]) - else: - remove_docker_container([environment_info.test_container_info.container_name, - environment_info.database_info.container_info.container_name]) - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) + if environment_info.database_info.container_info.volume_name is not None: + remove_docker_volumes([environment_info.database_info.container_info.volume_name]) remove_docker_networks([environment_info.network_info.network_name]) @@ -71,7 +71,7 @@ def spawn_test_environment_with_test_container( raises: TaskRuntimeError if spawning the test environment fails """ - def str_or_none(x: any) -> str: + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"): diff --git a/exasol_integration_test_docker_environment/lib/base/base_task.py b/exasol_integration_test_docker_environment/lib/base/base_task.py index f559b8dfd..028ef0f5f 100644 --- a/exasol_integration_test_docker_environment/lib/base/base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/base_task.py @@ -85,8 +85,8 @@ def get_output(self) -> Any: class BaseTask(Task): - caller_output_path = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) - job_id = luigi.Parameter() + caller_output_path : List[str] = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) # type: ignore + job_id : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): self._registered_tasks = [] @@ -153,7 +153,7 @@ def get_output_path(self) -> Path: def _get_output_path_for_job(self) -> Path: return Path(build_config().output_directory, - "jobs", self.job_id) + "jobs", self.job_id) # type: ignore def _extend_output_path(self): extension = self.extend_output_path() @@ -184,7 +184,7 @@ def get_log_path(self) -> Path: return path def get_cache_path(self) -> Path: - path = Path(build_config().output_directory, "cache") + path = Path(str(build_config().output_directory), "cache") path.mkdir(parents=True, exist_ok=True) return path diff --git a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py index 46c1c67b6..d6f444d13 100644 --- a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py +++ b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py @@ -3,7 +3,7 @@ import docker import time from docker import DockerClient -from typing import Protocol, runtime_checkable +from typing import Protocol, runtime_checkable, Optional from docker.models.containers import Container, ExecResult from exasol_integration_test_docker_environment \ .lib.base.ssh_access import SshKey @@ -49,7 +49,7 @@ class DockerExecutor(DbOsExecutor): def __init__(self, docker_client: DockerClient, container_name: str): self._client = docker_client self._container_name = container_name - self._container = None + self._container : Optional[Container] = None def __enter__(self): self._container = self._client.containers.get(self._container_name) @@ -62,6 +62,7 @@ def __del__(self): self.close() def exec(self, cmd: str) -> ExecResult: + assert self._container return self._container.exec_run(cmd) def close(self): @@ -75,7 +76,7 @@ class SshExecutor(DbOsExecutor): def __init__(self, connect_string: str, key_file: str): self._connect_string = connect_string self._key_file = key_file - self._connection = None + self._connection : Optional[fabric.Connection] = None def __enter__(self): key = SshKey.read_from(self._key_file) @@ -92,6 +93,7 @@ def __del__(self): self.close() def exec(self, cmd: str) -> ExecResult: + assert self._connection result = self._connection.run(cmd, warn=True, hide=True) output = result.stdout.encode("utf-8") return ExecResult(result.exited, output) @@ -146,6 +148,7 @@ def executor(self) -> DbOsExecutor: class SshExecFactory(DbOsExecFactory): @classmethod def from_database_info(cls, info: DatabaseInfo): + assert info.ssh_info return SshExecFactory( f"{info.ssh_info.user}@{info.host}:{info.ports.ssh}", info.ssh_info.key_file, diff --git a/exasol_integration_test_docker_environment/lib/base/flavor_task.py b/exasol_integration_test_docker_environment/lib/base/flavor_task.py index 9c7ac71d3..67f15f4db 100644 --- a/exasol_integration_test_docker_environment/lib/base/flavor_task.py +++ b/exasol_integration_test_docker_environment/lib/base/flavor_task.py @@ -8,7 +8,7 @@ class FlavorsBaseTask(DependencyLoggerBaseTask): - flavor_paths = luigi.ListParameter() + flavor_paths : List[str] = luigi.ListParameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py index b259e6f83..067d7e662 100644 --- a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py +++ b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any +from typing import Any, Optional import jsonpickle from luigi import LocalTarget @@ -10,7 +10,7 @@ class JsonPickleTarget(LocalTarget): def __init__(self, path: Path, is_tmp: bool = False): super().__init__(path=str(path), is_tmp=is_tmp) - def write(self, obj:Any, indent:int=None): + def write(self, obj:Any, indent: Optional[int] = None): jsonpickle.set_preferred_backend('simplejson') jsonpickle.set_encoder_options('simplejson', indent=indent) json_str=jsonpickle.encode(obj) diff --git a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py index bb7cd830c..8022f6244 100644 --- a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py +++ b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py @@ -2,7 +2,7 @@ import os import tempfile from pathlib import Path -from typing import Optional, Callable +from typing import Optional, Callable, Generator import jinja2 import logging @@ -52,16 +52,16 @@ def restore_logger(logger_creator: Callable[[], logging.Logger]): } yield after_logger = logger_creator() - after_logger.level = logger_info[LOG_LEVEL] - after_logger.handlers = logger_info[HANDLERS] - after_logger.filters = logger_info[FILTERS] - after_logger.propagate = logger_info[PROPAGATE] + after_logger.level = logger_info[LOG_LEVEL] # type: ignore + after_logger.handlers = logger_info[HANDLERS] # type: ignore + after_logger.filters = logger_info[FILTERS] # type: ignore + after_logger.propagate = logger_info[PROPAGATE] # type: ignore @contextlib.contextmanager def get_luigi_log_config(log_file_target: Path, use_job_specific_log_file: bool, - log_level: Optional[str] = None) -> Path: + log_level: Optional[str] = None) -> Generator[Path, None, None]: """ Yields a context manager containing the path of the log-config file. log_file_target contains the location of the log-file. diff --git a/exasol_integration_test_docker_environment/lib/base/ssh_access.py b/exasol_integration_test_docker_environment/lib/base/ssh_access.py index 8da3a585d..129dcc15b 100644 --- a/exasol_integration_test_docker_environment/lib/base/ssh_access.py +++ b/exasol_integration_test_docker_environment/lib/base/ssh_access.py @@ -95,7 +95,7 @@ def generate(cls) -> 'SshKey': return SshKey(rsa_key) @classmethod - def from_cache(cls, cache_directory: Path = None) -> 'SshKey': + def from_cache(cls, cache_directory: Optional[Path] = None) -> 'SshKey': cache = SshKeyCache(cache_directory) priv = cache.private_key with portalocker.Lock(_path(_LOCK_FILE), 'wb', timeout=10) as fh: @@ -106,6 +106,6 @@ def from_cache(cls, cache_directory: Path = None) -> 'SshKey': os.makedirs(cache.directory, mode=0o700, exist_ok=True) return ( cls.generate() - .write_private_key(priv) - .write_public_key(cache.public_key) + .write_private_key(str(priv)) + .write_public_key(str(cache.public_key)) ) diff --git a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py index ce204ee07..e7d61407a 100644 --- a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py @@ -46,7 +46,7 @@ def handle_failure(self, exception, exception_tb): f.write("%s" % self.task_id) def collect_failures(self) -> Dict[str,None]: - failures = OrderedDict() + failures : Dict[str, None] = OrderedDict() failures.update(self.collect_failures_of_child_tasks()) if self.get_failure_log_path().exists(): with self.get_failure_log_path().open("r") as f: @@ -59,7 +59,7 @@ def collect_failures(self) -> Dict[str,None]: return failures def collect_failures_of_child_tasks(self) -> Dict[str,None]: - failures_of_child_tasks = OrderedDict() + failures_of_child_tasks : Dict[str, None] = OrderedDict() if self._run_dependencies_target.exists(): _run_dependencies_tasks_from_target = self._run_dependencies_target.read() else: diff --git a/exasol_integration_test_docker_environment/lib/config/build_config.py b/exasol_integration_test_docker_environment/lib/config/build_config.py index 832edcbae..52dd43f81 100644 --- a/exasol_integration_test_docker_environment/lib/config/build_config.py +++ b/exasol_integration_test_docker_environment/lib/config/build_config.py @@ -4,13 +4,13 @@ class build_config(luigi.Config): - force_pull = luigi.BoolParameter(False) - force_load = luigi.BoolParameter(False) - force_rebuild = luigi.BoolParameter(False) - force_rebuild_from = luigi.ListParameter([]) - log_build_context_content = luigi.BoolParameter(False) + force_pull : bool = luigi.BoolParameter(False) # type: ignore + force_load : bool = luigi.BoolParameter(False) # type: ignore + force_rebuild : bool = luigi.BoolParameter(False) # type: ignore + force_rebuild_from : List[str] = luigi.ListParameter([]) # type: ignore + log_build_context_content : bool = luigi.BoolParameter(False) # type: ignore # keep_build_context = luigi.BoolParameter(False) - temporary_base_directory = luigi.OptionalParameter(None) - output_directory = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) - cache_directory = luigi.OptionalParameter("") - build_name = luigi.OptionalParameter("") + temporary_base_directory : Optional[str] = luigi.OptionalParameter(None) # type: ignore + output_directory : str = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) # type: ignore + cache_directory : Optional[str] = luigi.OptionalParameter("") # type: ignore + build_name : Optional[str] = luigi.OptionalParameter("") # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/data/container_info.py b/exasol_integration_test_docker_environment/lib/data/container_info.py index ed0afcb54..8e19a82e9 100644 --- a/exasol_integration_test_docker_environment/lib/data/container_info.py +++ b/exasol_integration_test_docker_environment/lib/data/container_info.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from exasol_integration_test_docker_environment.lib.base.info import Info from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo @@ -10,7 +10,7 @@ def __init__(self, container_name: str, ip_address: str, network_aliases: List[str], network_info: DockerNetworkInfo, - volume_name: str = None): + volume_name: Optional[str] = None): self.network_aliases = network_aliases self.ip_address = ip_address self.network_info = network_info diff --git a/exasol_integration_test_docker_environment/lib/data/database_info.py b/exasol_integration_test_docker_environment/lib/data/database_info.py index 5d08f903d..941adcb19 100644 --- a/exasol_integration_test_docker_environment/lib/data/database_info.py +++ b/exasol_integration_test_docker_environment/lib/data/database_info.py @@ -1,3 +1,5 @@ +from typing import Optional + from exasol_integration_test_docker_environment.lib.base.info import Info from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports @@ -10,9 +12,9 @@ def __init__( host: str, ports: Ports, reused: bool, - container_info: ContainerInfo = None, - ssh_info: SshInfo = None, - forwarded_ports: Ports = None, + container_info: Optional[ContainerInfo] = None, + ssh_info: Optional[SshInfo] = None, + forwarded_ports: Optional[Ports] = None, ): self.container_info = container_info self.ports = ports diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py index ca790270e..7daa9a2fa 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py @@ -85,12 +85,13 @@ def _create_build_task_for_image_info( def _build_with_depenencies_is_requested(self, image_info: ImageInfo, shortcut_build: bool): needs_to_be_build = image_info.image_state == ImageState.NEEDS_TO_BE_BUILD.name result = (not shortcut_build or needs_to_be_build) and \ - len(image_info.depends_on_images) > 0 + image_info.depends_on_images and len(image_info.depends_on_images) > 0 return result def _create_build_task_with_dependencies( self, image_info: ImageInfo, shortcut_build: bool = True) -> DockerCreateImageTask: + assert image_info.depends_on_images required_tasks = \ self._create_build_tasks_for_image_infos( image_info.depends_on_images, shortcut_build) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index 28ebf193d..e06eb482e 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -1,3 +1,4 @@ +import abc from pathlib import Path from typing import Dict, Type @@ -38,6 +39,7 @@ def __init__(self, *args, **kwargs): ) self._dockerfile = self.get_dockerfile() + @abc.abstractmethod def get_source_repository_name(self) -> str: """ Called by the constructor to get the image name for pulls. Sub classes need to implement this method. @@ -45,6 +47,7 @@ def get_source_repository_name(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_target_repository_name(self) -> str: """ Called by the constructor to get the image name for pushs. Sub classes need to implement this method. @@ -52,6 +55,7 @@ def get_target_repository_name(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_source_image_tag(self) -> str: """ Called by the constructor to get the image tag for pulls. Sub classes need to implement this method. @@ -59,6 +63,7 @@ def get_source_image_tag(self) -> str: """ raise AbstractMethodException() + @abc.abstractmethod def get_target_image_tag(self) -> str: """ Called by the constructor to get the image tag for pushs. Sub classes need to implement this method. @@ -76,6 +81,7 @@ def get_mapping_of_build_files_and_directories(self) -> Dict[str, str]: """ raise AbstractMethodException() + @abc.abstractmethod def get_dockerfile(self) -> str: """ Called by the constructor to get the path to the dockerfile. @@ -106,6 +112,7 @@ def get_transparent_build_arguments(self) -> Dict[str, str]: """ return dict() + @abc.abstractmethod def is_rebuild_requested(self) -> bool: raise AbstractMethodException() @@ -122,14 +129,15 @@ def register_required(self): tasks = None self.dependencies_futures = self.register_dependencies(tasks) - def keys_are_string(self, task_classes): + def keys_are_string(self, task_classes) -> bool: return all(isinstance(key, str) for key in task_classes.keys()) - def values_are_subclass_of_baseclass(self, task_classes): + def values_are_subclass_of_baseclass(self, task_classes) -> bool: return all(issubclass(value, DockerAnalyzeImageTask) for value in task_classes.values()) + @abc.abstractmethod def requires_tasks(self) -> Dict[str, Type["DockerAnalyzeImageTask"]]: pass diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py index 55fe41af2..71c313238 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py @@ -17,11 +17,11 @@ class DockerCreateImageTask(DockerBaseTask): - image_name = luigi.Parameter() + image_name : str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info = JsonPickleParameter(ImageInfo, + image_info : ImageInfo = JsonPickleParameter(ImageInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ImageInfo + significant=True) # type: ignore def run_task(self): new_image_info = yield from self.build(self.image_info) @@ -70,9 +70,9 @@ def rename_source_image_to_target_image(self, image_info): class DockerCreateImageTaskWithDeps(DockerCreateImageTask): # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - required_task_infos = JsonPickleParameter(RequiredTaskInfoDict, + required_task_infos : RequiredTaskInfoDict = JsonPickleParameter(RequiredTaskInfoDict, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: RequiredTaskInfoDict + significant=True) # type: ignore def register_required(self): self.required_tasks = {key: self.create_required_task(required_task_info) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py index bf3214acc..071f8c0c1 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py @@ -6,8 +6,8 @@ class DockerImageCreatorBaseTask(DockerBaseTask): - image_name = luigi.Parameter() + image_name : str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info = JsonPickleParameter(ImageInfo, + image_info : ImageInfo= JsonPickleParameter(ImageInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type:ImageInfo + significant=True) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py index 0f2da542e..3353a1317 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py @@ -35,7 +35,7 @@ def map(self, image: str, queue: mp.Queue): def check(self, image: str): log_handler = DockerRegistryImageCheckerPullLogHandler() - queue = mp.Queue() + queue : mp.Queue = mp.Queue() process = mp.Process(target=self.map, args=(image, queue)) process.start() try: diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py index 43b8f7e38..57a8569b0 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py @@ -233,6 +233,7 @@ def __init__(self, hashfunc: str = 'md5', def hash(self, path_mapping: DestinationMapping): src_path = path_mapping.source_path dest_path = path_mapping.destination_path + assert self.hash_func hasher = self.hash_func() hasher.update(str(dest_path).encode('utf-8')) if self.hash_permissions: @@ -252,6 +253,7 @@ def __init__(self, hashfunc: str = 'md5', blocksize: str = "64kb"): raise NotImplementedError('{} not implemented.'.format(hashfunc)) def hash(self, filepath: Path): + assert self.hash_func hasher = self.hash_func() with open(filepath, 'rb') as fp: while True: diff --git a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py index 63685d510..28203129b 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Dict, Any +from typing import Dict, Any, Optional from exasol_integration_test_docker_environment.lib.base.info import Info @@ -48,7 +48,7 @@ def __init__(self, build_name: str = "", build_date_time: datetime = datetime.utcnow(), image_state: ImageState = ImageState.NOT_EXISTING, - depends_on_images: Dict[str, "ImageInfo"] = None): + depends_on_images: Optional[Dict[str, "ImageInfo"]] = None): self.build_name = build_name self.date_time = str(build_date_time) self.commit = commit diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py index c574b86f7..fc6d38ea7 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py @@ -14,9 +14,9 @@ class DockerPushImageTask(DockerPushImageBaseTask): # don't want to wait for the push finishing before starting to build depended images, # but we also need to create a DockerPushImageTask for each DockerCreateImageTask of a goal - required_task_info = JsonPickleParameter(RequiredTaskInfo, + required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type:RequiredTaskInfo + significant=True) # type:ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py index 52c78da05..9accd37d4 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py @@ -14,9 +14,9 @@ class DockerSaveImageTask(DockerSaveImageBaseTask): # don't want to wait for the save finishing before starting to build depended images, # but we also need to create a DockerSaveImageTask for each DockerCreateImageTask of a goal - required_task_info = JsonPickleParameter(RequiredTaskInfo, + required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: RequiredTaskInfo + significant=True) # type: ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py index b1822570b..8b8adf5ed 100644 --- a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py @@ -1,10 +1,10 @@ import logging -from typing import Iterator +from typing import Iterator, List from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -def remove_docker_networks(networks: Iterator[str]): +def remove_docker_networks(networks: List[str]): """ Removes the given networks using docker API. """ diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index e50db2b1b..af70e8a4a 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -28,7 +28,7 @@ class AbstractSpawnTestEnvironment(DockerBaseTask, GeneralSpawnTestEnvironmentParameter, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() # type: str + environment_name : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -68,7 +68,7 @@ def create_test_environment_info_in_test_container( shell_variables: ShellVariables, json: str, ): - test_container_name = test_environment_info.test_container_info.container_name + test_container_name = test_environment_info.test_container_info.container_name # type: ignore with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(test_container_name) self.logger.info(f"Create test environment info in test container '{test_container_name}' at '/'") @@ -102,7 +102,7 @@ def create_test_environment_info_in_test_container_and_on_host( json, ) - def _default_bridge_ip_address(self, test_environment_info) -> str: + def _default_bridge_ip_address(self, test_environment_info) -> Optional[str]: if test_environment_info.database_info.container_info is None: return None container_name = test_environment_info.database_info.container_info.container_name @@ -111,27 +111,25 @@ def _default_bridge_ip_address(self, test_environment_info) -> str: return default_bridge_ip_address(db_container) def collect_shell_variables(self, test_environment_info) -> ShellVariables: - default_bridge_ip_address = self._default_bridge_ip_address(test_environment_info) return ShellVariables.from_test_environment_info( - default_bridge_ip_address, + self._default_bridge_ip_address(test_environment_info), # type: ignore test_environment_info, ) def _start_database(self, attempt) \ -> Generator[BaseTask, BaseTask, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: - network_info = yield from self._create_network(attempt) - ssl_volume_info = None - if self.create_certificates: - ssl_volume_info = yield from self._create_ssl_certificates() - database_info, test_container_info = \ - yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) - is_database_ready = yield from self._wait_for_database(database_info, attempt) - return network_info, database_info, is_database_ready, test_container_info - - def _create_ssl_certificates(self) -> DockerVolumeInfo: + network_info = yield from self._create_network(attempt) # type: ignore + ssl_volume_info = None # type: ignore + if self.create_certificates: # type: ignore + ssl_volume_info = yield from self._create_ssl_certificates() # type: ignore + database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) # type: ignore + is_database_ready = yield from self._wait_for_database(database_info, attempt) # type: ignore + return network_info, database_info, is_database_ready, test_container_info # type: ignore + + def _create_ssl_certificates(self) -> DockerVolumeInfo: # type: ignore ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) ssl_info = self.get_values_from_future(ssl_info_future) - return ssl_info + return ssl_info # type: ignore def create_ssl_certificates(self): raise AbstractMethodException() @@ -144,12 +142,12 @@ def _create_network(self, attempt): def create_network_task(self, attempt: int): raise AbstractMethodException() - def _spawn_database_and_test_container( + def _spawn_database_and_test_container( # type: ignore self, network_info: DockerNetworkInfo, certificate_volume_info: Optional[DockerVolumeInfo], attempt: int, - ) -> Tuple[DatabaseInfo, Optional[ContainerInfo]]: + ) -> Tuple[DatabaseInfo, Optional[ContainerInfo]]: # type: ignore def volume_name(info): return None if info is None else info.volume_name diff --git a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py index 5c570bf04..df4291662 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py @@ -42,7 +42,7 @@ def get_dockerfile(self): def is_rebuild_requested(self) -> bool: config = build_config() - return config.force_rebuild + return bool(config.force_rebuild) class DockerTestContainerBuildBase(DockerBuildBase, TestContainerParameter): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py index 032691a9c..edc5bbfb4 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py @@ -1,8 +1,11 @@ -from typing import Dict +from typing import Dict, Optional, Generator, Set import docker import luigi + +from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask +from exasol_integration_test_docker_environment.lib.base.pickle_target import PickleTarget from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import \ @@ -14,14 +17,14 @@ class CreateSSLCertificatesTask(DockerBaseTask): - environment_name = luigi.Parameter() - docker_runtime = luigi.OptionalParameter(None, significant=False) - db_container_name = luigi.Parameter(significant=False) - network_name = luigi.Parameter() - reuse = luigi.BoolParameter(False, significant=False) - no_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - volume_name = luigi.Parameter() + environment_name : str = luigi.Parameter() # type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + db_container_name : str = luigi.Parameter(significant=False) # type: ignore + network_name : str = luigi.Parameter() # type: ignore + reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + volume_name : str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -53,7 +56,7 @@ def run_task(self): self.return_object(self.volume_info) - def build_image(self) -> Dict[str, ImageInfo]: + def build_image(self) -> Generator["BaseTask", PickleTarget, Set[str]]: task = self.create_child_task(task_class=DockerCertificateContainerBuild, certificate_container_root_directory=self._temp_resource_directory.tmp_directory) image_infos_future = yield from self.run_dependencies(task) @@ -66,6 +69,7 @@ def get_volume_info(self, reused: bool) -> DockerVolumeInfo: if volume_properties['Name'] == self.volume_name: return DockerVolumeInfo(volume_name=str(self.volume_name), mount_point=volume_properties['Mountpoint'], reused=reused) + raise RuntimeError(f"Volume Info not found for created volume {self.volume_name}") def create_docker_volume(self) -> DockerVolumeInfo: self.remove_volume(self.volume_name) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py index 9a6ebe446..84af03192 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py @@ -1,4 +1,5 @@ from pathlib import PurePath +from typing import List import docker from exasol_integration_test_docker_environment.lib.base.db_os_executor \ @@ -21,7 +22,7 @@ def find_exaplus( exit, output = os_executor.exec("find /opt -name 'exaplus' -type f") if exit != 0: raise RuntimeError(f"Exaplus not found on docker db! Output is {output}") - found_paths = list(filter(None, output.decode("UTF-8").split("\n"))) + found_paths : List[str] = list(filter(None, output.decode("UTF-8").split("\n"))) if len(found_paths) != 1: raise RuntimeError(f"Error determining exaplus path! Output is {output}") exaplus_path = PurePath(found_paths[0]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py index f0ef9a025..1a472d941 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py @@ -13,9 +13,9 @@ class PopulateTestDataToDatabase(DockerBaseTask, DatabaseCredentialsParameter): logger = logging.getLogger('luigi-interface') - environment_name = luigi.Parameter() - test_environment_info = JsonPickleParameter( - EnvironmentInfo, significant=False) # type: EnvironmentInfo + environment_name : str = luigi.Parameter() # type: ignore + test_environment_info : EnvironmentInfo = JsonPickleParameter( + EnvironmentInfo, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py index cdf4b6a2d..61c08bca5 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py @@ -2,7 +2,7 @@ import time from pathlib import Path from threading import Thread -from typing import Callable, Optional +from typing import Callable, Optional, List from docker.models.containers import Container @@ -13,15 +13,15 @@ class DBContainerLogThread(Thread): def __init__(self, container: Container, logger, log_file: Path, description: str): super().__init__() - self.complete_log = [] + self.complete_log : List[str] = list() self.description = description self.logger = logger self.log_file = log_file self.container = container self.finish = False - self.previous_timestamp = None - self.current_timestamp = None - self.error_message = None + self.previous_timestamp : Optional[float] = None + self.current_timestamp : Optional[float] = None + self.error_message : Optional[str] = None self.ignore_error_return_codes = ( "(membership) returned with state 1", # exclude webui not found in 7.0.0 "rsyslogd) returned with state 1" # exclude rsyslogd which might crash when running itde under lima diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py index e15b05b0f..b39ce02bc 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py @@ -8,10 +8,10 @@ class WaitForTestExternalDatabase(DockerBaseTask, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - database_info = JsonPickleParameter(DatabaseInfo, significant=False) # type: DatabaseInfo - db_startup_timeout_in_seconds = luigi.IntParameter(1 * 60, significant=False) - attempt = luigi.IntParameter(1) + environment_name : str = luigi.Parameter() # type: ignore + database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds : int = luigi.IntParameter(1 * 60, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore def run_task(self): # Since we can't assume that the test container exists, we cannot connect easily here diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py index 268726dd2..9f6bc900d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py @@ -19,12 +19,12 @@ class WaitForTestDockerDatabase(DockerBaseTask, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - database_info = JsonPickleParameter(DatabaseInfo, significant=False) # type: DatabaseInfo - db_startup_timeout_in_seconds = luigi.IntParameter(10 * 60, significant=False) - attempt = luigi.IntParameter(1) - docker_db_image_version = luigi.Parameter() - executor_factory = JsonPickleParameter(DbOsExecFactory, significant=False) + environment_name : str = luigi.Parameter() # type: ignore + database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds : int = luigi.IntParameter(10 * 60, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore + docker_db_image_version : str = luigi.Parameter() # type: ignore + executor_factory : DbOsExecFactory = JsonPickleParameter(DbOsExecFactory, significant=False) # type: ignore def run_task(self): with self._get_docker_client() as docker_client: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py index e8f42199d..fe4efcaf4 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Tuple from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION @@ -13,14 +13,14 @@ def __init__(self, major, minor, stable): @classmethod def from_db_version_str(cls, db_version_str: Optional[str]): - db_version = db_version_str - if db_version_str in (None, DEFAULT_VERSION): + db_version : str = LATEST_DB_VERSION if db_version_str is None else db_version_str + if db_version_str == DEFAULT_VERSION: db_version = LATEST_DB_VERSION if db_version.endswith("-d1"): db_version = "-".join(db_version.split("-")[0:-1]) if db_version.startswith("prerelease-"): db_version = "-".join(db_version.split("-")[1:]) - version = tuple([int(v) for v in db_version.split(".")]) + version : Tuple[int,...] = tuple([int(v) for v in db_version.split(".")]) if len(version) != 3: raise ValueError(f"Invalid db version given: {db_version}") return DbVersion(version[0], version[1], version[2]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py index 9b15bce22..ab28d9a37 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py @@ -1,3 +1,5 @@ +from typing import Optional + from docker.models.containers import Container import io import tarfile @@ -8,8 +10,8 @@ class DockerContainerCopy: def __init__(self, container:Container): super().__init__() self.open = True - self.file_like_object = io.BytesIO() - self.tar = tarfile.open(fileobj=self.file_like_object, mode="x") + self.file_like_object : Optional[io.BytesIO] = io.BytesIO() + self.tar : Optional[tarfile.TarFile] = tarfile.open(fileobj=self.file_like_object, mode="x") self.container = container def __del__(self): @@ -31,17 +33,21 @@ def add_string_to_file(self, name: str, string: str): tar_info = tarfile.TarInfo(name=name) tar_info.mtime = time.time() tar_info.size = len(encoded) + assert self.tar self.tar.addfile(tarinfo=tar_info, fileobj=bytes_io) def add_file(self, host_path:str, path_in_tar:str): self.is_open_or_raise() + assert self.tar self.tar.add(host_path, path_in_tar) def copy(self, path_in_container:str): self.is_open_or_raise() + assert self.tar self.tar.close() self.open = False self.tar = None + assert self.file_like_object self.container.put_archive(path_in_container, self.file_like_object.getbuffer().tobytes()) self.file_like_object = None diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py index 539a62fe7..2b89f121a 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py @@ -1,3 +1,5 @@ +from typing import Optional + import luigi from luigi import Config @@ -5,34 +7,34 @@ class ExternalDatabaseXMLRPCParameter(Config): - external_exasol_xmlrpc_host = luigi.OptionalParameter() - external_exasol_xmlrpc_port = luigi.IntParameter(443) - external_exasol_xmlrpc_user = luigi.OptionalParameter() - external_exasol_xmlrpc_cluster_name = luigi.OptionalParameter() - external_exasol_xmlrpc_password = luigi.OptionalParameter( + external_exasol_xmlrpc_host : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_port : int = luigi.IntParameter(443) #type: ignore + external_exasol_xmlrpc_user : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_cluster_name : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_xmlrpc_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) + ) #type: ignore # See ticket https://github.com/exasol/integration-test-docker-environment/issues/341 class ExternalDatabaseHostParameter(Config): - external_exasol_db_host = luigi.OptionalParameter() - external_exasol_db_port = luigi.IntParameter() - external_exasol_bucketfs_port = luigi.IntParameter() - external_exasol_ssh_port = luigi.IntParameter() + external_exasol_db_host : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_db_port : int = luigi.IntParameter() #type: ignore + external_exasol_bucketfs_port : int = luigi.IntParameter() #type: ignore + external_exasol_ssh_port : int = luigi.IntParameter() #type: ignore class ExternalDatabaseCredentialsParameter( ExternalDatabaseHostParameter, ExternalDatabaseXMLRPCParameter, ): - external_exasol_db_user = luigi.OptionalParameter() - external_exasol_db_password = luigi.OptionalParameter( + external_exasol_db_user : Optional[str] = luigi.OptionalParameter() #type: ignore + external_exasol_db_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) - external_exasol_bucketfs_write_password = luigi.OptionalParameter( + ) #type: ignore + external_exasol_bucketfs_write_password : Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) + ) #type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/ports.py b/exasol_integration_test_docker_environment/lib/test_environment/ports.py index a7506fc17..25f3d1f32 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/ports.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/ports.py @@ -1,6 +1,6 @@ import socket from contextlib import ExitStack -from typing import List, Optional +from typing import List, Optional, Generator def find_free_ports(num_ports: int) -> List[int]: @@ -9,7 +9,7 @@ def new_socket(): def bind(sock: socket.socket, port: int): sock.bind(('', port)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - def acquire_port_numbers(num_ports: int) -> List[int]: + def acquire_port_numbers(num_ports: int) -> Generator[int, None, None]: with ExitStack() as stack: sockets = [stack.enter_context(new_socket()) for dummy in range(num_ports)] for sock in sockets: @@ -55,5 +55,8 @@ def __init__(self, database: int, bucketfs: int, ssh: Optional[int] = None): @classmethod def random_free(cls, ssh: bool = True) -> 'Ports': - ports = find_free_ports(3 if ssh else 2) + [None] - return Ports(*ports[:3]) + ports : List[int] = find_free_ports(3 if ssh else 2) + if ssh: + return Ports(ports[0], ports[1], ports[2]) + else: + return Ports(ports[0], ports[1], None) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py index 06e25f8f1..90ce2f47e 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py @@ -6,14 +6,14 @@ class PrepareDockerNetworkForTestEnvironment(DockerBaseTask): - environment_name = luigi.Parameter() - network_name = luigi.Parameter() - test_container_name = luigi.Parameter(significant=False) - db_container_name = luigi.OptionalParameter(None, significant=False) - reuse = luigi.BoolParameter(False, significant=False) - no_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - attempt = luigi.IntParameter(-1) + environment_name : str = luigi.Parameter() # type: ignore + network_name : str = luigi.Parameter() # type: ignore + test_container_name : str = luigi.Parameter(significant=False) # type: ignore + db_container_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + attempt : int = luigi.IntParameter(-1) # type: ignore def run_task(self): self.network_info = None diff --git a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py index c9f65af8d..78ecd7354 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py @@ -21,9 +21,9 @@ class SetupExternalDatabaseHost(DependencyLoggerBaseTask, ExternalDatabaseXMLRPCParameter, ExternalDatabaseHostParameter, DatabaseCredentialsParameter): - environment_name = luigi.Parameter() - network_info = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - attempt = luigi.IntParameter(1) + environment_name : str = luigi.Parameter() # type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore def run_task(self): database_host = self.external_exasol_db_host @@ -60,8 +60,8 @@ def setup_database(self): def get_xml_rpc_object(self, object_name: str = ""): uri = 'https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}'.format( - user=quote_plus(self.external_exasol_xmlrpc_user), - password=quote_plus(self.external_exasol_xmlrpc_password), + user=quote_plus(self.external_exasol_xmlrpc_user or ""), + password=quote_plus(self.external_exasol_xmlrpc_password or ""), host=self.external_exasol_xmlrpc_host, port=self.external_exasol_xmlrpc_port, cluster_name=self.external_exasol_xmlrpc_cluster_name, @@ -77,8 +77,8 @@ def start_database(self, cluster: ServerProxy): all_nodes_online = False while not all_nodes_online: all_nodes_online = True - for nodeName in cluster.getNodeList(): - node_state = self.get_xml_rpc_object(nodeName).getNodeState() + for nodeName in cluster.getNodeList(): # type: ignore + node_state = self.get_xml_rpc_object(nodeName).getNodeState() # type: ignore if node_state['status'] != 'Running': all_nodes_online = False break @@ -95,10 +95,10 @@ def start_database(self, cluster: ServerProxy): self.logger.info('EXAStorage already online; continuing startup process') # triggering database startup - for databaseName in cluster.getDatabaseList(): - database = self.get_xml_rpc_object('/db_' + quote_plus(databaseName)) + for databaseName in cluster.getDatabaseList(): # type: ignore + database = self.get_xml_rpc_object('/db_' + quote_plus(str(databaseName))) if not database.runningDatabase(): - self.logger.info('Starting database instance %s' % databaseName) + self.logger.info('Starting database instance %s' % str(databaseName)) database.startDatabase() else: - self.logger.info('Database instance %s already running' % databaseName) + self.logger.info('Database instance %s already running' % str(databaseName)) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index 174bcdf31..ee9e76926 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -20,13 +20,13 @@ def from_test_environment_info( default_bridge_ip_address and EnvironmentInfo. """ info = test_environment_info - env = { + env : Dict[str, str] = { "NAME": info.name, "TYPE": info.type, "DATABASE_HOST": info.database_info.host, - "DATABASE_DB_PORT": info.database_info.ports.database, - "DATABASE_BUCKETFS_PORT": info.database_info.ports.bucketfs, - "DATABASE_SSH_PORT": info.database_info.ports.ssh, + "DATABASE_DB_PORT": str(info.database_info.ports.database) if info.database_info.ports.database else "", + "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs) if info.database_info.ports.bucketfs else "", + "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh else "", } if info.database_info.container_info is not None: network_aliases = " ".join(info.database_info.container_info.network_aliases) @@ -34,7 +34,7 @@ def from_test_environment_info( "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, "DATABASE_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', "DATABASE_CONTAINER_IP_ADDRESS": info.database_info.container_info.ip_address, - "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name, + "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name or "", "DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS": default_bridge_ip_address, }) if info.test_container_info is not None: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index 3a6598e35..48ca628dc 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -22,31 +22,31 @@ class SpawnTestContainer(DockerBaseTask, TestContainerParameter): - environment_name = luigi.Parameter() - test_container_name = luigi.Parameter() - network_info = JsonPickleParameter( - DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - ip_address_index_in_subnet = luigi.IntParameter(significant=False) - attempt = luigi.IntParameter(1) - reuse_test_container = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - docker_runtime = luigi.OptionalParameter(None, significant=False) - certificate_volume_name = luigi.OptionalParameter(None, significant=False) + environment_name : str = luigi.Parameter() #type: ignore + test_container_name : str = luigi.Parameter() #type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter( + DockerNetworkInfo, significant=False) # type: ignore + ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) #type: ignore + attempt : int = luigi.IntParameter(1) #type: ignore + reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore + certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.ip_address_index_in_subnet < 0: + if self.ip_address_index_in_subnet < 0: # type: ignore raise Exception( "ip_address_index_in_subnet needs to be greater than 0 got %s" - % self.ip_address_index_in_subnet) + % self.ip_address_index_in_subnet) # type: ignore def register_required(self): self.test_container_image_future = \ self.register_dependency(self.create_child_task(task_class=DockerTestContainerBuild, test_container_content=self.test_container_content)) - def is_reuse_possible(self) -> bool: + def is_reuse_possible(self) -> luigi.BoolParameter | bool: test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ImageInfo test_container = None @@ -55,16 +55,16 @@ def is_reuse_possible(self) -> bool: test_container = docker_client.containers.get(self.test_container_name) except Exception as e: pass - ret_val = self.network_info.reused and self.reuse_test_container and \ + ret_val : bool = bool(self.network_info.reused and self.reuse_test_container and \ test_container is not None and \ test_container_image_info.get_target_complete_name() in test_container.image.tags and \ - test_container_image_info.image_state == ImageState.USED_LOCAL.name + test_container_image_info.image_state == ImageState.USED_LOCAL.name) return ret_val def run_task(self): subnet = netaddr.IPNetwork(self.network_info.subnet) - ip_address = str(subnet[2 + self.ip_address_index_in_subnet]) + ip_address = str(subnet[2 + self.ip_address_index_in_subnet]) # type: ignore container_info = None if self.is_reuse_possible(): @@ -101,17 +101,17 @@ def _try_to_reuse_test_container(self, ip_address: str, except Exception as e: self.logger.warning("Tried to reuse test container %s, but got Exeception %s. " "Fallback to create new database.", self.test_container_name, e) - return container_info + return container_info # type: ignore def _create_test_container(self, ip_address, network_info: DockerNetworkInfo) -> ContainerInfo: - self._remove_container(self.test_container_name) + self._remove_container(self.test_container_name) # type: ignore self.logger.info(f"Creating new test container {self.test_container_name}") test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] volumes = dict() - for runtime_mapping in self.test_container_content.runtime_mappings: + for runtime_mapping in self.test_container_content.runtime_mappings: # type: ignore volumes[runtime_mapping.source.absolute()] = { "bind": runtime_mapping.target, "mode": "rw" @@ -160,14 +160,14 @@ def create_container_info(self, ip_address: str, network_aliases: List[str], test_container = docker_client.containers.get(self.test_container_name) if test_container.status != "running": raise Exception(f"Container {self.test_container_name} not running") - container_info = ContainerInfo(container_name=self.test_container_name, + container_info = ContainerInfo(container_name=self.test_container_name, # type: ignore ip_address=ip_address, network_aliases=network_aliases, network_info=network_info) return container_info def _get_export_directory(self): - return self.get_values_from_future(self.export_directory_future) + return self.get_values_from_future(self.export_directory_future) # type: ignore def _remove_container(self, container_name: str): try: @@ -202,6 +202,6 @@ def cleanup_task(self, success: bool): (not success and not self.no_test_container_cleanup_after_failure): try: self.logger.info(f"Cleaning up container %s", self.test_container_name) - self._remove_container(self.test_container_name) + self._remove_container(self.test_container_name) # type: ignore except Exception as e: self.logger.error(f"Error during removing container %s: %s", self.test_container_name, e) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 4799134ff..364b7d484 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -49,16 +49,16 @@ def int_or_none(value: str) -> Optional[int]: class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): - environment_name = luigi.Parameter() # type: str - db_container_name = luigi.Parameter() # type: str - attempt = luigi.IntParameter(1) # type: int - network_info = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: DockerNetworkInfo - ip_address_index_in_subnet = luigi.IntParameter(significant=False) # type: int - docker_runtime = luigi.OptionalParameter(None, significant=False) # type: str - certificate_volume_name = luigi.OptionalParameter(None, significant=False) + environment_name : str = luigi.Parameter() # type: ignore + db_container_name : str = luigi.Parameter() # type: ignore + attempt : int = luigi.IntParameter(1) # type: ignore + network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) # type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore additional_db_parameter = luigi.ListParameter() - ssh_user = luigi.Parameter("root") - ssh_key_file = luigi.OptionalParameter(None, significant=False) + ssh_user : str = luigi.Parameter("root") # type: ignore + ssh_key_file : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -92,8 +92,6 @@ def run_task(self): def _try_to_reuse_database(self, db_ip_address: str) -> DatabaseInfo: self.logger.info("Try to reuse database container %s", self.db_container_name) - database_info = None - ssh_key = self._get_ssh_key() try: database_info = self._create_database_info(db_ip_address=db_ip_address, reused=True) except Exception as e: @@ -103,8 +101,8 @@ def _try_to_reuse_database(self, db_ip_address: str) -> DatabaseInfo: def _get_ssh_key(self) -> SshKey: if self.ssh_key_file: - return SshKey.read_from(self.ssh_key_file) - self.ssh_key_file = SshKeyCache().private_key + return SshKey.read_from(self.ssh_key_file) # type: ignore + self.ssh_key_file = SshKeyCache().private_key # type: ignore return SshKey.from_cache() def _handle_output(self, output_generator, image_info: ImageInfo): @@ -201,7 +199,7 @@ def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInf network_info=self.network_info, volume_name=self._get_db_volume_name(), ) - ssh_info = SshInfo(self.ssh_user, self.ssh_key_file) + ssh_info = SshInfo(self.ssh_user, self.ssh_key_file) # type: ignore database_info = DatabaseInfo( host=db_ip_address, ports=self.internal_ports, @@ -320,7 +318,7 @@ def _upload_init_db_files( ): copy = DockerContainerCopy(container) init_script = self._db_file("init_db.sh") - copy.add_string_to_file("init_db.sh", init_script.read_text()) + copy.add_string_to_file("init_db.sh", init_script.read_text()) # type: ignore self._add_exa_conf(copy, db_private_network, authorized_keys) copy.copy("/") @@ -336,8 +334,8 @@ def _add_exa_conf( certificate_dir = CERTIFICATES_MOUNT_DIR if self.certificate_volume_name is not None \ else CERTIFICATES_DEFAULT_DIR template_file = self._db_file("EXAConf") - template = Template(template_file.read_text()) - additional_db_parameter_str = " ".join(self.additional_db_parameter) + template = Template(template_file.read_text()) # type: ignore + additional_db_parameter_str = " ".join(self.additional_db_parameter) # type: ignore rendered_template = template.render(private_network=db_private_network, db_version=str(self.db_version), db_port=self.internal_ports.database, @@ -346,7 +344,7 @@ def _add_exa_conf( image_version=self.docker_db_image_version, mem_size=self.mem_size, disk_size=self.disk_size, - name_servers=",".join(self.nameservers), + name_servers=",".join(self.nameservers), # type: ignore certificate_dir=certificate_dir, additional_db_parameters=additional_db_parameter_str, authorized_keys=authorized_keys) diff --git a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py index cfe2b35f7..54333b504 100644 --- a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py +++ b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from types import ModuleType +from typing import Optional import importlib_resources as ir @@ -60,7 +61,7 @@ def __init__(self, resource_package: ModuleType): # We need to transform the module to a string and later back to a module # because this class will be pickled by luigi and modules are not supported for serialization self._resource_package_str = resource_package.__name__ - self._tmp_directory = None + self._tmp_directory : Optional[tempfile.TemporaryDirectory] = None @property def tmp_directory(self): @@ -71,14 +72,16 @@ def tmp_directory(self): def create(self) -> str: self._tmp_directory = tempfile.TemporaryDirectory() + assert self._tmp_directory source_path = ir.files(self._resource_package_str) LOG.debug(f"Copying resource package: '{self._resource_package_str}' to '{self._tmp_directory.name}'") _copy_importlib_resources_dir_tree(source_path, Path(self._tmp_directory.name)) return self._tmp_directory.name def cleanup(self): - self._tmp_directory.cleanup() - self._tmp_directory = None + if self._tmp_directory is not None: + self._tmp_directory.cleanup() + self._tmp_directory = None def __enter__(self): return self.create() diff --git a/exasol_integration_test_docker_environment/test/get_test_container_content.py b/exasol_integration_test_docker_environment/test/get_test_container_content.py index 85c0ef561..0f1cd4b72 100644 --- a/exasol_integration_test_docker_environment/test/get_test_container_content.py +++ b/exasol_integration_test_docker_environment/test/get_test_container_content.py @@ -11,7 +11,7 @@ def get_test_container_content(test_container_path: Path = FULL_TEST_CONTAINER_PATH, - runtime_mapping: Tuple[TestContainerRuntimeMapping] = tuple()) \ + runtime_mapping: Tuple[TestContainerRuntimeMapping, ...] = tuple()) \ -> TestContainerContentDescription: return TestContainerContentDescription( docker_file=str(test_container_path / "Dockerfile"), diff --git a/exasol_integration_test_docker_environment/test/test_api_logging.py b/exasol_integration_test_docker_environment/test/test_api_logging.py index 8e8ce6927..1670aaf77 100644 --- a/exasol_integration_test_docker_environment/test/test_api_logging.py +++ b/exasol_integration_test_docker_environment/test/test_api_logging.py @@ -241,14 +241,14 @@ def create_logger_infos(self) -> Dict[str, Dict[str, Any]]: def get_logger_info(self, logger: logging.Logger) -> Dict[str, Any]: logger_info = {} logger_info[LOGGER_STR] = str(logger) - logger_info[LEVEL] = logger.level + logger_info[LEVEL] = logger.level # type: ignore logger_info[LEVEL_NAME] = logging.getLevelName(logger.level) - logger_info[HANDLERS] = list(logger.handlers) + logger_info[HANDLERS] = list(logger.handlers) # type: ignore logger_info[LOGGER_NAME] = logger.name - logger_info[FILTERS] = list(logger.filters) - logger_info[DISABLED] = logger.disabled - logger_info[PROPAGATE] = logger.propagate - logger_info[PARENT] = logger.parent + logger_info[FILTERS] = list(logger.filters) # type: ignore + logger_info[DISABLED] = logger.disabled # type: ignore + logger_info[PROPAGATE] = logger.propagate # type: ignore + logger_info[PARENT] = logger.parent # type: ignore return logger_info diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_test_environment.py index c07f73dc2..7043ec82b 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment.py @@ -106,6 +106,7 @@ def tearDownClass(cls): def _assert_deployment_available(self, environment_info: EnvironmentInfo) -> None: with ContextDockerClient() as docker_client: + assert environment_info.test_container_info test_container = docker_client.containers.get(environment_info.test_container_info.container_name) exit_code, output = test_container.exec_run("cat /test/test.txt") self.assertEqual(exit_code, 0) @@ -113,6 +114,7 @@ def _assert_deployment_available(self, environment_info: EnvironmentInfo) -> Non def _assert_deployment_not_shared(self, environment_info: EnvironmentInfo, temp_path: Path): with ContextDockerClient() as docker_client: + assert environment_info.test_container_info test_container = docker_client.containers.get(environment_info.test_container_info.container_name) exit_code, output = test_container.exec_run("touch /test_target/test_new.txt") self.assertEqual(exit_code, 0) diff --git a/exasol_integration_test_docker_environment/test/test_docker_access_method.py b/exasol_integration_test_docker_environment/test/test_docker_access_method.py index b6adb7a89..259c8e7c9 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_access_method.py +++ b/exasol_integration_test_docker_environment/test/test_docker_access_method.py @@ -1,3 +1,5 @@ +from typing import Dict, Any + import luigi import pytest @@ -14,10 +16,10 @@ def run_task(self): @classmethod def make(cls, method: str) -> 'Testee': - kwargs={"task_class": Testee} + kwargs : Dict[str, Any] = {"task_class": Testee} if method: kwargs["db_os_access"] = method - task = generate_root_task(**kwargs) + task : Testee = generate_root_task(**kwargs) # type: ignore luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") return task diff --git a/exasol_integration_test_docker_environment/test/test_doctor.py b/exasol_integration_test_docker_environment/test/test_doctor.py index add497c86..a6b9927cf 100644 --- a/exasol_integration_test_docker_environment/test/test_doctor.py +++ b/exasol_integration_test_docker_environment/test/test_doctor.py @@ -1,6 +1,7 @@ import os import unittest from contextlib import contextmanager +from email.generator import Generator from typing import ContextManager, Mapping from unittest.mock import patch @@ -13,8 +14,8 @@ ) -@contextmanager -def temporary_env(env_vars) -> ContextManager[Mapping[str, str]]: +@contextmanager #type: ignore +def temporary_env(env_vars) -> Generator: #type: ignore """ Creates a temporary environment, containing the current environment variables. diff --git a/exasol_integration_test_docker_environment/test/test_populate_data.py b/exasol_integration_test_docker_environment/test/test_populate_data.py index e66e4b341..b535db05b 100644 --- a/exasol_integration_test_docker_environment/test/test_populate_data.py +++ b/exasol_integration_test_docker_environment/test/test_populate_data.py @@ -66,11 +66,12 @@ def setUp(self) -> None: def _execute_sql_on_db(self, sql: str) -> str: with ContextDockerClient() as docker_client: print(f"Executing sql on db: '{sql}'") - test_container = docker_client.containers.get(self.environment.environment_info. + environment = self.environment # type: ignore + test_container = docker_client.containers.get(environment.environment_info. test_container_info.container_name) - db_info = self.environment.environment_info.database_info - db_user_name = self.environment.db_username - db_password = self.environment.db_password + db_info = environment.environment_info.database_info + db_user_name = environment.db_username + db_password = environment.db_password cmd = f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.ports.database}' " \ f"-u '{db_user_name}' -p '{db_password}' -sql '{sql}' " \ f"-jdbcparam 'validateservercertificate=0'" diff --git a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py index c1a65c92d..d2f850824 100644 --- a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py +++ b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py @@ -50,7 +50,7 @@ def run_task(self): test_container_content=self.test_container_content ) test_container_future_1 = yield from self.run_dependencies(test_container_task_1) - container_info = test_container_future_1.get_output() # type: ContainerInfo + container_info : ContainerInfo = test_container_future_1.get_output() # type: ignore with ContextDockerClient() as docker_client: container = docker_client.containers.get(container_info.container_name) self.return_object({"container_id": container.id, "image_id": container.image.id}) diff --git a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py index 52a98fa3a..b6c809fa0 100644 --- a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py +++ b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, List, Tuple +from typing import Any, List, Tuple, Optional import click @@ -29,14 +29,14 @@ def is_click_command(obj: Any) -> bool: return isinstance(obj, click.Command) -def defaults_of_click_call(click_call: click.Command) -> List[Tuple[str, Any]]: +def defaults_of_click_call(click_call: click.Command) -> List[Tuple[Optional[str] , Any]]: """ Returns the default values of all None-required parameters of a click-command. """ return [(o.name, adjust_default_value_for_multiple(o)) for o in click_call.params if not o.required] -def param_names_of_click_call(click_call: click.Command) -> List[str]: +def param_names_of_click_call(click_call: click.Command) -> List[Optional[str]]: """ Returns names of all parameters of a click call """ @@ -48,7 +48,7 @@ def get_click_and_api_functions(click_module, api_module) -> Tuple[List[Any], Li click_commands = [c[1] for c in inspect.getmembers(click_module, is_click_command)] # Get all functions in module 'api_module' api_functions = [f[1] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] + if f[1].__cli_function__] # type: ignore return click_commands, api_functions @@ -57,5 +57,5 @@ def get_click_and_api_function_names(click_module, api_module) -> Tuple[List[Any click_command_names = [c[0] for c in inspect.getmembers(click_module, is_click_command)] # Get all function names in module 'api_module' api_function_names = [f[0] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] + if f[1].__cli_function__] # type: ignore return click_command_names, api_function_names diff --git a/exasol_integration_test_docker_environment/testing/api_test_environment.py b/exasol_integration_test_docker_environment/testing/api_test_environment.py index ac2eb95b4..fd4acd4df 100644 --- a/exasol_integration_test_docker_environment/testing/api_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/api_test_environment.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path from sys import stderr -from typing import Dict, Any +from typing import Dict, Any, Optional from exasol_integration_test_docker_environment.lib.api import spawn_test_environment from exasol_integration_test_docker_environment.lib.api import spawn_test_environment_with_test_container @@ -62,7 +62,7 @@ def spawn_docker_test_environment_with_test_container( self, name: str, test_container_content: TestContainerContentDescription, - additional_parameter: Dict[str, Any] = None, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() @@ -84,7 +84,7 @@ def spawn_docker_test_environment_with_test_container( def spawn_docker_test_environment( self, name: str, - additional_parameter: Dict[str, Any] = None, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index 732c892f9..3c86064b5 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -7,7 +7,7 @@ from pathlib import Path import shlex from sys import stderr -from typing import List +from typing import List, Optional from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient @@ -130,7 +130,7 @@ def close(self): except Exception as e: print(e, file=stderr) - def spawn_docker_test_environments(self, name: str, additional_parameter: List[str] = None) \ + def spawn_docker_test_environments(self, name: str, additional_parameter: Optional[List[str]] = None) \ -> SpawnedTestEnvironments: ports = Ports.random_free() on_host_parameter = ExaslctDockerTestEnvironment( @@ -154,7 +154,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s arguments.append(f'--docker-db-image-version "{db_version}"') if additional_parameter: arguments += additional_parameter - arguments = " ".join(arguments) + arguments = " ".join(arguments) #type: ignore command = f"{self.executable} spawn-test-environment {arguments}" completed_process = self.run_command(command, use_flavor_path=False, use_docker_repository=False, @@ -168,7 +168,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s with environment_info_json_path.open() as f: environment_info = EnvironmentInfo.from_json(f.read()) on_host_parameter.environment_info = environment_info - on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) + on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) #type: ignore if "RUN_SLC_TESTS_WITHIN_CONTAINER" in os.environ: slc_test_run_parameter = ExaslctDockerTestEnvironment( name=on_host_parameter.name, @@ -178,7 +178,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: List[s bucketfs_username=on_host_parameter.bucketfs_username, bucketfs_password=on_host_parameter.bucketfs_password, ports=Ports.default_ports, - environment_info=on_host_parameter.completed_process, + environment_info=on_host_parameter.completed_process, # type: ignore completed_process=on_host_parameter.completed_process ) diff --git a/noxfile.py b/noxfile.py index dcdebe0dd..26c475d7e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -68,7 +68,7 @@ def run_minimal_tests(session: nox.Session, db_version: str): "new-itest": ["test_cli_environment.py", "test_db_container_log_thread.py"], "unit": "./test/unit", } - session.run("pytest", minimal_tests["unit"]) + session.run("pytest", minimal_tests["unit"]) #type: ignore for test in minimal_tests["new-itest"]: session.run( "pytest", @@ -104,7 +104,7 @@ def starter_scripts_checksums(session: nox.Session): with session.chdir(start_script_dir): for start_script_entry in start_script_dir.iterdir(): if start_script_entry.is_file(): - sha512 = session.run("sha512sum", start_script_entry.name, silent=True) + sha512 : str = session.run("sha512sum", start_script_entry.name, silent=True) # type: ignore with open( start_script_dir /"checksums" / f"{start_script_entry.name}.sha512sum", "w") as f: f.write(sha512) session.run("git", "add", "starter_scripts/checksums") diff --git a/pyproject.toml b/pyproject.toml index f2e3f34bf..c2b170b6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,5 +57,11 @@ joblib = "^1.4.2" itde = 'exasol_integration_test_docker_environment.main:main' +[[tool.mypy.overrides]] +module = [ "luigi.*", "docker.*", "humanfriendly", "configobj", "toml", "netaddr", "joblib.testing", "networkx", + "fabric", "requests", "pyexasol", "paramiko.ssh_exception", "six", "jsonpickle", "exasol.toolbox.*", + "paramiko", "exasol"] +ignore_missing_imports = true + [tool.pylint.master] fail-under = 6.50 diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 32fa8fdaf..72d5b42d5 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,8 +1,12 @@ import contextlib import io +from contextlib import _GeneratorContextManager + import pytest -from typing import Any, Callable, Dict, Iterator, List, NewType, Optional +from typing import Any, Callable, Dict, Iterator, List, NewType, Optional, Generator + +from typing_extensions import Never from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from test.integration.helpers import normalize_request_name @@ -40,7 +44,7 @@ def api_isolation(request) -> Iterator[ApiTestEnvironment]: utils.close_environments(environment) -CliContextProvider = NewType( +CliContextProvider = NewType( # type: ignore "CliContextProvider", Callable[ [Optional[str], Optional[List[str]]], SpawnedTestEnvironments @@ -62,9 +66,9 @@ def test_case(database): ... """ - @contextlib.contextmanager - def create_context( - name: Optional[str] = None, + @contextlib.contextmanager # type: ignore + def create_context( # type: ignore + name: Optional[str] = None, # type: ignore additional_parameters: Optional[List[str]] = None, ) -> SpawnedTestEnvironments: name = name if name else cli_isolation.name @@ -75,10 +79,10 @@ def create_context( yield spawned utils.close_environments(spawned) - return create_context + return create_context # type: ignore -ApiContextProvider = NewType( +ApiContextProvider = NewType( # type: ignore "ApiContextProvider", Callable[ [Optional[str], Optional[Dict[str, Any]]], @@ -93,7 +97,7 @@ def api_database(api_isolation: ApiTestEnvironment) -> ApiContextProvider: def create_context( name: Optional[str] = None, additional_parameters: Optional[Dict[str, Any]] = None, - ) -> ExaslctDockerTestEnvironment: + ) -> Generator[ExaslctDockerTestEnvironment, None, None]: name = name if name else api_isolation.name spawned = api_isolation.spawn_docker_test_environment( name=name, @@ -102,7 +106,7 @@ def create_context( yield spawned utils.close_environments(spawned) - return create_context + return create_context # type: ignore @pytest.fixture def fabric_stdin(monkeypatch): diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 506f4eec3..2baae5c4c 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -44,6 +44,7 @@ def get_executor_factory( if db_os_access == DbOsAccess.SSH: return SshExecFactory.from_database_info(dbinfo) client_factory = DockerClientFactory(timeout=100000) + assert dbinfo.container_info return DockerExecFactory(dbinfo.container_info.container_name, client_factory) diff --git a/test/integration/test_cli_environment.py b/test/integration/test_cli_environment.py index 1eb16caa9..48f577e32 100644 --- a/test/integration/test_cli_environment.py +++ b/test/integration/test_cli_environment.py @@ -27,6 +27,7 @@ def count(self, selected: Optional[List[str]] = None): @property def log(self) -> str: + assert self.db.on_host_docker_environment.completed_process return ( self .db @@ -50,6 +51,7 @@ def smoke_test_sql(exaplus_path: str, env: ExaslctDockerTestEnvironment) -> str: def quote(s): return f"'{s}'" + assert env.environment_info db_info = env.environment_info.database_info command = [ str(exaplus_path), @@ -63,7 +65,7 @@ def quote(s): "-jdbcparam", "validateservercertificate=0", ] - command = " ".join(command) + command = " ".join(command) #type: ignore return f'bash -c "{command}" ' diff --git a/test/integration/test_db_container_log_thread.py b/test/integration/test_db_container_log_thread.py index 32093f75c..f398b9fb7 100644 --- a/test/integration/test_db_container_log_thread.py +++ b/test/integration/test_db_container_log_thread.py @@ -2,7 +2,7 @@ import tempfile import time from pathlib import Path -from typing import List +from typing import List, Optional import pytest @@ -24,7 +24,7 @@ def _build_docker_command(logs: List[str]): bash_command = f"while true; do {echo_commdands_str}; sleep 0.1; done" return ["bash", "-c", bash_command] -def _run_container_log_thread(logger, logs: List[str]) -> str: +def _run_container_log_thread(logger, logs: List[str]) -> Optional[str]: """ Starts a dummy docker container which prints logs in an endless loop, and calls DBContainerLogThread on that container. @@ -60,7 +60,7 @@ def test_container_log_thread_error(test_logger) -> None: Integration test which verifies that the DBContainerLogThread returns error message if error is logged. """ error_message = _run_container_log_thread(test_logger, ["confd returned with state 1"]) - assert "confd returned with state 1\n" in error_message + assert error_message and "confd returned with state 1\n" in error_message def test_container_log_thread_ignore_rsyslogd(test_logger) -> None: """ @@ -96,4 +96,4 @@ def test_container_log_thread_exception(test_logger) -> None: 'Exception: bad thing happend' ] error_message = _run_container_log_thread(test_logger, sshd_logs) - assert "exception: bad thing happend\n" in error_message + assert error_message and "exception: bad thing happend\n" in error_message diff --git a/test/unit/test_env_variable.py b/test/unit/test_env_variable.py index da33a3799..d536123dc 100644 --- a/test/unit/test_env_variable.py +++ b/test/unit/test_env_variable.py @@ -41,7 +41,7 @@ class LogPathCorrectnessMatcher: def __init__(self, expected_log_path: Path): self.expected_log_path = expected_log_path - def __eq__(self, used_log_path: UsedLogPath): + def __eq__(self, used_log_path: object): if not isinstance(used_log_path, UsedLogPath): return False log_path = used_log_path.log_path From cc1bd69daf2536c9c1ee70e70fafb5095eb357bf Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:43:28 -0300 Subject: [PATCH 02/60] Activated CI type checks --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6cc907772..2c873f7ab 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -88,7 +88,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run type-check - run: poetry run nox -s lint:typing || echo ignoring... + run: poetry run nox -s lint:typing Security: name: Security Checks (Python-${{ matrix.python-version }}) @@ -109,7 +109,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run security linter - run: poetry run nox -s lint:security || echo ignoring... + run: poetry run nox -s lint:security - name: Upload Artifacts uses: actions/upload-artifact@v4.4.0 From 072f39487e6bb2296ca66ebaf16e215ae43826cd Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:44:25 -0300 Subject: [PATCH 03/60] Updared changelog --- doc/changes/unreleased.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 01bae738c..252ba4848 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -9,4 +9,5 @@ Code name: * #119: Refactored `pkg_resources` usage to `importlib.resources` * #420: Added file `py.typed` to enable mypy to find project specific types * #418: Use exasol/python-toolbox -* #411: Removed usage of exasol-bucketfs +* #411: Removed usage of exasol-bucketfs +* #425: Fixed type checks found by MyPy From c807369ddabffaac5433b90dfe7599f401a795db Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:48:08 -0300 Subject: [PATCH 04/60] Fixed build_config.py --- .../lib/config/build_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol_integration_test_docker_environment/lib/config/build_config.py b/exasol_integration_test_docker_environment/lib/config/build_config.py index 52dd43f81..d1076f324 100644 --- a/exasol_integration_test_docker_environment/lib/config/build_config.py +++ b/exasol_integration_test_docker_environment/lib/config/build_config.py @@ -1,4 +1,5 @@ import luigi +from typing import List, Optional from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY From e44b13464b8af15b6386ce2e447dc64ff1a6b504 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:52:43 -0300 Subject: [PATCH 05/60] Fixed spawn_test_container.py --- .../lib/test_environment/spawn_test_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index 48ca628dc..658fb7cbf 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -46,7 +46,7 @@ def register_required(self): self.register_dependency(self.create_child_task(task_class=DockerTestContainerBuild, test_container_content=self.test_container_content)) - def is_reuse_possible(self) -> luigi.BoolParameter | bool: + def is_reuse_possible(self) -> bool: test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ImageInfo test_container = None From ec7117eb6c410f8663d74946657abb74ccdd9050 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:03:01 -0300 Subject: [PATCH 06/60] Fixed type error for protocol class --- .../lib/base/db_os_executor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py index d6f444d13..50d8463a9 100644 --- a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py +++ b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py @@ -37,12 +37,11 @@ class DbOsExecutor(Protocol): concrete implementations in sub-classes ``DockerExecutor`` and ``SshExecutor``. """ - @abstractmethod def exec(self, cmd: str) -> ExecResult: ... def prepare(self): - pass + ... class DockerExecutor(DbOsExecutor): @@ -65,6 +64,9 @@ def exec(self, cmd: str) -> ExecResult: assert self._container return self._container.exec_run(cmd) + def prepare(self): + pass + def close(self): self._container = None if self._client is not None: @@ -145,6 +147,7 @@ def executor(self) -> DbOsExecutor: return DockerExecutor(client, self._container_name) + class SshExecFactory(DbOsExecFactory): @classmethod def from_database_info(cls, info: DatabaseInfo): From 73cc9031df1ca05724f63f7781873d24c2e70ba7 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:07:40 -0300 Subject: [PATCH 07/60] Fixed imports --- .../lib/base/base_task.py | 2 +- .../lib/test_environment/spawn_test_container.py | 2 +- test/integration/conftest.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/base_task.py b/exasol_integration_test_docker_environment/lib/base/base_task.py index 028ef0f5f..42ff817cb 100644 --- a/exasol_integration_test_docker_environment/lib/base/base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/base_task.py @@ -3,7 +3,7 @@ import logging import shutil from pathlib import Path -from typing import Dict, List, Generator, Any, Union, Set, Optional +from typing import Dict, List, Generator, Any, Union, Set import luigi import six diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index 658fb7cbf..e182c0682 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import luigi import netaddr diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 72d5b42d5..bb2d23870 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,6 +1,5 @@ import contextlib import io -from contextlib import _GeneratorContextManager import pytest From b928e9e73e8394147ed1335ff3da71d0fc3e3db7 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:11:44 -0300 Subject: [PATCH 08/60] Fixed import --- .../lib/test_environment/prepare_network_for_test_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py index 90ce2f47e..a906573d3 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py @@ -1,3 +1,4 @@ +from typing import Optional import docker import luigi From 2a397a348190d36a588c9d431884f7a6262f2a4a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:13:28 -0300 Subject: [PATCH 09/60] [run all tests] From 97e834009cfe536b5c349b38fe903f5ae9b6172e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:53:54 -0300 Subject: [PATCH 10/60] Fixed docker_image_analyze_task.py --- .../lib/docker/images/create/docker_image_analyze_task.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index e06eb482e..8eace9b67 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -1,6 +1,6 @@ import abc from pathlib import Path -from typing import Dict, Type +from typing import Dict, Type, Optional import docker import git @@ -137,8 +137,7 @@ def values_are_subclass_of_baseclass(self, task_classes) -> bool: return all(issubclass(value, DockerAnalyzeImageTask) for value in task_classes.values()) - @abc.abstractmethod - def requires_tasks(self) -> Dict[str, Type["DockerAnalyzeImageTask"]]: + def requires_tasks(self) -> Optional[Dict[str, Type["DockerAnalyzeImageTask"]]]: pass def run_task(self): From a33a9adfa8185f5c5f802055f01cb88ed67a12dd Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:53:57 -0300 Subject: [PATCH 11/60] [run all tests] From 576ce4e9736744b18812bf159c4e97bd7feb8ce3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:55:36 -0300 Subject: [PATCH 12/60] Changed order in common.py --- .../lib/api/common.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index 97e170fc7..bff85d6f7 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -116,18 +116,6 @@ def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: return task_class(**params) -def _run_task_with_logging_config( - task: DependencyLoggerBaseTask, - log_file_path: Path, - log_level: Optional[str], - use_job_specific_log_file: bool, - workers: int) -> bool: - with _configure_logging(log_file_path, log_level, use_job_specific_log_file) as run_kwargs: - no_scheduling_errors = luigi.build([task], workers=workers, - local_scheduler=True, **run_kwargs) - return no_scheduling_errors - - def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int = 2, task_dependencies_dot_file: Optional[str] = None, log_level: Optional[str] = None, use_job_specific_log_file: bool = False) \ @@ -152,6 +140,16 @@ def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int f"The detailed log of the integration-test-docker-environment can be found at: {log_file_path}") task.cleanup(success) +def _run_task_with_logging_config( + task: DependencyLoggerBaseTask, + log_file_path: Path, + log_level: Optional[str], + use_job_specific_log_file: bool, + workers: int) -> bool: + with _configure_logging(log_file_path, log_level, use_job_specific_log_file) as run_kwargs: + no_scheduling_errors = luigi.build([task], workers=workers, + local_scheduler=True, **run_kwargs) + return no_scheduling_errors def _handle_task_result( no_scheduling_errors: bool, From 7a05968340a39a1800eb69d54e11c2b7a6e398b3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:55:39 -0300 Subject: [PATCH 13/60] [run all tests] From 28e994c1bb5a86372e622b3cc5b459daed20cb28 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:14:42 -0300 Subject: [PATCH 14/60] Fixed conversion of ints in shell_variables.py --- .../lib/test_environment/shell_variables.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index ee9e76926..e3959a488 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -24,9 +24,9 @@ def from_test_environment_info( "NAME": info.name, "TYPE": info.type, "DATABASE_HOST": info.database_info.host, - "DATABASE_DB_PORT": str(info.database_info.ports.database) if info.database_info.ports.database else "", - "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs) if info.database_info.ports.bucketfs else "", - "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh else "", + "DATABASE_DB_PORT": str(info.database_info.ports.database) if info.database_info.ports.database is not None else "", + "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs) if info.database_info.ports.bucketfs is not None else "", + "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh is not None else "", } if info.database_info.container_info is not None: network_aliases = " ".join(info.database_info.container_info.network_aliases) From 2ce0296bc28ead3da7edd5d32361c4150a483e9a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:14:46 -0300 Subject: [PATCH 15/60] [run all tests] From d7f85e05cc5b1582bee92aa87089ab9167a13e56 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:19:38 -0300 Subject: [PATCH 16/60] Fixed import in test_doctor.py --- .../test/test_doctor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/test/test_doctor.py b/exasol_integration_test_docker_environment/test/test_doctor.py index a6b9927cf..a17cf230e 100644 --- a/exasol_integration_test_docker_environment/test/test_doctor.py +++ b/exasol_integration_test_docker_environment/test/test_doctor.py @@ -1,8 +1,8 @@ import os import unittest from contextlib import contextmanager -from email.generator import Generator -from typing import ContextManager, Mapping + +from typing import Generator from unittest.mock import patch from exasol_integration_test_docker_environment.doctor import ( From 55f3d6f96daca408f934dae120d828daebf9c295 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:19:41 -0300 Subject: [PATCH 17/60] [run all tests] From 7ff8ffdb1067ac69078f456034398e98fc6691bc Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:32:53 -0300 Subject: [PATCH 18/60] Added type hints to luigi config classes --- .../lib/base/flavor_task.py | 2 +- .../docker_db_test_environment_parameter.py | 23 ++++++++++--------- ...eneral_spawn_test_environment_parameter.py | 17 +++++++------- .../test_environment/spawn_test_database.py | 4 ++-- .../test/test_common_run_task.py | 2 -- .../test/test_docker_build_base.py | 4 ++-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/flavor_task.py b/exasol_integration_test_docker_environment/lib/base/flavor_task.py index 67f15f4db..940f00b44 100644 --- a/exasol_integration_test_docker_environment/lib/base/flavor_task.py +++ b/exasol_integration_test_docker_environment/lib/base/flavor_task.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, List import luigi diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py index 1e72e8c67..738c7a281 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py @@ -1,6 +1,7 @@ import luigi from enum import Enum, auto from luigi import Config +from typing import List, Optional class DbOsAccess(Enum): """ @@ -19,15 +20,15 @@ class DbOsAccess(Enum): class DockerDBTestEnvironmentParameter(Config): - docker_db_image_name = luigi.OptionalParameter(None) - docker_db_image_version = luigi.OptionalParameter(None) - reuse_database = luigi.BoolParameter(False, significant=False) + docker_db_image_name : Optional[str] = luigi.OptionalParameter(None) # type: ignore + docker_db_image_version : Optional[str] = luigi.OptionalParameter(None) # type: ignore + reuse_database : bool = luigi.BoolParameter(False, significant=False) # type: ignore db_os_access = luigi.EnumParameter(DbOsAccess.DOCKER_EXEC, enum=DbOsAccess, significant=False) - no_database_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_database_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - database_port_forward = luigi.OptionalParameter(None, significant=False) - bucketfs_port_forward = luigi.OptionalParameter(None, significant=False) - ssh_port_forward = luigi.OptionalParameter(None, significant=False) - mem_size = luigi.OptionalParameter("2 GiB", significant=False) - disk_size = luigi.OptionalParameter("2 GiB", significant=False) - nameservers = luigi.ListParameter([], significant=False) + no_database_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_database_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore + database_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + bucketfs_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + ssh_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + mem_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + disk_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + nameservers : List[str] = luigi.ListParameter([], significant=False) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py index 8473b869b..f4917a3ff 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py @@ -1,16 +1,17 @@ import luigi from luigi import Config +from typing import List, Optional from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ OptionalTestContainerParameter class GeneralSpawnTestEnvironmentParameter(OptionalTestContainerParameter): - reuse_database_setup = luigi.BoolParameter(False, significant=False) - reuse_test_container = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_success = luigi.BoolParameter(False, significant=False) - no_test_container_cleanup_after_failure = luigi.BoolParameter(False, significant=False) - max_start_attempts = luigi.IntParameter(2, significant=False) - docker_runtime = luigi.OptionalParameter(None, significant=False) - create_certificates = luigi.BoolParameter() - additional_db_parameter = luigi.ListParameter() + reuse_database_setup : bool = luigi.BoolParameter(False, significant=False) #type: ignore + reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore + no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore + max_start_attempts : int = luigi.IntParameter(2, significant=False) #type: ignore + docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore + create_certificates : bool = luigi.BoolParameter() #type: ignore + additional_db_parameter : List[str] = luigi.ListParameter() #type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 364b7d484..5eb9333bc 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -1,5 +1,5 @@ import math -from typing import Optional, Tuple +from typing import Optional, Tuple, List import docker import humanfriendly @@ -56,7 +56,7 @@ class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) # type: ignore docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - additional_db_parameter = luigi.ListParameter() + additional_db_parameter : List[str] = luigi.ListParameter() # type: ignore ssh_user : str = luigi.Parameter("root") # type: ignore ssh_key_file : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore diff --git a/exasol_integration_test_docker_environment/test/test_common_run_task.py b/exasol_integration_test_docker_environment/test/test_common_run_task.py index 4f599aaaf..04422dd23 100644 --- a/exasol_integration_test_docker_environment/test/test_common_run_task.py +++ b/exasol_integration_test_docker_environment/test/test_common_run_task.py @@ -1,6 +1,4 @@ -import subprocess import unittest -from pathlib import Path import luigi diff --git a/exasol_integration_test_docker_environment/test/test_docker_build_base.py b/exasol_integration_test_docker_environment/test/test_docker_build_base.py index 9fe45918b..6935eda24 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_build_base.py +++ b/exasol_integration_test_docker_environment/test/test_docker_build_base.py @@ -5,7 +5,7 @@ import luigi from luigi import Parameter -from typing import Set, Dict +from typing import Set, Dict, List from exasol_integration_test_docker_environment.lib.api.common import generate_root_task from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient @@ -44,7 +44,7 @@ def is_rebuild_requested(self) -> bool: class TestDockerBuildBase(DockerBuildBase): - goals = luigi.ListParameter([]) + goals : List[str] = luigi.ListParameter([]) # type: ignore def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: goal_class_map = { From 6090b9249e07bd8f0f0426d8913ee5b3c22f26da Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:32:56 -0300 Subject: [PATCH 19/60] [run all tests] From f2b18e7c63a3424f6beb1c53a7bb7c6a1caa52b2 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:10:24 -0300 Subject: [PATCH 20/60] Integrated suggestsions from review --- .../spawn_test_environment_with_test_container.py | 15 ++++++++------- .../lib/test_environment/ports.py | 8 +++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py index 9a9057b98..c75a87ada 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py @@ -22,14 +22,15 @@ .docker_db_test_environment_parameter import DbOsAccess def _cleanup(environment_info: EnvironmentInfo) -> None: - if environment_info.test_container_info is not None: - remove_docker_container([environment_info.test_container_info.container_name]) - if environment_info.database_info.container_info is not None: - remove_docker_container([environment_info.database_info.container_info.container_name]) - if environment_info.database_info.container_info.volume_name is not None: - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) - remove_docker_networks([environment_info.network_info.network_name]) + if test_container_info := environment_info.test_container_info: + remove_docker_container([test_container_info.container_name]) + + if db_container_info := environment_info.database_info.container_info: + remove_docker_container([db_container_info.container_name]) + if name := db_container_info.volume_name: + remove_docker_volumes([name]) + remove_docker_networks([environment_info.network_info.network_name]) @no_cli_function def spawn_test_environment_with_test_container( diff --git a/exasol_integration_test_docker_environment/lib/test_environment/ports.py b/exasol_integration_test_docker_environment/lib/test_environment/ports.py index 25f3d1f32..92afa5665 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/ports.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/ports.py @@ -55,8 +55,6 @@ def __init__(self, database: int, bucketfs: int, ssh: Optional[int] = None): @classmethod def random_free(cls, ssh: bool = True) -> 'Ports': - ports : List[int] = find_free_ports(3 if ssh else 2) - if ssh: - return Ports(ports[0], ports[1], ports[2]) - else: - return Ports(ports[0], ports[1], None) + count = 3 if ssh else 2 + ports = find_free_ports(count) + return Ports(ports[0], ports[1], None) if not ssh else Ports(ports[0], ports[1], ports[2]) From 7bc8cc3917c2bcc16cc6bb19970eb57508fd1f26 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:10:28 -0300 Subject: [PATCH 21/60] [run all tests] From ff0f0d2ab90455e9c3e5f7c0a84db32b4158b74e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:20:28 -0300 Subject: [PATCH 22/60] Fixes from review --- exasol_integration_test_docker_environment/doctor.py | 11 +++++++---- .../lib/api/common.py | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/exasol_integration_test_docker_environment/doctor.py b/exasol_integration_test_docker_environment/doctor.py index 5640a8006..4fe8bc21b 100644 --- a/exasol_integration_test_docker_environment/doctor.py +++ b/exasol_integration_test_docker_environment/doctor.py @@ -3,7 +3,8 @@ package and also provide help to find potential fixes. """ import sys -from typing import Iterable +from collections.abc import Callable +from typing import Iterable, List, Tuple from exasol import error from enum import Enum @@ -78,11 +79,13 @@ def health_checkup() -> Iterable[error.ExaError]: return an iterator of error codes specifying which problems have been identified. """ - examinations = [ + check_function = Callable[[],bool] + diagnosis_function = Callable[[],Iterable[error.ExaError]] + examinations : List[Tuple[check_function, diagnosis_function]] = [ (is_docker_daemon_available, diagnose_docker_daemon_not_available), - (is_supported_platform, lambda: Error.TargetPlatformNotSupported), + (is_supported_platform, lambda: [Error.TargetPlatformNotSupported]), ] for is_fine, diagnosis in examinations: if not is_fine(): - for error_code in diagnosis(): # type: ignore + for error_code in diagnosis(): yield error_code diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index bff85d6f7..6d93671da 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -104,8 +104,9 @@ def import_build_steps(flavor_path: Tuple[str, ...]): path_to_build_steps = Path(path).joinpath("flavor_base/build_steps.py") module_name_for_build_steps = extract_modulename_for_build_steps(path) spec = importlib.util.spec_from_file_location(module_name_for_build_steps, path_to_build_steps) - module = importlib.util.module_from_spec(spec) # type: ignore - spec.loader.exec_module(module) # type: ignore + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: @@ -178,7 +179,7 @@ def _configure_logging( use_job_specific_log_file: bool) -> Iterator[Dict[str, str]]: with get_luigi_log_config(log_file_target=log_file_path, log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) as luigi_config: # type: Path + use_job_specific_log_file=use_job_specific_log_file) as luigi_config: no_configure_logging, run_kwargs = _configure_logging_parameter( log_level=log_level, luigi_config=luigi_config, From c9761fe86b98d9cbb8b1c35f5a0499841baae8c8 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:20:32 -0300 Subject: [PATCH 23/60] [run all tests] From e76c3acf0da0fb135d02ff14c9561bdc5c07c806 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:46:46 -0300 Subject: [PATCH 24/60] Fixes from review --- .../lib/base/base_task.py | 2 +- .../lib/base/luigi_log_config.py | 26 ++++++++++-------- .../create/docker_image_analyze_task.py | 4 +-- .../lib/docker/networks/utils.py | 4 +-- .../abstract_spawn_test_environment.py | 27 ++++++++++--------- test/integration/conftest.py | 2 +- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/base_task.py b/exasol_integration_test_docker_environment/lib/base/base_task.py index 42ff817cb..ed840cd01 100644 --- a/exasol_integration_test_docker_environment/lib/base/base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/base_task.py @@ -153,7 +153,7 @@ def get_output_path(self) -> Path: def _get_output_path_for_job(self) -> Path: return Path(build_config().output_directory, - "jobs", self.job_id) # type: ignore + "jobs", self.job_id) def _extend_output_path(self): extension = self.extend_output_path() diff --git a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py index 8022f6244..359d1fbdd 100644 --- a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py +++ b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py @@ -2,10 +2,12 @@ import os import tempfile from pathlib import Path -from typing import Optional, Callable, Generator +from typing import Optional, Callable, Generator, List, Any +from dataclasses import dataclass import jinja2 import logging + from exasol_integration_test_docker_environment.lib import PACKAGE_NAME from exasol_integration_test_docker_environment.lib.config.build_config import build_config @@ -40,22 +42,24 @@ def get_log_path(job_id: str) -> Path: log_path_dir.mkdir(parents=True, exist_ok=True) return log_path +@dataclass +class LogInfoStorage: + level : int + handlers: List[logging.Handler] + filters: List[Any] + propagate: bool @contextlib.contextmanager def restore_logger(logger_creator: Callable[[], logging.Logger]): before_logger = logger_creator() - logger_info = { - LOG_LEVEL: before_logger.level, - HANDLERS: list(before_logger.handlers), - FILTERS: list(before_logger.filters), - PROPAGATE: before_logger.propagate - } + logger_info_before = LogInfoStorage(level=before_logger.level, handlers=list(before_logger.handlers), + filters=list(before_logger.filters), propagate=before_logger.propagate) yield after_logger = logger_creator() - after_logger.level = logger_info[LOG_LEVEL] # type: ignore - after_logger.handlers = logger_info[HANDLERS] # type: ignore - after_logger.filters = logger_info[FILTERS] # type: ignore - after_logger.propagate = logger_info[PROPAGATE] # type: ignore + after_logger.level = logger_info_before.level + after_logger.handlers = logger_info_before.handlers + after_logger.filters = logger_info_before.filters + after_logger.propagate = logger_info_before.propagate @contextlib.contextmanager diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index 8eace9b67..376222075 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -137,8 +137,8 @@ def values_are_subclass_of_baseclass(self, task_classes) -> bool: return all(issubclass(value, DockerAnalyzeImageTask) for value in task_classes.values()) - def requires_tasks(self) -> Optional[Dict[str, Type["DockerAnalyzeImageTask"]]]: - pass + def requires_tasks(self) -> Dict[str, Type["DockerAnalyzeImageTask"]]: + return dict() def run_task(self): image_info_of_dependencies = self.get_values_from_futures(self.dependencies_futures) diff --git a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py index 8b8adf5ed..03d4bea7c 100644 --- a/exasol_integration_test_docker_environment/lib/docker/networks/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/networks/utils.py @@ -1,10 +1,10 @@ import logging -from typing import Iterator, List +from typing import Iterable from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -def remove_docker_networks(networks: List[str]): +def remove_docker_networks(networks: Iterable[str]): """ Removes the given networks using docker API. """ diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index af70e8a4a..aa62a4c5a 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -68,7 +68,8 @@ def create_test_environment_info_in_test_container( shell_variables: ShellVariables, json: str, ): - test_container_name = test_environment_info.test_container_info.container_name # type: ignore + assert test_environment_info.test_container_info + test_container_name = test_environment_info.test_container_info.container_name with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(test_container_name) self.logger.info(f"Create test environment info in test container '{test_container_name}' at '/'") @@ -102,31 +103,31 @@ def create_test_environment_info_in_test_container_and_on_host( json, ) - def _default_bridge_ip_address(self, test_environment_info) -> Optional[str]: - if test_environment_info.database_info.container_info is None: - return None - container_name = test_environment_info.database_info.container_info.container_name - with self._get_docker_client() as docker_client: - db_container = docker_client.containers.get(container_name) - return default_bridge_ip_address(db_container) + def _default_bridge_ip_address(self, test_environment_info) -> str: + if test_environment_info.database_info.container_info is not None: + container_name = test_environment_info.database_info.container_info.container_name + with self._get_docker_client() as docker_client: + db_container = docker_client.containers.get(container_name) + return default_bridge_ip_address(db_container) + raise RuntimeError("Could not find default bridge ip address") def collect_shell_variables(self, test_environment_info) -> ShellVariables: return ShellVariables.from_test_environment_info( - self._default_bridge_ip_address(test_environment_info), # type: ignore + self._default_bridge_ip_address(test_environment_info), test_environment_info, ) def _start_database(self, attempt) \ -> Generator[BaseTask, BaseTask, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: - network_info = yield from self._create_network(attempt) # type: ignore - ssl_volume_info = None # type: ignore - if self.create_certificates: # type: ignore + network_info = yield from self._create_network(attempt) + ssl_volume_info = None + if self.create_certificates: ssl_volume_info = yield from self._create_ssl_certificates() # type: ignore database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) # type: ignore is_database_ready = yield from self._wait_for_database(database_info, attempt) # type: ignore return network_info, database_info, is_database_ready, test_container_info # type: ignore - def _create_ssl_certificates(self) -> DockerVolumeInfo: # type: ignore + def _create_ssl_certificates(self) -> Generator: ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) ssl_info = self.get_values_from_future(ssl_info_future) return ssl_info # type: ignore diff --git a/test/integration/conftest.py b/test/integration/conftest.py index bb2d23870..096b6337b 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -67,7 +67,7 @@ def test_case(database): @contextlib.contextmanager # type: ignore def create_context( # type: ignore - name: Optional[str] = None, # type: ignore + name: Optional[str] = None, additional_parameters: Optional[List[str]] = None, ) -> SpawnedTestEnvironments: name = name if name else cli_isolation.name From 79e533e77629ebed0da1eb9991c3ff7c48881a74 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:46:53 -0300 Subject: [PATCH 25/60] [run all tests] From c735fd4488ca5f2d8e36b288976ca3f5afbc260c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 06:15:34 -0300 Subject: [PATCH 26/60] [run all tests] From 40697ae8a4670347368e8a6307a90af88304796d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 06:17:04 -0300 Subject: [PATCH 27/60] Fixed AbstractSpawnTestEnvironment._default_bridge_ip_address() Return optional. In function ShellVariables.from_test_environment_info() parameter default_bridge_ip_address now also accepts an optional string. Only if the database container is valid, the parameter default_bridge_ip_address must not be None. --- .../lib/test_environment/abstract_spawn_test_environment.py | 4 ++-- .../lib/test_environment/shell_variables.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index aa62a4c5a..9ab3587ab 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -103,13 +103,13 @@ def create_test_environment_info_in_test_container_and_on_host( json, ) - def _default_bridge_ip_address(self, test_environment_info) -> str: + def _default_bridge_ip_address(self, test_environment_info) -> Optional[str]: if test_environment_info.database_info.container_info is not None: container_name = test_environment_info.database_info.container_info.container_name with self._get_docker_client() as docker_client: db_container = docker_client.containers.get(container_name) return default_bridge_ip_address(db_container) - raise RuntimeError("Could not find default bridge ip address") + return None def collect_shell_variables(self, test_environment_info) -> ShellVariables: return ShellVariables.from_test_environment_info( diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index e3959a488..b28a6e301 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Optional from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo @@ -12,7 +12,7 @@ def __init__(self, env: Dict[str, str]): @classmethod def from_test_environment_info( cls, - default_bridge_ip_address: str, + default_bridge_ip_address: Optional[str], test_environment_info: EnvironmentInfo, ) -> 'ShellVariables': """ @@ -29,6 +29,7 @@ def from_test_environment_info( "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh is not None else "", } if info.database_info.container_info is not None: + assert default_bridge_ip_address network_aliases = " ".join(info.database_info.container_info.network_aliases) env.update({ "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, From bf0aa0b7de770a217ad48a29d54aea9fe4440a51 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 06:17:06 -0300 Subject: [PATCH 28/60] [run all tests] From 4a3e4fddcc2e33ce50ae9a18eb11cab355857362 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:51:50 -0300 Subject: [PATCH 29/60] Fixed Generator return types --- .../lib/test_environment/abstract_spawn_test_environment.py | 4 ++-- .../create_certificates/create_ssl_certificates_task.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index 9ab3587ab..607c2ffed 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -118,7 +118,7 @@ def collect_shell_variables(self, test_environment_info) -> ShellVariables: ) def _start_database(self, attempt) \ - -> Generator[BaseTask, BaseTask, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: + -> Generator[BaseTask, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: network_info = yield from self._create_network(attempt) ssl_volume_info = None if self.create_certificates: @@ -127,7 +127,7 @@ def _start_database(self, attempt) \ is_database_ready = yield from self._wait_for_database(database_info, attempt) # type: ignore return network_info, database_info, is_database_ready, test_container_info # type: ignore - def _create_ssl_certificates(self) -> Generator: + def _create_ssl_certificates(self) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) ssl_info = self.get_values_from_future(ssl_info_future) return ssl_info # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py index edc5bbfb4..30e46b560 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py @@ -56,7 +56,7 @@ def run_task(self): self.return_object(self.volume_info) - def build_image(self) -> Generator["BaseTask", PickleTarget, Set[str]]: + def build_image(self) -> Generator[BaseTask, None, Set[str]]: task = self.create_child_task(task_class=DockerCertificateContainerBuild, certificate_container_root_directory=self._temp_resource_directory.tmp_directory) image_infos_future = yield from self.run_dependencies(task) From e8a4e813e4a9406256625d48d5f53b72d3df2d8d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:48:25 -0300 Subject: [PATCH 30/60] Fixes from review findings --- .../setup_external_database_host.py | 5 ++-- .../lib/test_environment/shell_variables.py | 10 +++++--- .../test_environment/spawn_test_container.py | 25 ++++++++++--------- .../test_environment/spawn_test_database.py | 5 ++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py index 78ecd7354..0140ffe02 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py @@ -59,9 +59,10 @@ def setup_database(self): self.logger.info(e) def get_xml_rpc_object(self, object_name: str = ""): + assert self.external_exasol_xmlrpc_user and self.external_exasol_xmlrpc_password uri = 'https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}'.format( - user=quote_plus(self.external_exasol_xmlrpc_user or ""), - password=quote_plus(self.external_exasol_xmlrpc_password or ""), + user=quote_plus(self.external_exasol_xmlrpc_user), + password=quote_plus(self.external_exasol_xmlrpc_password), host=self.external_exasol_xmlrpc_host, port=self.external_exasol_xmlrpc_port, cluster_name=self.external_exasol_xmlrpc_cluster_name, diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index b28a6e301..a94db5563 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -20,22 +20,26 @@ def from_test_environment_info( default_bridge_ip_address and EnvironmentInfo. """ info = test_environment_info + assert info.database_info.ports.database is not None + assert info.database_info.ports.bucketfs is not None env : Dict[str, str] = { "NAME": info.name, "TYPE": info.type, "DATABASE_HOST": info.database_info.host, - "DATABASE_DB_PORT": str(info.database_info.ports.database) if info.database_info.ports.database is not None else "", - "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs) if info.database_info.ports.bucketfs is not None else "", + "DATABASE_DB_PORT": str(info.database_info.ports.database), + "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs), "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh is not None else "", } + if info.database_info.container_info is not None: + assert info.database_info.container_info.volume_name assert default_bridge_ip_address network_aliases = " ".join(info.database_info.container_info.network_aliases) env.update({ "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, "DATABASE_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', "DATABASE_CONTAINER_IP_ADDRESS": info.database_info.container_info.ip_address, - "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name or "", + "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name, "DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS": default_bridge_ip_address, }) if info.test_container_info is not None: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index e182c0682..adb65b406 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -1,4 +1,5 @@ -from typing import List, Optional +from pathlib import Path +from typing import List, Optional, Dict import luigi import netaddr @@ -36,10 +37,10 @@ class SpawnTestContainer(DockerBaseTask, TestContainerParameter): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.ip_address_index_in_subnet < 0: # type: ignore + if self.ip_address_index_in_subnet < 0: raise Exception( "ip_address_index_in_subnet needs to be greater than 0 got %s" - % self.ip_address_index_in_subnet) # type: ignore + % self.ip_address_index_in_subnet) def register_required(self): self.test_container_image_future = \ @@ -47,8 +48,8 @@ def register_required(self): test_container_content=self.test_container_content)) def is_reuse_possible(self) -> bool: - test_container_image_info = \ - self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ImageInfo + test_container_image_info : ImageInfo = \ + self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ignore test_container = None with self._get_docker_client() as docker_client: try: @@ -64,7 +65,7 @@ def is_reuse_possible(self) -> bool: def run_task(self): subnet = netaddr.IPNetwork(self.network_info.subnet) - ip_address = str(subnet[2 + self.ip_address_index_in_subnet]) # type: ignore + ip_address = str(subnet[2 + self.ip_address_index_in_subnet]) container_info = None if self.is_reuse_possible(): @@ -91,7 +92,7 @@ def _copy_runtime_targets(self): test_container.exec_run(cmd=f"cp -r {runtime_mapping.target} {runtime_mapping.deployment_target}") def _try_to_reuse_test_container(self, ip_address: str, - network_info: DockerNetworkInfo) -> ContainerInfo: + network_info: DockerNetworkInfo) -> Optional[ContainerInfo]: self.logger.info("Try to reuse test container %s", self.test_container_name) container_info = None @@ -101,17 +102,17 @@ def _try_to_reuse_test_container(self, ip_address: str, except Exception as e: self.logger.warning("Tried to reuse test container %s, but got Exeception %s. " "Fallback to create new database.", self.test_container_name, e) - return container_info # type: ignore + return container_info def _create_test_container(self, ip_address, network_info: DockerNetworkInfo) -> ContainerInfo: - self._remove_container(self.test_container_name) # type: ignore + self._remove_container(self.test_container_name) self.logger.info(f"Creating new test container {self.test_container_name}") test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] - volumes = dict() - for runtime_mapping in self.test_container_content.runtime_mappings: # type: ignore + volumes : Dict[str | Path, Dict[str, str]] = dict() + for runtime_mapping in self.test_container_content.runtime_mappings: volumes[runtime_mapping.source.absolute()] = { "bind": runtime_mapping.target, "mode": "rw" @@ -160,7 +161,7 @@ def create_container_info(self, ip_address: str, network_aliases: List[str], test_container = docker_client.containers.get(self.test_container_name) if test_container.status != "running": raise Exception(f"Container {self.test_container_name} not running") - container_info = ContainerInfo(container_name=self.test_container_name, # type: ignore + container_info = ContainerInfo(container_name=self.test_container_name, ip_address=ip_address, network_aliases=network_aliases, network_info=network_info) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 5eb9333bc..40bffc13f 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -89,15 +89,16 @@ def run_task(self): database_info = self._create_database_container(db_ip_address, db_private_network) self.return_object(database_info) - def _try_to_reuse_database(self, db_ip_address: str) -> DatabaseInfo: + def _try_to_reuse_database(self, db_ip_address: str) -> Optional[DatabaseInfo]: self.logger.info("Try to reuse database container %s", self.db_container_name) try: database_info = self._create_database_info(db_ip_address=db_ip_address, reused=True) + return database_info except Exception as e: self.logger.warning("Tried to reuse database container %s, but got Exeception %s. " "Fallback to create new database.", self.db_container_name, e) - return database_info + return None def _get_ssh_key(self) -> SshKey: if self.ssh_key_file: From 989ea221389a59c91029b808dae578437e866ad6 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:49:51 -0300 Subject: [PATCH 31/60] Fixed return types in abstract_spawn_test_environment.py --- .../abstract_spawn_test_environment.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index 607c2ffed..ea809acb9 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Generator, Tuple, Optional +from typing import Generator, Tuple, Optional, Any from exasol_integration_test_docker_environment.lib.docker.container.utils import default_bridge_ip_address import luigi @@ -118,19 +118,19 @@ def collect_shell_variables(self, test_environment_info) -> ShellVariables: ) def _start_database(self, attempt) \ - -> Generator[BaseTask, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: + -> Generator[Any, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: network_info = yield from self._create_network(attempt) ssl_volume_info = None if self.create_certificates: - ssl_volume_info = yield from self._create_ssl_certificates() # type: ignore - database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) # type: ignore - is_database_ready = yield from self._wait_for_database(database_info, attempt) # type: ignore - return network_info, database_info, is_database_ready, test_container_info # type: ignore + ssl_volume_info = yield from self._create_ssl_certificates() + database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) + is_database_ready = yield from self._wait_for_database(database_info, attempt) + return network_info, database_info, is_database_ready, test_container_info def _create_ssl_certificates(self) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) - ssl_info = self.get_values_from_future(ssl_info_future) - return ssl_info # type: ignore + ssl_info : Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_info_future) # type: ignore + return ssl_info def create_ssl_certificates(self): raise AbstractMethodException() @@ -143,12 +143,12 @@ def _create_network(self, attempt): def create_network_task(self, attempt: int): raise AbstractMethodException() - def _spawn_database_and_test_container( # type: ignore + def _spawn_database_and_test_container( self, network_info: DockerNetworkInfo, certificate_volume_info: Optional[DockerVolumeInfo], attempt: int, - ) -> Tuple[DatabaseInfo, Optional[ContainerInfo]]: # type: ignore + ) -> Generator[BaseTask, None, Tuple[DatabaseInfo, Optional[ContainerInfo]]]: def volume_name(info): return None if info is None else info.volume_name @@ -168,8 +168,8 @@ def volume_name(info): ) futures = yield from self.run_dependencies(child_tasks) results = self.get_values_from_futures(futures) - database_info = results[DATABASE] - test_container_info = results[TEST_CONTAINER] if self.test_container_content is not None else None + database_info : DatabaseInfo = results[DATABASE] # type: ignore + test_container_info : Optional[ContainerInfo] = results[TEST_CONTAINER] if self.test_container_content is not None else None # type: ignore return database_info, test_container_info def create_spawn_database_task(self, From fcd71bb1c7d8370c04a0ba5229f239d4f82e66d9 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:50:07 -0300 Subject: [PATCH 32/60] Fixed types in test_container_parameter.py --- .../parameter/test_container_parameter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py index 52675552d..445f908b3 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py @@ -6,11 +6,11 @@ class TestContainerParameter: - test_container_content = JsonPickleParameter(TestContainerContentDescription, - visibility=ParameterVisibility.HIDDEN) + test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, + visibility=ParameterVisibility.HIDDEN) # type: ignore class OptionalTestContainerParameter: - test_container_content = JsonPickleParameter(TestContainerContentDescription, + test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, visibility=ParameterVisibility.HIDDEN, - is_optional=True) + is_optional=True) # type: ignore From 82af67b0a3253f14985943d52acd28f73ec6d46d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:52:34 -0300 Subject: [PATCH 33/60] Removed type ignore in test_api_logging.py --- .../test/test_api_logging.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exasol_integration_test_docker_environment/test/test_api_logging.py b/exasol_integration_test_docker_environment/test/test_api_logging.py index 1670aaf77..47064064a 100644 --- a/exasol_integration_test_docker_environment/test/test_api_logging.py +++ b/exasol_integration_test_docker_environment/test/test_api_logging.py @@ -239,16 +239,16 @@ def create_logger_infos(self) -> Dict[str, Dict[str, Any]]: return logger_infos def get_logger_info(self, logger: logging.Logger) -> Dict[str, Any]: - logger_info = {} + logger_info : Dict[str, Any] = dict() logger_info[LOGGER_STR] = str(logger) - logger_info[LEVEL] = logger.level # type: ignore + logger_info[LEVEL] = logger.level logger_info[LEVEL_NAME] = logging.getLevelName(logger.level) - logger_info[HANDLERS] = list(logger.handlers) # type: ignore + logger_info[HANDLERS] = list(logger.handlers) logger_info[LOGGER_NAME] = logger.name - logger_info[FILTERS] = list(logger.filters) # type: ignore - logger_info[DISABLED] = logger.disabled # type: ignore - logger_info[PROPAGATE] = logger.propagate # type: ignore - logger_info[PARENT] = logger.parent # type: ignore + logger_info[FILTERS] = list(logger.filters) + logger_info[DISABLED] = logger.disabled + logger_info[PROPAGATE] = logger.propagate + logger_info[PARENT] = logger.parent return logger_info From 9b500edb1d3b03d50caf90e22b64bd8381ee5925 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:58:46 -0300 Subject: [PATCH 34/60] Changed type hints for ssh access --- .../lib/base/ssh_access.py | 2 +- .../lib/test_environment/spawn_test_database.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/ssh_access.py b/exasol_integration_test_docker_environment/lib/base/ssh_access.py index 129dcc15b..ece4e748a 100644 --- a/exasol_integration_test_docker_environment/lib/base/ssh_access.py +++ b/exasol_integration_test_docker_environment/lib/base/ssh_access.py @@ -84,7 +84,7 @@ def opener(path, flags): return self @classmethod - def read_from(cls, private_key_file: Path) -> 'SshKey': + def read_from(cls, private_key_file: Path | str) -> 'SshKey': with open(private_key_file, "r") as file: rsa_key = paramiko.RSAKey.from_private_key(file) return SshKey(rsa_key) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 40bffc13f..83baa5489 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -58,7 +58,7 @@ class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore additional_db_parameter : List[str] = luigi.ListParameter() # type: ignore ssh_user : str = luigi.Parameter("root") # type: ignore - ssh_key_file : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + ssh_key_file : str | Path | None = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -102,8 +102,8 @@ def _try_to_reuse_database(self, db_ip_address: str) -> Optional[DatabaseInfo]: def _get_ssh_key(self) -> SshKey: if self.ssh_key_file: - return SshKey.read_from(self.ssh_key_file) # type: ignore - self.ssh_key_file = SshKeyCache().private_key # type: ignore + return SshKey.read_from(self.ssh_key_file) + self.ssh_key_file = SshKeyCache().private_key return SshKey.from_cache() def _handle_output(self, output_generator, image_info: ImageInfo): From 99064ab24838b4c937d8609918fd43145bc27ee1 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:09:22 -0300 Subject: [PATCH 35/60] Fixed wrong parameter in exaslct_test_environment.py --- .../testing/exaslct_test_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index 3c86064b5..c5fbc1d5b 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -178,7 +178,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option bucketfs_username=on_host_parameter.bucketfs_username, bucketfs_password=on_host_parameter.bucketfs_password, ports=Ports.default_ports, - environment_info=on_host_parameter.completed_process, # type: ignore + environment_info=on_host_parameter.environment_info, completed_process=on_host_parameter.completed_process ) From 40c89801b1c093ca00666fd5436ed3705931daeb Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:09:51 -0300 Subject: [PATCH 36/60] Removed unecessary type ignore in spawn_test_container.py --- .../lib/test_environment/spawn_test_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index adb65b406..be2c7362c 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -203,6 +203,6 @@ def cleanup_task(self, success: bool): (not success and not self.no_test_container_cleanup_after_failure): try: self.logger.info(f"Cleaning up container %s", self.test_container_name) - self._remove_container(self.test_container_name) # type: ignore + self._remove_container(self.test_container_name) except Exception as e: self.logger.error(f"Error during removing container %s: %s", self.test_container_name, e) From e5f57f8b2f18e7c9c425e6e62ebfc57a473dd316 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:11:09 -0300 Subject: [PATCH 37/60] 1. Fixed return type in spawn_test_database.py 2. Fixed instantiation of SshInfo in spawn_test_database.py --- .../lib/test_environment/spawn_test_database.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 83baa5489..bc10e2e5f 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -10,6 +10,7 @@ from docker.models.containers import Container from docker.models.volumes import Volume from docker.client import DockerClient +from importlib_resources.abc import Traversable from jinja2 import Template from exasol_integration_test_docker_environment.lib import PACKAGE_NAME @@ -200,7 +201,8 @@ def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInf network_info=self.network_info, volume_name=self._get_db_volume_name(), ) - ssh_info = SshInfo(self.ssh_user, self.ssh_key_file) # type: ignore + assert self.ssh_key_file + ssh_info = SshInfo(self.ssh_user, str(self.ssh_key_file)) database_info = DatabaseInfo( host=db_ip_address, ports=self.internal_ports, @@ -304,7 +306,7 @@ def _prepare_volume( ) return volume, container - def _db_file(self, filename: str) -> str: + def _db_file(self, filename: str) -> Traversable: return ( importlib_resources.files(PACKAGE_NAME) / self.docker_db_config_resource_name @@ -319,7 +321,7 @@ def _upload_init_db_files( ): copy = DockerContainerCopy(container) init_script = self._db_file("init_db.sh") - copy.add_string_to_file("init_db.sh", init_script.read_text()) # type: ignore + copy.add_string_to_file("init_db.sh", init_script.read_text()) self._add_exa_conf(copy, db_private_network, authorized_keys) copy.copy("/") From fc520743f7984ebf5e416f49e97656f539e0b42b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:11:13 -0300 Subject: [PATCH 38/60] [run all tests] From 2724a72d08f1fedca18f9018a3c293cb48cc4373 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:46:43 -0300 Subject: [PATCH 39/60] Fixed missing import in spawn_test_database.py --- .../lib/test_environment/spawn_test_database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index bc10e2e5f..b1bf0e117 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -1,5 +1,6 @@ import math from typing import Optional, Tuple, List +from pathlib import Path import docker import humanfriendly From cba4a3677de4575c3ff947e5478d5806d7dccf5e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:46:53 -0300 Subject: [PATCH 40/60] [run all tests] From 19e16ad1de35b7e5b66e4262d862069e6f25cc87 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:50:59 -0300 Subject: [PATCH 41/60] Fixed type hint which was incompatible with Python3.9 --- .../lib/base/ssh_access.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/base/ssh_access.py b/exasol_integration_test_docker_environment/lib/base/ssh_access.py index ece4e748a..0a792cab9 100644 --- a/exasol_integration_test_docker_environment/lib/base/ssh_access.py +++ b/exasol_integration_test_docker_environment/lib/base/ssh_access.py @@ -6,8 +6,7 @@ import portalocker from pathlib import Path from string import Template -from typing import Optional - +from typing import Optional, Union _LOCK_FILE = "$TMP/$MODULE-ssh-access.lock" _DEFAULT_CACHE_DIR = "$HOME/.cache/exasol/$MODULE" @@ -84,7 +83,7 @@ def opener(path, flags): return self @classmethod - def read_from(cls, private_key_file: Path | str) -> 'SshKey': + def read_from(cls, private_key_file: Union[Path, str]) -> 'SshKey': with open(private_key_file, "r") as file: rsa_key = paramiko.RSAKey.from_private_key(file) return SshKey(rsa_key) From a1b081fe3e62233d95a4845cdc76b51357782f53 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:51:02 -0300 Subject: [PATCH 42/60] [run all tests] From 659ee73d38e97c907e52dc736f89fee719b4e8ea Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:55:52 -0300 Subject: [PATCH 43/60] Fixed type hint which was incompatible with Python3.9 --- .../lib/test_environment/spawn_test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index b1bf0e117..f305c731d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -60,7 +60,7 @@ class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore additional_db_parameter : List[str] = luigi.ListParameter() # type: ignore ssh_user : str = luigi.Parameter("root") # type: ignore - ssh_key_file : str | Path | None = luigi.OptionalParameter(None, significant=False) # type: ignore + ssh_key_file : Union[str, Path, None] = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 02bbb59d65b1fcad050a5c4d5c2a3effbf14fae6 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:55:54 -0300 Subject: [PATCH 44/60] [run all tests] From 5342429fb6984e675a59f7784ff4b1ff0f51f75d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:01:12 -0300 Subject: [PATCH 45/60] Fixed missing import in spawn_test_database.py --- .../lib/test_environment/spawn_test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index f305c731d..af8810013 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -1,5 +1,5 @@ import math -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Union from pathlib import Path import docker From b064b9608f1a234ba2daded082c156d595dcb6df Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:01:16 -0300 Subject: [PATCH 46/60] [run all tests] From e25dce23dcfe4b86d21c61c2d3231607c59351db Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:05:42 -0300 Subject: [PATCH 47/60] Use Union for type in spawn_test_container.py as it was incompatible with Python3.9 --- .../lib/test_environment/spawn_test_container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index be2c7362c..217c362eb 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import List, Optional, Dict +from typing import List, Optional, Dict, Union import luigi import netaddr @@ -111,7 +111,7 @@ def _create_test_container(self, ip_address, test_container_image_info = \ self.get_values_from_futures(self.test_container_image_future)["test-container"] - volumes : Dict[str | Path, Dict[str, str]] = dict() + volumes : Dict[Union[str, Path], Dict[str, str]] = dict() for runtime_mapping in self.test_container_content.runtime_mappings: volumes[runtime_mapping.source.absolute()] = { "bind": runtime_mapping.target, From e864f880681d0b4c305a60815e035bd007582bfa Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:05:44 -0300 Subject: [PATCH 48/60] [run all tests] From 8481cb488d1caf2ed1dc768b7b519f8024329009 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:42:48 -0300 Subject: [PATCH 49/60] Removed assertion which broke test: test_test_env_reuse.TestContainerReuseTest --- .../lib/test_environment/spawn_test_database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index af8810013..5d8369c90 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -202,8 +202,7 @@ def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInf network_info=self.network_info, volume_name=self._get_db_volume_name(), ) - assert self.ssh_key_file - ssh_info = SshInfo(self.ssh_user, str(self.ssh_key_file)) + ssh_info = SshInfo(self.ssh_user, str(self.ssh_key_file or "")) database_info = DatabaseInfo( host=db_ip_address, ports=self.internal_ports, From 95cdb49bfcdef646c32329f3e075954f6db0613e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:42:56 -0300 Subject: [PATCH 50/60] [run all tests] From 40604abde55f73b5cfadbed922b2a45e3e67db7c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:27:03 -0300 Subject: [PATCH 51/60] Fixed variable name in AbstractSpawnTestEnvironment._create_ssl_certificates() --- .../lib/test_environment/abstract_spawn_test_environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index ea809acb9..4a88daa70 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -128,9 +128,9 @@ def _start_database(self, attempt) \ return network_info, database_info, is_database_ready, test_container_info def _create_ssl_certificates(self) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: - ssl_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) - ssl_info : Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_info_future) # type: ignore - return ssl_info + ssl_volume_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) + ssl_volume_info : Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_volume_info_future) # type: ignore + return ssl_volume_info def create_ssl_certificates(self): raise AbstractMethodException() From ee5cf203fac2ea8be45215348498346d46950a3d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:27:31 -0300 Subject: [PATCH 52/60] Fixed return type in CreateSSLCertificatesTask.build_image() --- .../create_certificates/create_ssl_certificates_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py index 30e46b560..ae5a6f6ec 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py @@ -56,11 +56,11 @@ def run_task(self): self.return_object(self.volume_info) - def build_image(self) -> Generator[BaseTask, None, Set[str]]: + def build_image(self) -> Generator[BaseTask, None, Set[ImageInfo]]: task = self.create_child_task(task_class=DockerCertificateContainerBuild, certificate_container_root_directory=self._temp_resource_directory.tmp_directory) image_infos_future = yield from self.run_dependencies(task) - image_infos = self.get_values_from_future(image_infos_future) + image_infos: Set[ImageInfo] = self.get_values_from_future(image_infos_future) # type: ignore return image_infos def get_volume_info(self, reused: bool) -> DockerVolumeInfo: From 0049e3d6073fe59557abeaa7e96cc9f866eef9b3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:58:03 -0300 Subject: [PATCH 53/60] Fixed return type in conftest.py --- test/integration/conftest.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 096b6337b..3994bb89b 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -3,11 +3,8 @@ import pytest -from typing import Any, Callable, Dict, Iterator, List, NewType, Optional, Generator +from typing import Any, Callable, Dict, Iterator, List, NewType, Optional, Generator, ContextManager -from typing_extensions import Never - -from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from test.integration.helpers import normalize_request_name from exasol_integration_test_docker_environment.testing import utils from exasol_integration_test_docker_environment \ @@ -52,7 +49,7 @@ def api_isolation(request) -> Iterator[ApiTestEnvironment]: @pytest.fixture -def cli_database(cli_isolation) -> CliContextProvider: +def cli_database(cli_isolation) -> Callable[[Optional[str], Optional[list[str]]], ContextManager[SpawnedTestEnvironments]]: """ Returns a method that test case implementations can use to create a context with a database. @@ -65,11 +62,11 @@ def test_case(database): ... """ - @contextlib.contextmanager # type: ignore - def create_context( # type: ignore + @contextlib.contextmanager + def create_context( name: Optional[str] = None, additional_parameters: Optional[List[str]] = None, - ) -> SpawnedTestEnvironments: + ) -> Iterator[SpawnedTestEnvironments]: name = name if name else cli_isolation.name spawned = cli_isolation.spawn_docker_test_environments( name=name, @@ -78,7 +75,7 @@ def create_context( # type: ignore yield spawned utils.close_environments(spawned) - return create_context # type: ignore + return create_context ApiContextProvider = NewType( # type: ignore From c36d74d8534e584be0b0746b2e2908f94c023a5b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:58:45 -0300 Subject: [PATCH 54/60] Fixed types for str.join() --- .../lib/test_environment/spawn_test_database.py | 6 +++--- .../testing/exaslct_test_environment.py | 6 +++--- test/integration/test_cli_environment.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index 5d8369c90..a54dfcce7 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -337,8 +337,8 @@ def _add_exa_conf( certificate_dir = CERTIFICATES_MOUNT_DIR if self.certificate_volume_name is not None \ else CERTIFICATES_DEFAULT_DIR template_file = self._db_file("EXAConf") - template = Template(template_file.read_text()) # type: ignore - additional_db_parameter_str = " ".join(self.additional_db_parameter) # type: ignore + template = Template(template_file.read_text()) + additional_db_parameter_str = " ".join(self.additional_db_parameter) rendered_template = template.render(private_network=db_private_network, db_version=str(self.db_version), db_port=self.internal_ports.database, @@ -347,7 +347,7 @@ def _add_exa_conf( image_version=self.docker_db_image_version, mem_size=self.mem_size, disk_size=self.disk_size, - name_servers=",".join(self.nameservers), # type: ignore + name_servers=",".join(self.nameservers), certificate_dir=certificate_dir, additional_db_parameters=additional_db_parameter_str, authorized_keys=authorized_keys) diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index c5fbc1d5b..3cbcab4a1 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -143,7 +143,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option ports=ports, ) - arguments = [ + arguments : List[str] = [ f"--environment-name {on_host_parameter.name}", f"--database-port-forward {on_host_parameter.ports.database}", f"--bucketfs-port-forward {on_host_parameter.ports.bucketfs}", @@ -154,9 +154,9 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option arguments.append(f'--docker-db-image-version "{db_version}"') if additional_parameter: arguments += additional_parameter - arguments = " ".join(arguments) #type: ignore + arguments_str = " ".join(arguments) - command = f"{self.executable} spawn-test-environment {arguments}" + command = f"{self.executable} spawn-test-environment {arguments_str}" completed_process = self.run_command(command, use_flavor_path=False, use_docker_repository=False, capture_output=True) on_host_parameter.completed_process = completed_process diff --git a/test/integration/test_cli_environment.py b/test/integration/test_cli_environment.py index 48f577e32..29f28a067 100644 --- a/test/integration/test_cli_environment.py +++ b/test/integration/test_cli_environment.py @@ -53,7 +53,7 @@ def quote(s): assert env.environment_info db_info = env.environment_info.database_info - command = [ + command : List[str] = [ str(exaplus_path), "-c", quote(f"{db_info.host}:{db_info.ports.database}"), "-u", quote(env.db_username), @@ -65,8 +65,8 @@ def quote(s): "-jdbcparam", "validateservercertificate=0", ] - command = " ".join(command) #type: ignore - return f'bash -c "{command}" ' + command_str = " ".join(command) + return f'bash -c "{command_str}" ' def test_db_container_started(cli_database): From f1a37e890940bf6d418551d0938cea4bbeaefc26 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:59:14 -0300 Subject: [PATCH 55/60] Fixed types in run_minimal_tests in noxfile.py --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 26c475d7e..29c59e176 100644 --- a/noxfile.py +++ b/noxfile.py @@ -66,9 +66,9 @@ def run_minimal_tests(session: nox.Session, db_version: str): "test_termination_handler.py", ], "new-itest": ["test_cli_environment.py", "test_db_container_log_thread.py"], - "unit": "./test/unit", + "unit": ["./test/unit"], } - session.run("pytest", minimal_tests["unit"]) #type: ignore + session.run("pytest", *minimal_tests["unit"]) for test in minimal_tests["new-itest"]: session.run( "pytest", From 5314cad5deee8cc522679b02e80662855e3005e7 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:59:36 -0300 Subject: [PATCH 56/60] Added comment in test_populate_data.py for type ignore --- .../test/test_populate_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol_integration_test_docker_environment/test/test_populate_data.py b/exasol_integration_test_docker_environment/test/test_populate_data.py index b535db05b..9aeb79c7e 100644 --- a/exasol_integration_test_docker_environment/test/test_populate_data.py +++ b/exasol_integration_test_docker_environment/test/test_populate_data.py @@ -66,6 +66,7 @@ def setUp(self) -> None: def _execute_sql_on_db(self, sql: str) -> str: with ContextDockerClient() as docker_client: print(f"Executing sql on db: '{sql}'") + #environment is a class variable, need to suppress type check environment = self.environment # type: ignore test_container = docker_client.containers.get(environment.environment_info. test_container_info.container_name) From 35946b2824b261f3a4957403868bf8c811afe69a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:59:40 -0300 Subject: [PATCH 57/60] [run all tests] From c6a9bebe84297ef52e243e874191d6b269187ba3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 25 Nov 2024 06:16:42 -0300 Subject: [PATCH 58/60] Added a comment in common.py --- exasol_integration_test_docker_environment/lib/api/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index 6d93671da..37921ca32 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -42,6 +42,7 @@ def __new__(cls): return cls._instance def get_next_value(self) -> int: + # self._counter is a class variable and because of this we need to suppress type checks self._counter += 1 # type: ignore return self._counter # type: ignore From faca385e81a045db4bd1f3a7d9d342f07c23318c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 25 Nov 2024 06:16:57 -0300 Subject: [PATCH 59/60] Fixed type hints in test_doctor.py --- .../test/test_doctor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol_integration_test_docker_environment/test/test_doctor.py b/exasol_integration_test_docker_environment/test/test_doctor.py index a17cf230e..7ed1767a4 100644 --- a/exasol_integration_test_docker_environment/test/test_doctor.py +++ b/exasol_integration_test_docker_environment/test/test_doctor.py @@ -14,8 +14,8 @@ ) -@contextmanager #type: ignore -def temporary_env(env_vars) -> Generator: #type: ignore +@contextmanager +def temporary_env(env_vars) -> Generator[os._Environ, None, None]: """ Creates a temporary environment, containing the current environment variables. From b62912a96ea541619c17bceb18f0ae00494016bb Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 25 Nov 2024 06:17:01 -0300 Subject: [PATCH 60/60] [run all tests]