From 75d3f4333fb6d57db4cc1a217a1cc87de75b24ad Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 22 Apr 2024 12:51:01 +0100 Subject: [PATCH] Try build and run docker only once per session. --- tests/ssh_test_utils.py | 50 --------- tests/tests_integration/base.py | 106 ++++++++++++++++++ .../test_ssh_file_transfer.py | 45 +------- tests/tests_integration/test_ssh_setup.py | 14 ++- 4 files changed, 119 insertions(+), 96 deletions(-) diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index 5db0bbdd..7391d853 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -4,21 +4,15 @@ import builtins import copy -import os -import platform import stat import subprocess import sys import warnings -from pathlib import Path import paramiko from datashuttle.utils import rclone, ssh -PORT = 3306 # https://github.com/orgs/community/discussions/25550 -os.environ["DS_SSH_PORT"] = str(PORT) - def setup_project_for_ssh( project, central_path, central_host_id, central_host_username @@ -95,50 +89,6 @@ def setup_ssh_connection(project, setup_ssh_key_pair=True): return verified -def setup_project_and_container_for_ssh(project): - """""" - assert docker_is_running(), ( - "docker is not running, " - "this should be checked at the top of test script" - ) - - image_path = Path(__file__).parent / "ssh_test_images" - os.chdir(image_path) - - if platform.system() == "Linux": - build_command = "sudo docker build -t ssh_server ." - run_command = f"sudo docker run -d -p {PORT}:22 ssh_server" - else: - build_command = "docker build ." - run_command = f"docker run -d -p {PORT}:22 ssh_server" - - build_output = subprocess.run( - build_command, - shell=True, - capture_output=True, - ) - assert ( - build_output.returncode == 0 - ), f"docker build failed with: STDOUT-{build_output.stdout} STDERR-{build_output.stderr}" - - run_output = subprocess.run( - run_command, - shell=True, - capture_output=True, - ) - - assert ( - run_output.returncode == 0 - ), f"docker run failed with: STDOUT-{run_output.stdout} STDERR-{run_output.stderr}" - - setup_project_for_ssh( - project, - central_path=f"/home/sshuser/datashuttle/{project.project_name}", - central_host_id="localhost", - central_host_username="sshuser", - ) - - def sftp_recursive_file_search(sftp, path_, all_filenames): try: sftp.stat(path_) diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index 50371179..479a1e5d 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -1,8 +1,13 @@ import os +import platform +import subprocess import warnings +from pathlib import Path import pytest +import ssh_test_utils import test_utils +from file_conflicts_pathtable import get_pathtable from datashuttle.datashuttle import DataShuttle @@ -56,3 +61,104 @@ def clean_project_name(self): test_utils.delete_project_if_it_exists(project_name) yield project_name test_utils.delete_project_if_it_exists(project_name) + + @pytest.fixture( + scope="class", + ) + def pathtable_and_project(self, tmpdir_factory): + """ + Create a new test project with a test project folder + and file structure (see `get_pathtable()` for definition). + """ + tmp_path = tmpdir_factory.mktemp("test") + + base_path = tmp_path / "test with space" + test_project_name = "test_file_conflicts" + + project, cwd = test_utils.setup_project_fixture( + base_path, test_project_name + ) + + pathtable = get_pathtable(project.cfg["local_path"]) + + self.create_all_pathtable_files(pathtable) + + yield [pathtable, project] + + test_utils.teardown_project(cwd, project) + + @pytest.fixture( + scope="session", + ) + def setup_ssh_container(self): + """""" + PORT = 3306 # https://github.com/orgs/community/discussions/25550 + os.environ["DS_SSH_PORT"] = str(PORT) + + assert ssh_test_utils.docker_is_running(), ( + "docker is not running, " + "this should be checked at the top of test script" + ) + + image_path = Path(__file__).parent.parent / "ssh_test_images" + os.chdir(image_path) + + if platform.system() == "Linux": + build_command = "sudo docker build -t ssh_server ." + run_command = f"sudo docker run -d -p {PORT}:22 ssh_server" + else: + build_command = "docker build ." + run_command = f"docker run -d -p {PORT}:22 ssh_server" + + build_output = subprocess.run( + build_command, + shell=True, + capture_output=True, + ) + assert build_output.returncode == 0, ( + f"docker build failed with: STDOUT-{build_output.stdout} STDERR-" + f"{build_output.stderr}" + ) + + run_output = subprocess.run( + run_command, + shell=True, + capture_output=True, + ) + + assert run_output.returncode == 0, ( + f"docker run failed with: STDOUT-{run_output.stdout} STDE" + f"RR-{run_output.stderr}" + ) + + # setup_project_for_ssh( + # project, + # central_path=f"/home/sshuser/datashuttle/{project.project_name}", + # central_host_id="localhost", + # central_host_username="sshuser", + # ) + + @pytest.fixture( + scope="class", + ) + def ssh_setup(self, pathtable_and_project, setup_ssh_container): + """ + After initial project setup (in `pathtable_and_project`) + setup a container and the project's SSH connection to the container. + Then upload the test project to the `central_path`. + """ + pathtable, project = pathtable_and_project + + ssh_test_utils.setup_project_for_ssh( + project, + central_path=f"/home/sshuser/datashuttle/{project.project_name}", + central_host_id="localhost", + central_host_username="sshuser", + ) + + # ssh_test_utils.setup_project_and_container_for_ssh(project) + ssh_test_utils.setup_ssh_connection(project) + + project.upload_rawdata() + + return [pathtable, project] diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 045755de..4601cfd3 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -10,7 +10,7 @@ import pytest import ssh_test_utils import test_utils -from file_conflicts_pathtable import get_pathtable +from base import BaseTest from datashuttle.utils import ssh @@ -44,48 +44,7 @@ ] -class TestFileTransfer: - @pytest.fixture( - scope="class", - ) - def pathtable_and_project(self, tmpdir_factory): - """ - Create a new test project with a test project folder - and file structure (see `get_pathtable()` for definition). - """ - tmp_path = tmpdir_factory.mktemp("test") - - base_path = tmp_path / "test with space" - test_project_name = "test_file_conflicts" - - project, cwd = test_utils.setup_project_fixture( - base_path, test_project_name - ) - - pathtable = get_pathtable(project.cfg["local_path"]) - - self.create_all_pathtable_files(pathtable) - - yield [pathtable, project] - - test_utils.teardown_project(cwd, project) - - @pytest.fixture( - scope="class", - ) - def ssh_setup(self, pathtable_and_project): - """ - After initial project setup (in `pathtable_and_project`) - setup a container and the project's SSH connection to the container. - Then upload the test project to the `central_path`. - """ - pathtable, project = pathtable_and_project - ssh_test_utils.setup_project_and_container_for_ssh(project) - ssh_test_utils.setup_ssh_connection(project) - - project.upload_rawdata() - - return [pathtable, project] +class TestFileTransfer(BaseTest): # ---------------------------------------------------------------------------------- # Test File Transfer - All Options diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index 1aa4b75a..e2f032bd 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,6 +1,7 @@ import pytest import ssh_test_utils import test_utils +from base import BaseTest # from pytest import ssh_config from datashuttle.utils import ssh @@ -9,9 +10,9 @@ @pytest.mark.skipif("not TEST_SSH", reason="TEST_SSH is false") -class TestSSH: +class TestSSH(BaseTest): @pytest.fixture(scope="function") - def project(test, tmp_path): + def project(test, tmp_path, setup_ssh_container): """ Make a project as per usual, but now add in test ssh configurations @@ -23,7 +24,14 @@ def project(test, tmp_path): tmp_path, test_project_name ) - ssh_test_utils.setup_project_and_container_for_ssh(project) + # ssh_test_utils.setup_project_and_container_for_ssh(project) + + ssh_test_utils.setup_project_for_ssh( + project, + central_path=f"/home/sshuser/datashuttle/{project.project_name}", + central_host_id="localhost", + central_host_username="sshuser", + ) yield project test_utils.teardown_project(cwd, project)