[Testing] Test SSH file transfer with images.

.github/workflows/code_test_and_deploy.yml

@@ -46,7 +46,16 @@ jobs:
           python -m pip install --upgrade pip
           pip install .[dev]
       - name: Test
-        run: pytest
+        # run SSH tests only on Linux because Windows and macOS
+        # are already run within a virtual container and so cannot
+        # run Linux containers because nested containerisation is disabled.
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+             sudo service mysql stop  # free up port  3306 for ssh tests: https://github.com/orgs/community/discussions/25550
+             pytest
+          else
+             pytest -k "not test_combinations_ssh_transfer and not test_ssh_setup"
+          fi
 
   build_sdist_wheels:
     name: Build source distribution

datashuttle/configs/canonical_configs.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import (
     TYPE_CHECKING,
     Dict,
@@ -54,6 +55,16 @@ def get_datatypes() -> List[str]:
     return ["ephys", "behav", "funcimg", "anat"]
 
 
+def get_default_ssh_port() -> int:
+    """
+    Get the default port used for SSH connections.
+    """
+    if "DS_SSH_PORT" in os.environ:
+        return int(os.environ["DS_SSH_PORT"])
+    else:
+        return 22
+
+
 # -----------------------------------------------------------------------------
 # Check Configs
 # -----------------------------------------------------------------------------

datashuttle/utils/data_transfer.py

@@ -157,7 +157,6 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]:
                     self.update_list_with_non_ses_sub_level_folders(
                         extra_folder_names, extra_filenames, sub
                     )
-
                     continue
 
                 # Datatype (sub and ses level) --------------------------------

datashuttle/utils/rclone.py

@@ -3,6 +3,7 @@
 from subprocess import CompletedProcess
 from typing import Dict, List, Literal
 
+from datashuttle.configs import canonical_configs
 from datashuttle.configs.config_class import Configs
 from datashuttle.utils import utils
 from datashuttle.utils.custom_types import TopLevelFolder
@@ -99,7 +100,7 @@ def setup_rclone_config_for_ssh(
         f"sftp "
         f"host {cfg['central_host_id']} "
         f"user {cfg['central_host_username']} "
-        f"port 22 "
+        f"port {canonical_configs.get_default_ssh_port()} "
         f"key_file {ssh_key_path.as_posix()}",
         pipe_std=True,
     )

datashuttle/utils/ssh.py

@@ -14,6 +14,7 @@
 
 import paramiko
 
+from datashuttle.configs import canonical_configs
 from datashuttle.utils import utils
 
 # -----------------------------------------------------------------------------
@@ -42,6 +43,7 @@ def connect_client_core(
             else None
         ),
         look_for_keys=True,
+        port=canonical_configs.get_default_ssh_port(),
     )
 
 
@@ -83,15 +85,21 @@ def get_remote_server_key(central_host_id: str):
     connection.
     """
     transport: paramiko.Transport
-    with paramiko.Transport(central_host_id) as transport:
+    with paramiko.Transport(
+        (central_host_id, canonical_configs.get_default_ssh_port())
+    ) as transport:
         transport.connect()
         key = transport.get_remote_server_key()
     return key
 
 
 def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None:
     client = paramiko.SSHClient()
-    client.get_host_keys().add(central_host_id, key.get_name(), key)
+    client.get_host_keys().add(
+        f"[{central_host_id}]:{canonical_configs.get_default_ssh_port()}",
+        key.get_name(),
+        key,
+    )
     client.get_host_keys().save(hostkeys_path.as_posix())
 
 
@@ -179,15 +187,16 @@ def connect_client_with_logging(
                 f"Connection to { cfg['central_host_id']} made successfully."
             )
 
-    except Exception:
+    except Exception as e:
         utils.log_and_raise_error(
             f"Could not connect to server. Ensure that \n"
             f"1) You have run setup_ssh_connection() \n"
             f"2) You are on VPN network if required. \n"
             f"3) The central_host_id: {cfg['central_host_id']} is"
             f" correct.\n"
             f"4) The central username:"
-            f" {cfg['central_host_username']}, and password are correct.",
+            f" {cfg['central_host_username']}, and password are correct."
+            f"Original error: {e}",
             ConnectionError,
         )
 

tests/ssh_test_images/Dockerfile

@@ -0,0 +1,25 @@
+# Use a base image with the desired OS (e.g., Ubuntu, Debian, etc.)
+FROM ubuntu:latest
+
+# Install SSH server
+RUN apt-get update && \
+    apt-get upgrade -y
+RUN apt-get install openssh-server -y supervisor
+RUN apt-get install nano
+
+# Create an SSH user
+RUN useradd -rm -d /home/sshuser -s /bin/bash -g root -G sudo -u 1000 sshuser
+
+# Set the SSH user's password (replace "password" with your desired password)
+RUN echo "sshuser:password" | chpasswd
+
+# Allow SSH access
+RUN mkdir /var/run/sshd
+
+RUN /usr/bin/ssh-keygen -A
+
+# Expose the SSH port
+EXPOSE 22
+
+# Start SSH server on container startup
+CMD ["/usr/sbin/sshd", "-D"]

tests/ssh_test_utils.py

@@ -1,55 +1,206 @@
 import builtins
 import copy
+import os
+import platform
+import stat
+import subprocess
+import sys
+import warnings
+from pathlib import Path
+
+import paramiko
 
 from datashuttle.utils import rclone, ssh
 
+# Choose port 3306 for running on GH actions
+# suggested in https://github.com/orgs/community/discussions/25550
+PORT = 3306
+os.environ["DS_SSH_PORT"] = str(PORT)
+
 
 def setup_project_for_ssh(
-    project, central_path, central_host_id, central_host_username
+    project,
 ):
     """
-    Set up the project configs to use SSH connection
-    to central
+    Set up the project configs to use
+    SSH connection to central. The settings
+    set up a connection to the Dockerfile image
+    found in /ssh_test_images.
     """
     project.update_config_file(
-        central_path=central_path,
+        connection_method="ssh",
+        central_path=f"/home/sshuser/datashuttle/{project.project_name}",
+        central_host_id="localhost",
+        central_host_username="sshuser",
     )
-    project.update_config_file(central_host_id=central_host_id)
-    project.update_config_file(central_host_username=central_host_username)
-    project.update_config_file(connection_method="ssh")
-
     rclone.setup_rclone_config_for_ssh(
         project.cfg,
         project.cfg.get_rclone_config_name("ssh"),
         project.cfg.ssh_key_path,
     )
 
 
-def setup_mock_input(input_):
+def setup_ssh_connection(project, setup_ssh_key_pair=True):
     """
-    This is very similar to pytest monkeypatch but
-    using that was giving me very strange output,
-    monkeypatch.setattr('builtins.input', lambda _: "n")
-    i.e. pdb went deep into some unrelated code stack
+    Convenience function to verify the server hostkey and ssh
+    key pairs to the Dockerfile image for ssh tests.
+
+    This requires monkeypatching a number of functions involved
+    in the SSH setup process. `input()` is patched to always
+    return the required hostkey confirmation "y". `getpass()` is
+    patched to always return the password for the container in which
+    SSH tests are run. `isatty()` is patched because when running this
+    for some reason it appears to be in a TTY - this might be a
+    container thing.
     """
+    # Monkeypatch
     orig_builtin = copy.deepcopy(builtins.input)
-    builtins.input = lambda _: input_  # type: ignore
-    return orig_builtin
+    builtins.input = lambda _: "y"  # type: ignore
+
+    orig_getpass = copy.deepcopy(ssh.getpass.getpass)
+    ssh.getpass.getpass = lambda _: "password"  # type: ignore
+
+    orig_isatty = copy.deepcopy(sys.stdin.isatty)
+    sys.stdin.isatty = lambda: True
+
+    # Run setup
+    verified = ssh.verify_ssh_central_host(
+        project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
+    )
+
+    if setup_ssh_key_pair:
+        ssh.setup_ssh_key(project.cfg, log=False)
+
+    # Restore functions
+    builtins.input = orig_builtin
+    ssh.getpass.getpass = orig_getpass
+    sys.stdin.isatty = orig_isatty
+
+    return verified
 
 
-def restore_mock_input(orig_builtin):
+def setup_ssh_container(container_name):
     """
-    orig_builtin: the copied, original builtins.input
+    Build and run the docker container used for
+    ssh tests.
     """
-    builtins.input = orig_builtin
+    assert docker_is_running(), (
+        "docker is not running, "
+        "this should be checked at the top of test script"
+    )
+
+    image_path = Path(__file__).parent / "ssh_test_images"
+    os.chdir(image_path)
+
+    if platform.system() != "Windows":
+        build_command = "sudo docker build -t ssh_server ."
+        run_command = (
+            f"sudo docker run -d -p {PORT}:22 "
+            f"--name {container_name} ssh_server"
+        )
+    else:
+        build_command = "docker build ."
+        run_command = (
+            f"docker run -d -p {PORT}:22 --name {container_name}  ssh_server"
+        )
+
+    build_output = subprocess.run(
+        build_command,
+        shell=True,
+        capture_output=True,
+    )
+    assert build_output.returncode == 0, (
+        f"docker build failed with: STDOUT-{build_output.stdout} "
+        f"STDERR-{build_output.stderr}"
+    )
+
+    run_output = subprocess.run(
+        run_command,
+        shell=True,
+        capture_output=True,
+    )
+
+    assert run_output.returncode == 0, (
+        f"docker run failed with: STDOUT-{run_output.stdout} "
+        f"STDERR-{run_output.stderr}"
+    )
 
 
-def setup_hostkeys(project):
+def recursive_search_central(project):
     """
-    Convenience function to verify the server hostkey.
+    A convenience function to recursively search a
+    project for files through SSH, used  during testing
+    across an SSH connection to collected names of
+    files that were transferred.
     """
-    orig_builtin = setup_mock_input(input_="y")
-    ssh.verify_ssh_central_host(
-        project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
+    with paramiko.SSHClient() as client:
+        ssh.connect_client_core(client, project.cfg)
+
+        sftp = client.open_sftp()
+
+        all_filenames = []
+
+        sftp_recursive_file_search(
+            sftp,
+            (project.cfg["central_path"] / "rawdata").as_posix(),
+            all_filenames,
+        )
+    return all_filenames
+
+
+def sftp_recursive_file_search(sftp, path_, all_filenames):
+    """
+    Append all filenames found within a folder,
+    when searching over a sftp connection.
+    """
+    try:
+        sftp.stat(path_)
+    except FileNotFoundError:
+        return
+
+    for file_or_folder in sftp.listdir_attr(path_):
+        if stat.S_ISDIR(file_or_folder.st_mode):
+            sftp_recursive_file_search(
+                sftp,
+                path_ + "/" + file_or_folder.filename,
+                all_filenames,
+            )
+        else:
+            all_filenames.append(path_ + "/" + file_or_folder.filename)
+
+
+def get_test_ssh():
+    """
+    Return bool indicating whether Docker is installed and running,
+    which is required for ssh tests.
+    """
+    docker_installed = docker_is_running()
+    if not docker_installed:
+        warnings.warn(
+            "SSH tests are not run as docker either not installed or running."
+        )
+    return docker_installed
+
+
+def docker_is_running():
+    if not is_docker_installed():
+        return False
+
+    is_running = check_sys_command_returns_0("docker stats --no-stream")
+    return is_running
+
+
+def is_docker_installed():
+    return check_sys_command_returns_0("docker -v")
+
+
+def check_sys_command_returns_0(command):
+    return (
+        subprocess.run(
+            command,
+            shell=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        ).returncode
+        == 0
     )
-    restore_mock_input(orig_builtin)