From bdda21f7c7aa954b54b56ac49ec9a9324c9b6753 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Nov 2023 15:41:10 +0100 Subject: [PATCH] Use paramiko module to upload archives. --- .github/scripts/common.py | 147 ++++++++++++++++++++++++++------------ .github/workflows/ci.yml | 9 ++- setup.py | 3 +- 3 files changed, 111 insertions(+), 48 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index f1433ccf8..ef8efcda5 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -1,6 +1,6 @@ import os from os import environ as _environ -from pathlib import Path +from pathlib import Path, PurePosixPath from datetime import date import tarfile import zipfile @@ -197,55 +197,112 @@ def run_kiwix_build( subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") +try: + import paramiko -def upload(file_to_upload, host, dest_path): - if not file_to_upload.exists(): - print_message("No {} to upload!", file_to_upload) - return + def upload(file_to_upload, host, dest_path): + if not file_to_upload.exists(): + print_message("No {} to upload!", file_to_upload) + return - if ":" in host: - host, port = host.split(":", 1) - else: - port = "22" + if ":" in host: + host, port = host.split(":", 1) + else: + port = "22" + if '@' in host: + user, host = host.split('@', 1) + else: + user = None + + from contextlib import contextmanager + + @contextmanager + def get_client(): + client = paramiko.client.SSHClient() + client.set_missing_host_key_policy(paramiko.client.WarningPolicy) + print_message(f"Connect to {host}:{port}") + client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True) + try: + yield client + finally: + client.close() + + @contextmanager + def get_sftp(): + with get_client() as client: + sftp = client.open_sftp() + try: + yield sftp + finally: + sftp.close() + + dest_path = PurePosixPath(dest_path) + remote_file = dest_path.joinpath(file_to_upload.name) + + with get_sftp() as sftp: + for part in list(reversed(dest_path.parents)) + [dest_path]: + part = str(part) + try: + sftp.stat(part) + except FileNotFoundError: + sftp.mkdir(part) + + print_message(f"Sending archive {file_to_upload} to {remote_file}") + sftp.put(str(file_to_upload), str(remote_file), confirm=True) + + +except ModuleNotFoundError: + # On old system (bionic) paramiko is really complex to install + # Keep the old implementaion on sush system. + + def upload(file_to_upload, host, dest_path): + if not file_to_upload.exists(): + print_message("No {} to upload!", file_to_upload) + return + + if ":" in host: + host, port = host.split(":", 1) + else: + port = "22" - # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode - # as the latter would exit on any mkdir error while it is most likely - # the first parts of the destination is already present and thus can't be created - sftp_commands = "\n".join( - [ - f"mkdir {part}" - for part in list(reversed(Path(dest_path).parents)) + [dest_path] + # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode + # as the latter would exit on any mkdir error while it is most likely + # the first parts of the destination is already present and thus can't be created + sftp_commands = "\n".join( + [ + f"mkdir {part}" + for part in list(reversed(Path(dest_path).parents)) + [dest_path] + ] + ) + command = [ + "sftp", + "-i", + _environ.get("SSH_KEY"), + "-P", + port, + "-o", + "StrictHostKeyChecking=no", + host, ] - ) - command = [ - "sftp", - "-i", - _environ.get("SSH_KEY"), - "-P", - port, - "-o", - "StrictHostKeyChecking=no", - host, - ] - print_message("Creating dest path {}", dest_path) - subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True) + print_message("Creating dest path {}", dest_path) + subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True) - command = [ - "scp", - "-c", - "aes128-ctr", - "-rp", - "-P", - port, - "-i", - _environ.get("SSH_KEY"), - "-o", - "StrictHostKeyChecking=no", - str(file_to_upload), - "{}:{}".format(host, dest_path), - ] - print_message("Sending archive with command {}", command) - subprocess.check_call(command) + command = [ + "scp", + "-c", + "aes128-ctr", + "-rp", + "-P", + port, + "-i", + _environ.get("SSH_KEY"), + "-o", + "StrictHostKeyChecking=no", + str(file_to_upload), + "{}:{}".format(host, dest_path), + ] + print_message("Sending archive with command {}", command) + subprocess.check_call(command) def upload_archive(archive, project, make_release, dev_branch=None): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88f66d68f..e78c60546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Install python modules shell: bash run: | - pip3 install meson pytest requests distro + pip3 install meson pytest requests distro paramiko pip3 install --no-deps $GITHUB_WORKSPACE - name: Setup MSVC compiler uses: bus1/cabuild/action/msdevshell@v1 @@ -118,6 +118,10 @@ jobs: pip3 install --user --no-deps . env: REP: ${{github.repository}} + - name: Install paramiko + if: ${{matrix.image_variant != 'bionic' }} + shell: bash + run: pip3 install --user paramiko - name: secret shell: bash run: | @@ -167,6 +171,7 @@ jobs: git clone https://github.com/${REP} cd ./${REP##*/} git checkout --force ${GITHUB_SHA} + pip3 install --user paramiko pip3 install --user --no-deps . env: REP: ${{github.repository}} @@ -227,7 +232,7 @@ jobs: brew install pkg-config ninja automake autoconf - name: Install python modules run: | - pip3 install meson pytest requests distro + pip3 install meson pytest requests distro paramiko pip3 install --no-deps $GITHUB_WORKSPACE - name: secret shell: bash diff --git a/setup.py b/setup.py index e3fad6cf4..ea3c68b42 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,8 @@ include_package_data=True, install_requires=[ 'meson', - 'distro' + 'distro', + 'paramiko' ], entry_points={ 'console_scripts': [