From 2e08c98b675b1049e967dc55892853a3b1459452 Mon Sep 17 00:00:00 2001 From: Lewis Gaul Date: Sun, 13 Oct 2024 21:46:18 +0100 Subject: [PATCH] Remove download docker client functionality (#633) Removes dependency on typer/tqdm/requests by dropping support for downloading the docker client. Initially agreed at https://github.com/gabrieldemarmiesse/python-on-whales/issues/512#issuecomment-1872322431, this has been deprecated since https://github.com/gabrieldemarmiesse/python-on-whales/pull/577 (v0.71.0 released in April 2024), and agreed we can now remove at https://github.com/gabrieldemarmiesse/python-on-whales/issues/552#issuecomment-2331356820. If accepted, this supercedes https://github.com/gabrieldemarmiesse/python-on-whales/pull/513 as a resolution to https://github.com/gabrieldemarmiesse/python-on-whales/issues/512. We will also be able to close https://github.com/gabrieldemarmiesse/python-on-whales/issues/575. --- docs/template/docker_client.md | 26 ++-- pyproject.toml | 3 - python_on_whales/client_config.py | 38 +---- python_on_whales/command_line_entrypoint.py | 63 -------- python_on_whales/download_binaries.py | 137 ------------------ requirements.txt | 3 - .../test_download_binaries.py | 52 ------- 7 files changed, 18 insertions(+), 304 deletions(-) delete mode 100644 python_on_whales/command_line_entrypoint.py delete mode 100644 python_on_whales/download_binaries.py delete mode 100644 tests/python_on_whales/test_download_binaries.py diff --git a/docs/template/docker_client.md b/docs/template/docker_client.md index cdff0c20..b012e5fc 100644 --- a/docs/template/docker_client.md +++ b/docs/template/docker_client.md @@ -92,29 +92,25 @@ In the end, unless you use this type of logic in your code, Python-on-whales is safe to use with multithreading and multiprocessing. -# The Docker CLI +# The Docker/Podman CLI -Python-on-whales needs the Docker CLI to work (unlike docker-py). +Python-on-whales needs the Docker or Podman CLI to work (unlike docker-py). Most of the time, users already have the CLI installed on their machines. It's possible to -verify that the CLI is there by doing `docker --help` in the command line. +verify that the CLI is there by doing `docker --help` (or `podman --help`) in the command line. Sometimes, the CLI might not be available on the system, it can happen if you want to control Docker from within a container with `-v /var/run/docker.sock:/var/run/docker.sock`, or if you want to connect to a remote daemon with the `host` argument. -In this case, when using python-on-whales, the CLI will be downloaded automatically -(it's a single binary), and will be put in -```python -pathlib.Path.home() / ".cache/python-on-whales/docker" -``` -Since it's not in the PATH and was not downloaded with the package manager, it's only seen and -used by python-on-whales. +Instructions for installing Docker can be found at , +and Podman at . Note that if connecting to Docker/Podman +remotely then the Docker daemon (`dockerd`) is not needed, and similarly for Podman it is +possible to use `podman-remote` (available as a static binary from +). + +Previously, when using python-on-whales, the Docker CLI was downloaded automatically, but +this functionality was removed under . -If you want to trigger the download manually (to avoid downloading the CLI at runtime), -you can run from your shell: -```bash -python-on-whales download-cli -``` # Handling an unavailable client diff --git a/pyproject.toml b/pyproject.toml index 63b3dbfe..81f4d018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,9 +23,6 @@ dev = ["ruff==0.5.6"] "Documentation" = "https://gabrieldemarmiesse.github.io/python-on-whales/" "Bug Tracker" = "https://github.com/gabrieldemarmiesse/python-on-whales/issues" -[project.scripts] -python-on-whales = "python_on_whales.command_line_entrypoint:main" - [tool.setuptools.packages.find] exclude = ["tests*", "docs*"] diff --git a/python_on_whales/client_config.py b/python_on_whales/client_config.py index d6663e8a..dd7edcc9 100644 --- a/python_on_whales/client_config.py +++ b/python_on_whales/client_config.py @@ -9,11 +9,6 @@ import pydantic -from python_on_whales.download_binaries import ( - download_docker_cli, - get_docker_binary_path_in_cache, -) - from . import utils from .utils import ValidPath, run, to_list @@ -92,32 +87,13 @@ def get_client_call_with_path(self) -> List[Union[Path, str]]: def _get_docker_path(self) -> str: which_result = shutil.which(self.client_call[0]) - if which_result is not None: - return which_result - if self.client_call[0] == "docker": - if not get_docker_binary_path_in_cache().exists(): - warnings.warn( - "The docker client binary file was not found on your system. \n" - "Docker on whales will try to download it for you. \n" - "Don't worry, it " - "won't be in the PATH and won't have anything to do with " - "the package manager of your system. \n" - "Note: We are not installing the docker daemon, which is a lot " - "heavier and harder to install. We're just downloading a single " - "standalone binary file.\n" - "If you want to trigger the download of the client binary file " - "manually (for example if you want to do it in a Dockerfile), " - "you can run the following command:\n " - "$ python-on-whales download-cli \n" - ) - download_docker_cli() - return get_docker_binary_path_in_cache() - - raise ClientNotFoundError( - f"The binary '{self.client_call[0]}' could not be found on your PATH. " - f"Please ensure that your PATH is has the directory of the binary you're looking for. " - f"You can use `print(os.environ['PATH'])` to verify what directories are in your PATH." - ) + if which_result is None: + raise ClientNotFoundError( + f"The binary '{self.client_call[0]}' could not be found on your PATH. " + "Please ensure that your PATH has the directory of the binary you're looking for. " + "You can use `print(os.environ['PATH'])` to verify what directories are in your PATH." + ) + return which_result @property def docker_cmd(self) -> Command: diff --git a/python_on_whales/command_line_entrypoint.py b/python_on_whales/command_line_entrypoint.py deleted file mode 100644 index b1964ec9..00000000 --- a/python_on_whales/command_line_entrypoint.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import Optional - -import typer - -from python_on_whales import docker -from python_on_whales.download_binaries import download_docker_cli - -app = typer.Typer() - - -volume_app = typer.Typer() - - -@volume_app.command() -def copy(source: str, destination: str): - if ":" in source: - source_volume, source_path = source.split(":") - docker.volume.copy((source_volume, source_path), destination) - elif ":" in destination: - destination_volume, destination_path = destination.split(":") - docker.volume.copy(source, (destination_volume, destination_path)) - else: - raise ValueError("No volume was specified. The format is 'volume:path'") - - -image_app = typer.Typer() - - -@image_app.command() -def copy_from(docker_image: str, source: str, destination: str, pull: str = "missing"): - docker.image.copy_from(docker_image, source, destination, pull) - - -@image_app.command() -def copy_to( - docker_image: str, - source: str, - destination: str, - new_tag: Optional[str] = None, - push: bool = False, - pull: str = "missing", -): - docker.image.copy_to(docker_image, source, destination, new_tag, pull) - if push: - docker.image.push(new_tag) - - -@app.command() -def download_cli(): - download_docker_cli() - - -@app.command() -def download_buildx(): - raise NotImplementedError("Downloading the buildx binary isn't supported yet.") - - -app.add_typer(volume_app, name="volume") -app.add_typer(image_app, name="image") - - -def main(): - app() diff --git a/python_on_whales/download_binaries.py b/python_on_whales/download_binaries.py deleted file mode 100644 index a065d471..00000000 --- a/python_on_whales/download_binaries.py +++ /dev/null @@ -1,137 +0,0 @@ -__all__ = ("download_docker_cli", "get_docker_binary_path_in_cache") - -import platform -import shutil -import tempfile -import warnings -from pathlib import Path - -import requests -from tqdm import tqdm - -DOCKER_VERSION = "20.10.5" -BUILDX_VERSION = "0.5.1" - -CACHE_DIR = Path.home() / ".cache" / "python-on-whales" - -TEMPLATE_CLI = ( - "https://download.docker.com/{os}/static/stable/{arch}/docker-{version}.tgz" -) -WINDOWS_CLI_URL = "https://github.com/StefanScherer/docker-cli-builder/releases/download/{version}/docker.exe" - - -def get_docker_binary_path_in_cache(): - return CACHE_DIR / "docker-cli" / DOCKER_VERSION / "docker" - - -def get_docker_cli_url(): - user_os = get_user_os() - if user_os == "windows": - return WINDOWS_CLI_URL.format(version=DOCKER_VERSION) - arch = get_arch_for_docker_cli_url() - return TEMPLATE_CLI.format(os=user_os, arch=arch, version=DOCKER_VERSION) - - -def download_docker_cli(): - warnings.warn( - "Downloading the docker client with python-on-whales is being " - "deprecated, and this functionality will be removed in v1.0. " - "See https://github.com/gabrieldemarmiesse/python-on-whales/issues/556 " - "for planned v1.0 changes.", - DeprecationWarning, - ) - file_to_download = get_docker_cli_url() - - extension = file_to_download.split(".")[-1] - with tempfile.TemporaryDirectory() as tmp_dir: - tmp_dir = Path(tmp_dir) - downloaded_file_path = tmp_dir / f"docker.{extension}" - download_from_url(file_to_download, downloaded_file_path) - - docker_binary_path = get_docker_binary_path_in_cache() - docker_binary_path.parent.mkdir(exist_ok=True, parents=True) - - if extension == "tgz": - extract_dir = tmp_dir / "extracted" - shutil.unpack_archive(str(downloaded_file_path), str(extract_dir)) - shutil.move(extract_dir / "docker" / "docker", docker_binary_path) - elif extension == "exe": - shutil.move(downloaded_file_path, docker_binary_path) - - warnings.warn( - f"The docker client binary file {DOCKER_VERSION} was downloaded and put " - f"in `{docker_binary_path.absolute()}`. \n" - f"You can feel free to remove it if you wish, Python on whales will download " - f"it again if needed." - ) - - -def download_from_url(url, dst): - try: - _download_from_url(url, dst) - except Exception as e: - raise ConnectionError(f"Error while downloading {url}") from e - - -def _download_from_url(url, dst): - # Streaming, so we can iterate over the response. - response = requests.get(url, stream=True) - total_size_in_bytes = int(response.headers.get("content-length", 0)) - block_size = 1024 - progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) - with open(dst, "wb") as file: - for data in response.iter_content(block_size): - progress_bar.update(len(data)) - file.write(data) - progress_bar.close() - if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes: - raise ConnectionError( - f"Total size should be {total_size_in_bytes}, downloaded {progress_bar.n}" - ) - - -def get_user_os(): - user_os = platform.system() - if user_os == "Linux": - return "linux" - elif user_os == "Darwin": - return "mac" - elif user_os == "Windows": - return "windows" - else: - raise NotImplementedError( - f"Unknown OS: {user_os}, cannot determine which Docker CLI binary file to " - f"download. \n" - f"Please open an issue at \n" - f"https://github.com/gabrieldemarmiesse/python-on-whales/issues \n" - f"and in the meantime, install Docker manually to make python-on-whales " - f"work." - ) - - -def get_arch_for_docker_cli_url(): - arch = platform.architecture()[0] - - # I don't know the exact list of possible architectures, - # so if a user reports a NotImplementedError, we can easily add - # his/her platform here. - arch_mapping = { - "NotImplementedError": "aarch64", - "NotImplementedError2": "armel", - "NotImplementedError3": "armhf", - "NotImplementedError4": "ppc64le", - "NotImplementedError5": "s390x", - "64bit": "x86_64", - } - - try: - return arch_mapping[arch] - except KeyError: - raise NotImplementedError( - f"The architecture detected on your system is `{arch}`, the list of " - f"available architectures is {list(arch_mapping.values())}. \n" - f"Please open an issue at \n" - f"https://github.com/gabrieldemarmiesse/python-on-whales/issues " - f"and make sure to copy past this error message. \n" - f"In the meantime, install Docker manually on your system." - ) diff --git a/requirements.txt b/requirements.txt index 59ed2b33..f676afa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ pydantic>=2,<3,!=2.0.* -requests -tqdm -typer>=0.4.1 typing_extensions diff --git a/tests/python_on_whales/test_download_binaries.py b/tests/python_on_whales/test_download_binaries.py deleted file mode 100644 index b48d4a0c..00000000 --- a/tests/python_on_whales/test_download_binaries.py +++ /dev/null @@ -1,52 +0,0 @@ -import platform - -from python_on_whales import download_binaries -from python_on_whales.utils import run - - -def test_download_from_url(tmp_path): - url = "https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_92x30dp.png" - download_binaries.download_from_url(url, tmp_path / "dodo.png") - assert (tmp_path / "dodo.png").exists() - - -def test_download_cli(mocker, tmp_path): - mocker.patch.object(download_binaries, "CACHE_DIR", tmp_path) - - download_binaries.download_docker_cli() - simple_command = [ - download_binaries.get_docker_binary_path_in_cache(), - "run", - "--rm", - "hello-world", - ] - output = run(simple_command, capture_stdout=True, capture_stderr=True) - assert "Hello from Docker!" in output - - -def test_download_cli_from_cli(): - try: - download_binaries.get_docker_binary_path_in_cache().unlink() - except FileNotFoundError: - pass - run(["python-on-whales", "download-cli"]) - simple_command = [ - download_binaries.get_docker_binary_path_in_cache(), - "run", - "--rm", - "hello-world", - ] - output = run(simple_command, capture_stdout=True, capture_stderr=True) - assert "Hello from Docker!" in output - - -def test_download_windows_binaries(mocker, tmp_path): - mocker.patch.object(platform, "system", lambda: "Windows") - mocker.patch.object(download_binaries, "CACHE_DIR", tmp_path) - - assert not download_binaries.get_docker_binary_path_in_cache().exists() - assert download_binaries.get_user_os() == "windows" - - download_binaries.download_docker_cli() - - assert download_binaries.get_docker_binary_path_in_cache().exists()