From d0a15c3e1d2dd63d70313308834a7fe0145f879b Mon Sep 17 00:00:00 2001 From: Maxim Lapan Date: Wed, 6 Dec 2023 14:07:43 +0100 Subject: [PATCH] Retrieve jar function and tests --- exasol/github.py | 51 ++++++++++++++++++++++++++--- test/unit/test_github.py | 71 +++++++++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 12 deletions(-) diff --git a/exasol/github.py b/exasol/github.py index de4a3da..dc33bc1 100644 --- a/exasol/github.py +++ b/exasol/github.py @@ -1,11 +1,24 @@ """ Github-related utility functions - check for latest release of project, retrieval of artefacts, etc. """ +import enum import requests -from typing import Tuple +import pathlib +import logging +from typing import Tuple, Optional +_logger = logging.getLogger(__name__) -def get_latest_version_and_jar_url(project: str) -> Tuple[str, str]: + +class Project(enum.Enum): + """ + Names of github projects to be retrieved. + """ + CLOUD_STORAGE_EXTENSION = "cloud-storage-extension" + KAFKA_CONNECTOR_EXTENSION = "kafka-connector-extension" + + +def get_latest_version_and_jar_url(project: Project) -> Tuple[str, str]: """ Retrieves the latest version of stable project release and url with jar file from the release. @@ -13,16 +26,46 @@ def get_latest_version_and_jar_url(project: str) -> Tuple[str, str]: :param project: name of the project :return: tuple with version and url to retrieve the artefact. """ - r = requests.get(f"https://api.github.com/repos/exasol/{project}/releases/latest") + r = requests.get(f"https://api.github.com/repos/exasol/{project.value}/releases/latest") if r.status_code != 200: raise RuntimeError("Error sending request to the github api, code: %d" % r.status_code) data = r.json() version = data.get('tag_name') if version is None: - raise RuntimeError(f"The latest version of {project} has no tag, something is wrong") + raise RuntimeError(f"The latest version of {project.value} has no tag, something is wrong") for asset in data.get('assets', []): name = asset['name'] if name.endswith(f"{version}.jar"): dl_url = asset['browser_download_url'] return version, dl_url raise RuntimeError("Could not find proper jar url for the latest release") + + +def retrieve_jar(project: Project, use_local_cache: bool = True, + storage_path: Optional[pathlib.Path] = None) -> pathlib.Path: + """ + Returns latest jar file for the project, possibly using local cache. + :param project: project to be used + :param use_local_cache: should local cache be used or file always retrieved anew + :param storage_path: path to be used for downloading. If None, current directory will be used. + :return: path to the jar file on the local filesystem + """ + version, jar_url = get_latest_version_and_jar_url(project) + _, local_jar_name = jar_url.rsplit('/', maxsplit=1) + local_jar_path = pathlib.Path(local_jar_name) + if storage_path is not None: + if not storage_path.exists(): + raise ValueError(f"Local storage path doesn't exist: {storage_path}") + local_jar_path = storage_path / local_jar_path + + if use_local_cache and local_jar_path.exists(): + _logger.info(f"Jar for version {version} already exists in {local_jar_path}, skip downloading") + else: + _logger.info(f"Fetching jar for version {version} from {jar_url}...") + r = requests.get(jar_url, stream=True) + try: + count_bytes = local_jar_path.write_bytes(r.content) + _logger.info(f"Saved {count_bytes} bytes in {local_jar_path}") + finally: + r.close() + return local_jar_path diff --git a/test/unit/test_github.py b/test/unit/test_github.py index 1c8c02b..5df8c89 100644 --- a/test/unit/test_github.py +++ b/test/unit/test_github.py @@ -1,27 +1,82 @@ +import os import pytest +import pathlib +import requests from unittest import mock from exasol import github +CSE_MOCK_URL = "https://github.com/some_path/exasol-cloud-storage-extension-2.7.8.jar" MOCKED_RELEASES_RESULT = { "tag_name": "2.7.8", "assets": [ { "name": "cloud-storage-extension-2.7.8-javadoc.jar", - "browser_download_url": "url1", + "browser_download_url": "should_not_be_used", }, { "name": "exasol-cloud-storage-extension-2.7.8.jar", - "browser_download_url": "url2", + "browser_download_url": CSE_MOCK_URL, } ] } -@mock.patch("requests.get") +def mocked_requests_get(*args, **kwargs): + res = mock.create_autospec(requests.Response) + res.status_code = 404 + url = args[0] + if url.endswith("/releases/latest"): + if github.Project.CLOUD_STORAGE_EXTENSION.value in url: + res.status_code = 200 + res.json = mock.MagicMock(return_value=MOCKED_RELEASES_RESULT) + elif github.Project.KAFKA_CONNECTOR_EXTENSION.value in url: + # used to test error handling + res.status_code = 500 + elif url == CSE_MOCK_URL: + res.status_code = 200 + res.content = b'binary data' + return res + + +@mock.patch("requests.get", side_effect=mocked_requests_get) def test_get_latest_version_and_jar_url(get_mock: mock.MagicMock): - get_mock.return_value = mock.MagicMock() - get_mock.return_value.status_code = 200 - get_mock.return_value.json = mock.MagicMock(return_value=MOCKED_RELEASES_RESULT) - res = github.get_latest_version_and_jar_url("cloud-storage-extension") - assert res == ("2.7.8", "url2") + res = github.get_latest_version_and_jar_url(github.Project.CLOUD_STORAGE_EXTENSION) + assert res == ("2.7.8", CSE_MOCK_URL) + + with pytest.raises(RuntimeError, match="Error sending request"): + github.get_latest_version_and_jar_url(github.Project.KAFKA_CONNECTOR_EXTENSION) + + +@mock.patch("requests.get", side_effect=mocked_requests_get) +def test_retrieve_jar(get_mock: mock.MagicMock, tmpdir, caplog): + # need this as retrieve_jar works with current directory in some cases + os.chdir(tmpdir) + + # fetch for the first time, local dir + jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION) + assert jar_path.exists() + assert jar_path.read_bytes() == b'binary data' + + # ensure file is recreated without cache + old_ts = jar_path.lstat().st_ctime + jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=False) + assert jar_path.exists() + assert old_ts < jar_path.lstat().st_ctime + + # but with enabled cache, file is preserved + caplog.set_level("INFO") + caplog.clear() + old_ts = jar_path.lstat().st_ctime_ns + jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=True) + assert jar_path.lstat().st_ctime_ns == old_ts + assert "skip downloading" in caplog.text + + # test storage path specification + caplog.clear() + stg_path = pathlib.Path(tmpdir.mkdir("sub")) + jar_path_sub = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, + use_local_cache=True, storage_path=stg_path) + assert jar_path_sub.exists() + assert jar_path != jar_path_sub + assert "Fetching jar" in caplog.text