Retrieve jar function and tests

exasol/github.py

@@ -1,28 +1,71 @@
 """
 Github-related utility functions - check for latest release of project, retrieval of artefacts, etc.
 """
+import enum
 import requests
-from typing import Tuple
+import pathlib
+import logging
+from typing import Tuple, Optional
 
+_logger = logging.getLogger(__name__)
 
-def get_latest_version_and_jar_url(project: str) -> Tuple[str, str]:
+
+class Project(enum.Enum):
+    """
+    Names of github projects to be retrieved.
+    """
+    CLOUD_STORAGE_EXTENSION = "cloud-storage-extension"
+    KAFKA_CONNECTOR_EXTENSION = "kafka-connector-extension"
+
+
+def get_latest_version_and_jar_url(project: Project) -> Tuple[str, str]:
     """
     Retrieves the latest version of stable project release
     and url with jar file from the release.
 
     :param project: name of the project
     :return: tuple with version and url to retrieve the artefact.
     """
-    r = requests.get(f"https://api.github.com/repos/exasol/{project}/releases/latest")
+    r = requests.get(f"https://api.github.com/repos/exasol/{project.value}/releases/latest")
     if r.status_code != 200:
         raise RuntimeError("Error sending request to the github api, code: %d" % r.status_code)
     data = r.json()
     version = data.get('tag_name')
     if version is None:
-        raise RuntimeError(f"The latest version of {project} has no tag, something is wrong")
+        raise RuntimeError(f"The latest version of {project.value} has no tag, something is wrong")
     for asset in data.get('assets', []):
         name = asset['name']
         if name.endswith(f"{version}.jar"):
             dl_url = asset['browser_download_url']
             return version, dl_url
     raise RuntimeError("Could not find proper jar url for the latest release")
+
+
+def retrieve_jar(project: Project, use_local_cache: bool = True,
+                 storage_path: Optional[pathlib.Path] = None) -> pathlib.Path:
+    """
+    Returns latest jar file for the project, possibly using local cache.
+    :param project: project to be used
+    :param use_local_cache: should local cache be used or file always retrieved anew
+    :param storage_path: path to be used for downloading. If None, current directory will be used.
+    :return: path to the jar file on the local filesystem
+    """
+    version, jar_url = get_latest_version_and_jar_url(project)
+    _, local_jar_name = jar_url.rsplit('/', maxsplit=1)
+    local_jar_path = pathlib.Path(local_jar_name)
+    if storage_path is not None:
+        if not storage_path.exists():
+            raise ValueError(f"Local storage path doesn't exist: {storage_path}")
+        local_jar_path = storage_path / local_jar_path
+
+    if use_local_cache and local_jar_path.exists():
+        _logger.info(f"Jar for version {version} already exists in {local_jar_path}, skip downloading")
+    else:
+        _logger.info(f"Fetching jar for version {version} from {jar_url}...")
+        r = requests.get(jar_url, stream=True)
+        try:
+            count_bytes = local_jar_path.write_bytes(r.content)
+            _logger.info(f"Saved {count_bytes} bytes in {local_jar_path}")
+        finally:
+            r.close()
+    return local_jar_path

test/unit/test_github.py

@@ -1,27 +1,82 @@
+import os
 import pytest
+import pathlib
+import requests
 from unittest import mock
 from exasol import github
 
+CSE_MOCK_URL = "https://github.com/some_path/exasol-cloud-storage-extension-2.7.8.jar"
 
 MOCKED_RELEASES_RESULT = {
     "tag_name": "2.7.8",
     "assets": [
         {
             "name": "cloud-storage-extension-2.7.8-javadoc.jar",
-            "browser_download_url": "url1",
+            "browser_download_url": "should_not_be_used",
         },
         {
             "name": "exasol-cloud-storage-extension-2.7.8.jar",
-            "browser_download_url": "url2",
+            "browser_download_url": CSE_MOCK_URL,
         }
     ]
 }
 
 
-@mock.patch("requests.get")
+def mocked_requests_get(*args, **kwargs):
+    res = mock.create_autospec(requests.Response)
+    res.status_code = 404
+    url = args[0]
+    if url.endswith("/releases/latest"):
+        if github.Project.CLOUD_STORAGE_EXTENSION.value in url:
+            res.status_code = 200
+            res.json = mock.MagicMock(return_value=MOCKED_RELEASES_RESULT)
+        elif github.Project.KAFKA_CONNECTOR_EXTENSION.value in url:
+            # used to test error handling
+            res.status_code = 500
+    elif url == CSE_MOCK_URL:
+        res.status_code = 200
+        res.content = b'binary data'
+    return res
+
+
+@mock.patch("requests.get", side_effect=mocked_requests_get)
 def test_get_latest_version_and_jar_url(get_mock: mock.MagicMock):
-    get_mock.return_value = mock.MagicMock()
-    get_mock.return_value.status_code = 200
-    get_mock.return_value.json = mock.MagicMock(return_value=MOCKED_RELEASES_RESULT)
-    res = github.get_latest_version_and_jar_url("cloud-storage-extension")
-    assert res == ("2.7.8", "url2")
+    res = github.get_latest_version_and_jar_url(github.Project.CLOUD_STORAGE_EXTENSION)
+    assert res == ("2.7.8", CSE_MOCK_URL)
+
+    with pytest.raises(RuntimeError, match="Error sending request"):
+        github.get_latest_version_and_jar_url(github.Project.KAFKA_CONNECTOR_EXTENSION)
+
+
+@mock.patch("requests.get", side_effect=mocked_requests_get)
+def test_retrieve_jar(get_mock: mock.MagicMock, tmpdir, caplog):
+    # need this as retrieve_jar works with current directory in some cases
+    os.chdir(tmpdir)
+
+    # fetch for the first time, local dir
+    jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION)
+    assert jar_path.exists()
+    assert jar_path.read_bytes() == b'binary data'
+
+    # ensure file is recreated without cache
+    old_ts = jar_path.lstat().st_ctime
+    jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=False)
+    assert jar_path.exists()
+    assert old_ts < jar_path.lstat().st_ctime
+
+    # but with enabled cache, file is preserved
+    caplog.set_level("INFO")
+    caplog.clear()
+    old_ts = jar_path.lstat().st_ctime_ns
+    jar_path = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION, use_local_cache=True)
+    assert jar_path.lstat().st_ctime_ns == old_ts
+    assert "skip downloading" in caplog.text
+
+    # test storage path specification
+    caplog.clear()
+    stg_path = pathlib.Path(tmpdir.mkdir("sub"))
+    jar_path_sub = github.retrieve_jar(github.Project.CLOUD_STORAGE_EXTENSION,
+                                       use_local_cache=True, storage_path=stg_path)
+    assert jar_path_sub.exists()
+    assert jar_path != jar_path_sub
+    assert "Fetching jar" in caplog.text