Add bundle map

.github/workflows/publish_draft_bundle.yml

@@ -127,3 +127,30 @@ jobs:
           title="Test Install Successful"
           message="Installation and version command run successful for os_platform=${{ matrix.os }}, Python=${{ matrix.python-version }}, version=${{ inputs.tag }}"
           echo "::notice $title::$message"
+
+  update-latest-bundle-map:
+    needs: [test-bundle]
+    name: "Update bundle map with ${{ inputs.tag }}"
+
+    steps:
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+
+      - name: "Set up Python 3.8"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: "Install release_creation and deps"
+        run: |
+          python -m pip install --user --upgrade pip
+          pip install -r requirements.txt
+          python -m pip install -e . 
+
+      - name: "Run update latest map"
+        run: |
+          python -u -m release_creation.main \
+           --input-version=${{ inputs.tag }} \
+           --operation=update_latest_bundle
+
+

latest_bundle.yml

@@ -0,0 +1,8 @@
+release_version:
+  '1.0': 1.0.30
+  '1.1': 1.1.21
+  '1.2': 1.2.19
+  '1.3': 1.3.19
+  '1.4': 1.4.19
+  '1.5': 1.5.20
+  '1.7': 1.17.21

release_creation/bundle_map.py

@@ -0,0 +1,47 @@
+import os
+from typing import List
+
+import github
+from semantic_version import Version
+import yaml
+
+from release_creation.github_client.client import get_bundle_repo, update_file
+
+_BUNDLE_MAP_FILE_NAME = "latest_bundle.yml"
+_BUNDLE_MAP_FILE_BRANCH = os.getenv("BUNDLE_MAP_FILE_BRANCH", "main")
+
+
+def _get_release_version(bundle_version: Version):
+    return f"{bundle_version.major}.{bundle_version.minor}"
+
+
+def get_latest_bundle_from_map(bundle_version: Version):
+    release_version = _get_release_version(bundle_version)
+    repo = get_bundle_repo()
+    bundle_map_file = repo.get_contents(_BUNDLE_MAP_FILE_NAME,
+                                        ref=_BUNDLE_MAP_FILE_BRANCH)
+    bundles = yaml.safe_load(bundle_map_file.decoded_content)
+    latest = bundles['release_version'][release_version]
+    print(latest)
+    return latest
+
+
+def update_latest_bundles_in_map(bundle_versions: List[Version]):
+    repo = get_bundle_repo()
+    github.enable_console_debug_logging()
+    bundle_map_file = repo.get_contents(_BUNDLE_MAP_FILE_NAME,
+                                        ref=_BUNDLE_MAP_FILE_BRANCH)
+    bundles = yaml.safe_load(bundle_map_file.decoded_content)
+    updated = []
+    for bundle_version in bundle_versions:
+        release_version = _get_release_version(bundle_version)
+        bundles['release_version'][release_version] = str(bundle_version)
+        updated.append(f"{release_version}={bundle_version}")
+    update_file(
+        repo=repo,
+        path=bundle_map_file.path,
+        message="Updated: " + ", ".join(updated),
+        branch=_BUNDLE_MAP_FILE_BRANCH,
+        content=yaml.safe_dump(bundles),
+        sha=bundle_map_file.sha
+    )

release_creation/github_client/client.py

@@ -1,16 +1,54 @@
-import os
+import os, base64
 
 from github import Github
 from github.Repository import Repository
 from urllib3 import Retry
+from requests import request
 
 _GH_BUNDLE_REPO = "dbt-labs/dbt-core-bundles"
 _RETRY = Retry(total=5, backoff_factor=0.2)
 
+
 def get_github_client() -> Github:
     return Github(os.environ.get("GH_ACCESS_TOKEN"), retry=_RETRY)
 
 
 def get_bundle_repo() -> Repository:
     gh = get_github_client()
     return gh.get_repo(_GH_BUNDLE_REPO)
+
+
+def update_file(repo: Repository, path: str, message: str, branch: str, content: str, sha: str):
+    """
+    Note: this requires a PAT with repo scope be set in the GH_ACCESS_TOKEN env var
+
+    Args:
+        repo:
+        path:
+        message:
+        branch:
+        content:
+        sha:
+
+    Returns: Status code of the request
+
+    """
+    url = f"https://api.github.com/repos/{repo.full_name}/contents/{path}?ref={branch}"
+    body = {
+        "message": message,
+        "branch": branch,
+        "content": base64.b64encode(content.encode()).decode(),
+        "sha": sha,
+    }
+
+    response = request(
+        method="PUT",
+        url=url,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {os.environ.get('GH_ACCESS_TOKEN')}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+        json=body,
+    )
+    return response.raise_for_status()

release_creation/main.py

@@ -3,6 +3,7 @@
 import os
 import argparse
 
+from release_creation.bundle_map import get_latest_bundle_from_map, update_latest_bundles_in_map
 from release_creation.github_client import create_release
 from release_creation.bundle import create
 from release_creation.github_client import get_release
@@ -16,6 +17,8 @@
 class ReleaseOperations(StrEnum):
     create = "create"
     update = "update"
+    get_latest = "get_latest_bundle"
+    update_latest = "update_latest_bundle"
 
 
 def set_output(name, value):
@@ -86,7 +89,11 @@ def main():
         execute_create_bundle_release(version)
     elif operation == ReleaseOperations.update:
         execute_update_bundle_release(version)
-
+    elif operation == ReleaseOperations.get_latest:
+        get_latest_bundle_from_map(Version(version))
+    elif operation == ReleaseOperations.update_latest:
+        versions = [Version(v) for v in version.split(",")]
+        update_latest_bundles_in_map(versions)
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -1,5 +1,5 @@
 StrEnum
 requests
 semantic-version~=2.10.0
-PyGithub~=1.0
+PyGithub==2.1.1
 pydantic

test/integration/test_bundle_map.py

@@ -0,0 +1,17 @@
+from unittest.mock import patch
+
+from semantic_version import Version
+
+from release_creation.bundle_map import get_latest_bundle_from_map, update_latest_bundles_in_map
+
+
+@patch("release_creation.bundle_map._BUNDLE_MAP_FILE_BRANCH", "test")
+def test_map_is_gettable_and_updateable():
+    version = "1.1.0"
+    current_version = get_latest_bundle_from_map(Version(version))
+    current_version = Version(current_version)
+    to_update = current_version.next_patch()
+    update_latest_bundles_in_map([to_update])
+    updated_version = get_latest_bundle_from_map(Version(version))
+    assert updated_version == str(to_update)
+

test/unit/test_bundle_map.py

@@ -0,0 +1,53 @@
+from unittest.mock import patch, Mock, ANY
+
+import yaml
+from semantic_version import Version
+
+from release_creation.bundle_map import get_latest_bundle_from_map, update_latest_bundles_in_map
+
+test_map = yaml.safe_load("""
+                    release_version:
+                      '1.0': 1.0.30
+                      '1.1': 1.1.19
+                      '1.2': 1.2.19
+                    """)
+
+
+def mock_safe_load(*args, **kwargs):
+    return test_map
+
+
+def mock_repo_get_contents(*args, **kwargs):
+    return Mock(decoded_content=yaml.safe_dump(test_map))
+
+
+_MOCK_REPO_UPDATE = Mock()
+_MOCK_REPO = Mock(get_contents=mock_repo_get_contents, update_file=_MOCK_REPO_UPDATE)
+
+
+@patch("release_creation.bundle_map.get_bundle_repo", Mock(return_value=_MOCK_REPO))
+def test_get_latest_bundle_from_map_with_valid_version():
+    bundle_version = Version.coerce("1.1.0")
+    expected_version = "1.1.19"
+
+    actual_version = get_latest_bundle_from_map(bundle_version)
+
+    assert actual_version == expected_version
+
+
+@patch("release_creation.bundle_map.get_bundle_repo", Mock(return_value=_MOCK_REPO))
+def test_update_latest_bundles_in_map():
+    bundle_versions = [Version.coerce("1.1.20"), Version.coerce("1.3.0")]
+    expected_map = yaml.safe_load("""
+                    release_version:
+                        '1.0': 1.0.30
+                        '1.1': 1.1.20
+                        '1.2': 1.2.19
+                        '1.3': 1.3.0
+                    """)
+
+    update_latest_bundles_in_map(bundle_versions)
+    calls = _MOCK_REPO.mock_calls
+    kwargs_call = calls[0].kwargs
+    assert yaml.safe_load(kwargs_call['content']) == expected_map
+    assert kwargs_call['message'] == "Updated: 1.1=1.1.20, 1.3=1.3.0"