From 9d2615ffb2b6d69c625ec3bfd50450258e6f21db Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sun, 12 Nov 2023 13:04:04 +0100 Subject: [PATCH] feat(packages): Clean older registry packages To avoid exceed the limit of package storage for the organization on the defined registry, all packages not tagged as latest or version will be removed to the minimum limit defined. Default is allow last 3 intermediary packages (snapshots) remains on registry. Signed-off-by: Helio Chissini de Castro --- .../clean_up_package_registry/action.yml | 61 ++++++++ .../clean_up_package_registry.py | 148 ++++++++++++++++++ .../requirements.txt | 9 ++ .../workflows/clean_up_package_registry.yml | 49 ++++++ 4 files changed, 267 insertions(+) create mode 100644 .github/actions/clean_up_package_registry/action.yml create mode 100644 .github/actions/clean_up_package_registry/clean_up_package_registry.py create mode 100644 .github/actions/clean_up_package_registry/requirements.txt create mode 100644 .github/workflows/clean_up_package_registry.yml diff --git a/.github/actions/clean_up_package_registry/action.yml b/.github/actions/clean_up_package_registry/action.yml new file mode 100644 index 0000000000000..df3a756f6ac20 --- /dev/null +++ b/.github/actions/clean_up_package_registry/action.yml @@ -0,0 +1,61 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: 'Delete old non-release packages from Github package registry' +description: 'Delete older packages set by a minimal level input' +author: 'The ORT Project Authors' + +inputs: + registry: + description: 'Github container registry' + default: 'ghcr.io' + token: + description: 'Github token' + required: true + keep: + description: 'Number of non-release packages to keep' + required: false + default: '3' + packages: + description: 'Name of the packages to be cleaned up' + required: true + dry-run: + description: 'Execute a dry run operation to check the execution is correct' + default: 'false' + +runs: + using: 'composite' + + steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Execute the operation + id: check_image + shell: bash + env: + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_TOKEN: ${{ inputs.token }} + INPUT_KEEP: ${{ inputs.keep }} + INPUT_PACKAGES: ${{ inputs.packages }} + INPUT_DRY_RUN: ${{ inputs.dry-run}} + run: | + pip install -q -U pip requests rich + python ./.github/actions/clean_up_package_registry/clean_up_package_registry.py diff --git a/.github/actions/clean_up_package_registry/clean_up_package_registry.py b/.github/actions/clean_up_package_registry/clean_up_package_registry.py new file mode 100644 index 0000000000000..ccbde8b749bbb --- /dev/null +++ b/.github/actions/clean_up_package_registry/clean_up_package_registry.py @@ -0,0 +1,148 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + + +import os +import sys +from typing import Any +from urllib.parse import parse_qs, urlparse + +import requests +from rich import print + +""" Use current Github API to list packages + in registry and remove all but last 3 or custom + set number of packages. + Reference: https://docs.github.com/en/rest/packages/packages?apiVersion=2022-11-28#about-github-packages +""" + +dry_run: bool = True if os.getenv("INPUT_DRY_RUN") == "true" else False +keep = int(os.getenv("INPUT_KEEP")) +org = os.getenv("GITHUB_REPOSITORY_OWNER") +packages = os.getenv("INPUT_PACKAGES").split("\n") +token = os.getenv("INPUT_TOKEN") + +headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", +} + +# Assembly organization packages url string +pkg_url: str = f"https://api.github.com/orgs/{org}/packages" + + +def get_last_page(headers: dict[str, Any]) -> int: + if "link" not in headers: + return 1 + + links = headers["link"].split(", ") + + last_page = None + for link in links: + if 'rel="last"' in link: + last_page = link + break + + if last_page: + parsed_url = urlparse( + last_page[last_page.index("<") + 1 : last_page.index(">")] + ) + return int(parse_qs(parsed_url.query)["page"][0]) + + return 1 + + +def delete_packages(): + for package in packages: + + # Start page is 1 as stated by documentation + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions?page=1&per_page=50" + + # Get the header + response = requests.head(url, headers=headers) + pages: int | None = get_last_page(response.headers) + print(f":package: {package} - Attempt to delete {pages*50} packes minus restricted") + + for page in range(pages, 0, -1): + print(f"Page: {page}") + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions?page={page}&per_page=50" + response = requests.get(url, headers=headers) + if response.status_code == 404: + print(f":cross_mark: Not found - {url}") + continue + elif response.status_code == 401: + print(f":cross_mark: Requires authentication - {url}") + sys.exit(1) + elif response.status_code == 403: + print(f":cross_mark: Forbidden - {url}") + sys.exit(1) + + # Sort all images on id. + images = sorted(response.json(), key=lambda x: x["id"], reverse=True) + + # Slice and remove all + if len(images) > keep: + for image in images if page != 1 else images[keep + 1 :]: + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions/{image['id']}" + + # Never remove latest or non snapshot tagged images + if restrict_delete_tags(image["metadata"]["container"]["tags"]): + print( + f":package: Skip tagged {package} id {image['id']} tags {image['metadata']['container']['tags']}" + ) + continue + + if not dry_run: + response = requests.delete(url, headers=headers) + if response.status_code == 404: + print(f":cross_mark: Failed to delete package {package} version id {image['id']}.") + continue + elif response.status_code == 401: + print(f":cross_mark: Requires authentication - {url}") + sys.exit(1) + elif response.status_code == 403: + print(f":cross_mark: Forbidden - {url}") + sys.exit(1) + + tags = image["metadata"]["container"]["tags"] + if tags: + print( + f":white_heavy_check_mark: Deleted tagged package {package} version id {image['id']}" + f" with tags {tags}." + ) + else: + print( + f":white_heavy_check_mark: Deleted untagged package {package} version id {image['id']}" + ) + + +def restrict_delete_tags(tags: list) -> bool: + if not tags: + return False + for tag in tags: + if tag == "latest": + return True + elif ".sha." in tag: + return False + elif "SNAPSHOT" in tag: + return False + return False + + +if __name__ == "__main__": + delete_packages() diff --git a/.github/actions/clean_up_package_registry/requirements.txt b/.github/actions/clean_up_package_registry/requirements.txt new file mode 100644 index 0000000000000..65f11586b3d18 --- /dev/null +++ b/.github/actions/clean_up_package_registry/requirements.txt @@ -0,0 +1,9 @@ +certifi==2023.7.22 +charset-normalizer==3.3.2 +idna==3.4 +markdown-it-py==3.0.0 +mdurl==0.1.2 +Pygments==2.16.1 +requests==2.31.0 +rich==13.6.0 +urllib3==2.1.0 diff --git a/.github/workflows/clean_up_package_registry.yml b/.github/workflows/clean_up_package_registry.yml new file mode 100644 index 0000000000000..1f95fec4d6b09 --- /dev/null +++ b/.github/workflows/clean_up_package_registry.yml @@ -0,0 +1,49 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Clean up packages in Github package registry + +on: + workflow_dispatch: + # Runs always Sunday Midnight + schedule: + - cron: "0 0 * * 0" + +jobs: + clean_all: + name: Clean up package registry + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean_up_package_registry + with: + token: ${{ secrets.GITHUB_TOKEN }} + packages: |- + ort + ort-minimal + ort/base + ort/rust + ort/golang + ort/nodejs + ort/python + ort/ruby + ort/dotnet + ort/dart + ort/android + ort/haskell + ort/swift + ort/scala