From 52be3e93555b2aab60a50fd690206e5661e6c0ec Mon Sep 17 00:00:00 2001 From: Helio Chissini de Castro Date: Sun, 12 Nov 2023 13:04:04 +0100 Subject: [PATCH] feat(packages): Clean older registry packages To avoid exceed the limit of package storage for the organization on the defined registry, all packages not tagged as latest or version will be removed to the minimum limit defined. Default is allow last 3 intermediary packages (snapshots) remains on registry. Signed-off-by: Helio Chissini de Castro --- .github/actions/clean_packages/action.yml | 61 +++++++++++ .../actions/clean_packages/clean_packages.py | 101 ++++++++++++++++++ .github/workflows/clean_packages.yml | 52 +++++++++ 3 files changed, 214 insertions(+) create mode 100644 .github/actions/clean_packages/action.yml create mode 100644 .github/actions/clean_packages/clean_packages.py create mode 100644 .github/workflows/clean_packages.yml diff --git a/.github/actions/clean_packages/action.yml b/.github/actions/clean_packages/action.yml new file mode 100644 index 0000000000000..9f102c4fcafb6 --- /dev/null +++ b/.github/actions/clean_packages/action.yml @@ -0,0 +1,61 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: 'Clean Github registry older packages in the organization' +description: 'Delete older packages set by a minimal level input' +author: 'The ORT Project Authors' + +inputs: + registry: + description: 'GitHub container registry' + default: 'ghcr.io' + token: + description: 'GitHub token' + required: true + keep: + description: 'Number of packages we keep after latest' + required: false + default: 3 + packages: + description: 'Name of the packages should be clean' + required: true + dry-run: + description: 'Execute a dry run operation to check the execution is correct' + default: false + +runs: + using: 'composite' + + steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Execute the operation + id: check_image + shell: bash + env: + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_TOKEN: ${{ inputs.token }} + INPUT_KEEP: ${{ inputs.keep }} + INPUT_PACKAGES: ${{ inputs.packages }} + INPUT_DRY_RUN: ${{ inputs.dry-run}} + run: | + pip install -q -U pip requests rich + python ./.github/actions/clean_packages/clean_packages.py diff --git a/.github/actions/clean_packages/clean_packages.py b/.github/actions/clean_packages/clean_packages.py new file mode 100644 index 0000000000000..bb1bb911ccb00 --- /dev/null +++ b/.github/actions/clean_packages/clean_packages.py @@ -0,0 +1,101 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + + +import os + +import requests +from rich import print + +""" Use current GitHub API to check to llst packages + in registry and remove all but last 3 or custom + set number of packages. + Reference: https://docs.github.com/en/rest/packages/packages?apiVersion=2022-11-28#about-github-packages +""" + +dry_run: bool = True if os.getenv("INPUT_DRY_RUN") == "true" else False +keep = int(os.getenv("INPUT_KEEP")) +org = os.getenv("GITHUB_REPOSITORY_OWNER") +packages = os.getenv("INPUT_PACKAGES").split("\n") +token = os.getenv("INPUT_TOKEN") + +print(packages) + +headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", +} + +# Assembly organization packages url string +pkg_url: str = f"https://api.github.com/orgs/{org}/packages" + + +def remove_packages(): + for package in packages: + print(f":package: {package}") + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions?per_page=100" + response = requests.get(url, headers=headers) + + if response.status_code == 404: + print(f":cross_mark: Not found - {url}") + continue + + # Sort all images on id based. + images = sorted(response.json(), key=lambda x: x["id"], reverse=True) + + # Slice and remove all + if len(images) > keep: + for image in images[keep + 1 :]: + url = f"{pkg_url}/container/{package.replace('/', '%2F')}/versions/{image['id']}" + + # Never remove latest or non snapshot tagged images + if restrict_delete_tags(image["metadata"]["container"]["tags"]): + print( + f":package: Skip tagged {package} id {image['id']} tags {image['metadata']['container']['tags']}" + ) + continue + + if not dry_run: + response = requests.delete(url, headers=headers) + if response.status_code != 204: + print( + f":cross_mark: Failed to delete package {package} version id {image['id']}." + ) + continue + print( + f":white_heavy_check_mark: Deleted package {package} version id {image['id']}." + ) + + +def restrict_delete_tags(tags: list) -> bool: + if not tags: + return False + for tag in tags: + if tag == "latest": + return True + elif ".sha." in tag: + return False + elif "SNAPSHOT" in tag: + return False + else: + return True + return False + + +if __name__ == "__main__": + remove_packages() diff --git a/.github/workflows/clean_packages.yml b/.github/workflows/clean_packages.yml new file mode 100644 index 0000000000000..01c547472ca09 --- /dev/null +++ b/.github/workflows/clean_packages.yml @@ -0,0 +1,52 @@ +# Copyright (C) 2023 The ORT Project Authors (see ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# License-Filename: LICENSE + +name: Clean packages in registry + +on: + workflow_dispatch: + # Runs always Sunday Midnight + schedule: + - cron: '0 0 * * 0' + +jobs: + clean_all: + name: Cleaning older packages + runs-on: ubuntu-22.04 + steps: + - name: Checkout default branch + uses: actions/checkout@v4 + - name: Clean Packages + uses: ./.github/actions/clean_packages + with: + token: ${{ secrets.GITHUB_TOKEN }} + packages: |- + ort + ort-minimal + ort-extended + ort/base + ort/rust + ort/golang + ort/nodejs + ort/python + ort/ruby + ort/dotnet + ort/dart + ort/android + ort/haskell + ort/swift + ort/scala