From 159ae722ae0988f3ea587e4f6549bf7e7f0e27cd Mon Sep 17 00:00:00 2001 From: Avi Biton Date: Mon, 5 Feb 2024 09:53:53 +0200 Subject: [PATCH] chore(RHTAPWATCH-717): update image reference in tekton-tools update image reference in tekton-tools Signed-off-by: Avi Biton --- .tekton/tools-pull-request.yaml | 104 +++------- tasks/create-pull-request.yaml | 324 ++++++++++++++++++++++++++++++++ 2 files changed, 348 insertions(+), 80 deletions(-) create mode 100644 tasks/create-pull-request.yaml diff --git a/.tekton/tools-pull-request.yaml b/.tekton/tools-pull-request.yaml index 100affd..4b0363c 100644 --- a/.tekton/tools-pull-request.yaml +++ b/.tekton/tools-pull-request.yaml @@ -9,6 +9,7 @@ metadata: pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch == "main" + pipelinesascode.tekton.dev/task: tasks/create-pull-request.yaml creationTimestamp: null labels: appstudio.openshift.io/application: tools @@ -30,6 +31,8 @@ spec: value: . - name: revision value: '{{revision}}' + - name: pr-url + value: '{{body.head_commit.url}}' pipelineSpec: finally: - name: show-sbom @@ -291,92 +294,33 @@ spec: operator: in values: - "false" - - name: clair-scan - params: - - name: image-digest - value: $(tasks.build-container.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container - taskRef: - params: - - name: name - value: clair-scan - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:63b42c0fc23d05e26776a0e7c4f0ab00750096ebfe1eed9a7ba96f8b27713fbf - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-snyk-check + - name: update-image-ref runAfter: - clone-repository - taskRef: - params: - - name: name - value: sast-snyk-check - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:47515cb119225bba55c593876610bd890f8efcbb66bb57fb0c0881ddd47ce558 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" workspaces: - - name: workspace + - name: artifacts workspace: workspace - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-container.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container taskRef: - params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sbom-json-check + name: create-pull-request + kind: Task params: - - name: IMAGE_URL - value: $(tasks.build-container.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-container.results.IMAGE_DIGEST) - runAfter: - - build-container - taskRef: - params: - - name: name - value: sbom-json-check - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:bf49861b3bbee2129e8d1b5966fc2a7c3f259d96a5fcef5674d05c9cb21ab540 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" + - name: ORIGIN_REPO + value: $(params.git-url) + - name: REVISION + value: $(params.revision) + - name: ARTIFACTS_PATH + value: /workspace/artifacts/source + - name: TARGET_GH_REPO + value: redhat-appstudio/tekton-tools + - name: PR_URL + value: $(params.pr-url) + - name: image + value: $(params.output-image) + - name: SCRIPT + value: | + set -x + pwd + ls -laR workspaces: - name: workspace - name: git-auth diff --git a/tasks/create-pull-request.yaml b/tasks/create-pull-request.yaml new file mode 100644 index 0000000..629995b --- /dev/null +++ b/tasks/create-pull-request.yaml @@ -0,0 +1,324 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "appstudio, hacbs" + name: create-pull-request +spec: + description: | + Clones the target repository, runs script in 'SCRIPT' parameter, generates + pull-request for target repository. + params: + - name: SCRIPT + description: Bash script for changing the target repo + - name: ORIGIN_REPO + description: URL of github repository which was built by the Pipeline + - name: REVISION + description: Git reference which was built by the Pipeline + - name: TARGET_GH_REPO + description: GitHub repository to push changes to + - name: GIT_IMAGE + description: Image reference containing the git command + default: registry.redhat.io/openshift-pipelines/pipelines-git-init-rhel8:v1.8.2-8@sha256:a538c423e7a11aae6ae582a411fdb090936458075f99af4ce5add038bb6983e8 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. + - name: SCRIPT_IMAGE + description: Image reference for SCRIPT execution + # this image is built using https://github.com/redhat-appstudio/build-tasks-dockerfiles/blob/main/update-infra-deployments-task-scripts-image/Dockerfile + default: quay.io/redhat-appstudio/update-infra-deployments-task-script-image@sha256:2748f1a4f1af4e35214745aed4e56a9d06f6bdbd30572e7ade13729e67f23cc9 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. + - name: shared-secret + default: infra-deployments-pr-creator + description: secret in the namespace which contains private key for the GitHub App + - name: GITHUB_APP_ID + description: ID of Github app used for updating PR + default: "305606" + - name: GITHUB_APP_INSTALLATION_ID + description: Installation ID of Github app in the organization + default: "35269675" + - name: ARTIFACTS_PATH + description: path to mount artifacts directory + default: "." + - name: PR_URL + description: Pull request URL + volumes: + - name: infra-deployments-pr-creator + secret: + # 'private-key' - private key for Github app + secretName: $(params.shared-secret) + - name: shared-dir + emptyDir: {} + steps: + - name: git-clone-target-repo + image: $(params.GIT_IMAGE) + volumeMounts: + - name: shared-dir + mountPath: /shared + workingDir: /shared + env: + - name: TARGET_GH_REPO + value: "$(params.TARGET_GH_REPO)" + - name: ORIGIN_REPO + value: $(params.ORIGIN_REPO) + script: | + set -ex + git clone "https://github.com/${TARGET_GH_REPO}.git" cloned + - name: run-update-script + image: $(params.SCRIPT_IMAGE) + volumeMounts: + - name: shared-dir + mountPath: /shared + workingDir: /shared + env: + - name: SCRIPT + value: $(params.SCRIPT) + - name: ARTIFACTS_PATH + value: $(params.ARTIFACTS_PATH) + script: | + set -x + cd cloned + echo "$SCRIPT" | sh + - name: get-diff-files + image: $(params.GIT_IMAGE) + volumeMounts: + - name: shared-dir + mountPath: /shared + workingDir: /shared + script: | + set -ex + cd cloned + git status --untracked-files -s --porcelain | cut -c4- > ../updated_files.txt + # Based on https://github.com/tektoncd/catalog/tree/main/task/github-app-token/0.2/ + - name: create-pr + image: quay.io/redhat-appstudio/github-app-token@sha256:b4f2af12e9beea68055995ccdbdb86cfe1be97688c618117e5da2243dc1da18e + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. + volumeMounts: + - name: infra-deployments-pr-creator + mountPath: /secrets/deploy-key + - name: shared-dir + mountPath: /shared + workingDir: /shared + env: + - name: GITHUBAPP_KEY_PATH + value: /secrets/deploy-key/private-key + - name: GITHUBAPP_APP_ID + value: "$(params.GITHUB_APP_ID)" + - name: GITHUBAPP_INSTALLATION_ID + value: "$(params.GITHUB_APP_INSTALLATION_ID)" + - name: GITHUB_API_URL + value: https://api.github.com + - name: ORIGIN_REPO + value: $(params.ORIGIN_REPO) + - name: REVISION + value: $(params.REVISION) + - name: TARGET_GH_REPO + value: "$(params.TARGET_GH_REPO)" + - name: PR_URL + value: $(params.PR_URL) + script: | + #!/usr/bin/env python3 + import json + import logging + import os + import time + import base64 + + import requests + from jwcrypto import jwk, jwt + + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s:%(name)s:%(levelname)s:%(message)s") + logger = logging.getLogger("updater") + + EXPIRE_MINUTES_AS_SECONDS = int(os.environ.get('GITHUBAPP_TOKEN_EXPIRATION_MINUTES', 10)) * 60 + # TODO support github enteprise + GITHUB_API_URL = os.environ["GITHUB_API_URL"] + + ORIGIN_REPO = os.environ["ORIGIN_REPO"] + TARGET_GH_REPO = os.environ["TARGET_GH_REPO"] + + + class GitHub(): + token = None + + def __init__(self, private_key, app_id=None, installation_id=None): + if not isinstance(private_key, bytes): + raise ValueError(f'"{private_key}" parameter must be byte-string') + self._private_key = private_key + self.app_id = app_id + self.token = self._get_token(installation_id) + + def _load_private_key(self, pem_key_bytes): + return jwk.JWK.from_pem(pem_key_bytes) + + def _app_token(self, expire_in=EXPIRE_MINUTES_AS_SECONDS): + key = self._load_private_key(self._private_key) + now = int(time.time()) + token = jwt.JWT( + header={"alg": "RS256"}, + claims={ + "iat": now, + "exp": now + expire_in, + "iss": self.app_id + }, + algs=["RS256"], + ) + token.make_signed_token(key) + return token.serialize() + + def _get_token(self, installation_id=None): + app_token = self._app_token() + if not installation_id: + return app_token + + req = self._request( + "POST", + f"/app/installations/{installation_id}/access_tokens", + headers={ + "Authorization": f"Bearer {app_token}", + "Accept": "application/vnd.github.machine-man-preview+json" + }) + + ret = req.json() + if 'token' not in ret: + raise Exception(f"Authentication errors: {ret}") + + return ret['token'] + + def _request(self, method, url, headers={}, data={}): + if self.token and 'Authorization' not in headers: + headers.update({"Authorization": "Bearer " + self.token}) + if not url.startswith("http"): + url = f"{GITHUB_API_URL}{url}" + return requests.request(method, + url, + headers=headers, + data=json.dumps(data)) + + def create_mr(self): + repo_name = ORIGIN_REPO.split('/')[-1] + logger.info("Create update pull request, head: %s, base: main", repo_name) + req = self._request( + "POST", + f"/repos/{TARGET_GH_REPO}/pulls", + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github.v3+json" + }, + data={ + "head": repo_name, + "base": "main", + "title": f"{repo_name} update", + "maintainer_can_modify": False + }) + json_output = req.json() + print(json_output) + return json_output + + def create_reset_branch(self): + branch = ORIGIN_REPO.split('/')[-1] + target_branch = self._request("GET", f"/repos/{TARGET_GH_REPO}/git/refs/heads/{branch}").json() + main_branch_sha = self._request("GET", f"/repos/{TARGET_GH_REPO}/git/refs/heads/main").json()['object']['sha'] + if "ref" in target_branch: + logger.info("Update branch %s", branch) + self._request( + "PATCH", + f"/repos/{TARGET_GH_REPO}/git/refs/heads/{branch}", + data={"sha": main_branch_sha, "force": True} + ) + else: + logger.info("Create branch %s", branch) + self._request( + "POST", + f"/repos/{TARGET_GH_REPO}/git/refs", + data={"sha": main_branch_sha, "ref": f"refs/heads/{branch}"} + ) + + def upload_content(self): + branch = ORIGIN_REPO.split('/')[-1] + for file in open('updated_files.txt').readlines(): + file = file.strip() + with open(f"cloned/{file}", "rb") as fd: + encoded_string = base64.b64encode(fd.read()).decode("utf-8") + old_sha = self._request("GET", f'/repos/{TARGET_GH_REPO}/contents/{file}').json().get("sha") + if old_sha is None: + logger.info("Upload a new file %s", file) + self._request("PUT", f'/repos/{TARGET_GH_REPO}/contents/{file}', data={"message": f"update {file}", "branch": branch, "content": encoded_string}) + else: + logger.info("Update file %s", file) + self._request("PUT", f'/repos/{TARGET_GH_REPO}/contents/{file}', data={"message": f"update {file}", "branch": branch, "content": encoded_string, "sha": old_sha}) + + def get_pr(self): + repo_name = ORIGIN_REPO.split('/')[-1] + req = self._request( + "GET", + f"/repos/{TARGET_GH_REPO}/pulls", + headers={ + "Accept": "application/vnd.github.v3+json" + }) + json_output = req.json() + for item in json_output: + if item["user"]["login"].endswith("[bot]") and item["head"]["ref"] == repo_name: + return item + + def update_mr_description(self, pr_url, description): + req = self._request( + "PATCH", + pr_url, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github.v3+json" + }, + data={ "body": description }) + json_output = req.json() + print(json_output) + + + def main(): + with open(os.environ['GITHUBAPP_KEY_PATH'], 'rb') as key_file: + key = key_file.read() + + if os.environ.get('GITHUBAPP_APP_ID'): + app_id = os.environ['GITHUBAPP_APP_ID'] + else: + raise Exception("application id is not set") + + print(f"Getting user token for application_id: {app_id}") + github_app = GitHub( + key, + app_id=app_id, + installation_id=os.environ.get('GITHUBAPP_INSTALLATION_ID')) + + github_app.create_reset_branch() + github_app.upload_content() + infra_pr = github_app.create_mr() + if "url" not in infra_pr: + logger.info("No field 'url' is included in %r.", infra_pr) + logger.info("Try to find out the pull request in another way") + infra_pr = github_app.get_pr() + if not infra_pr: + logger.warning("No pull request is found to update the description.") + return + description = infra_pr["body"] + if description == None: + description = "Included PRs:" + new_pr_link = os.environ["PR_URL"] + new_description = f"{description}\r\n- {new_pr_link}" + logger.info("Update description of pull request %s:\n%s", infra_pr["url"], new_description) + github_app.update_mr_description(infra_pr["url"], new_description) + + + if __name__ == '__main__': + main() + + workspaces: + - name: artifacts + description: Workspace containing arbitrary artifacts used during the task run. + optional: true