From d834455004932eb9c991f19e50c54f9ce0849490 Mon Sep 17 00:00:00 2001 From: Joe Stuart Date: Thu, 14 Mar 2024 14:16:57 -0500 Subject: [PATCH 1/3] Get changed task files for git resolvers After a merge, collect any changed task files then add them to a data bundle for policy evaluation --- .tekton/push.yaml | 34 ++++--------- .tekton/scripts/build-acceptable-bundles.sh | 55 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 24 deletions(-) create mode 100755 .tekton/scripts/build-acceptable-bundles.sh diff --git a/.tekton/push.yaml b/.tekton/push.yaml index a957555a9d..eaeab8c126 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -34,8 +34,10 @@ spec: value: $(params.git-url) - name: revision value: "$(params.revision)" + # the task "build-acceptable-bundles" uses git diff-tree which needs history to compare the current + # revision to, so this must be set to 0 or > 1 for the task to work - name: depth - value: "0" + value: "2" taskRef: name: git-clone workspaces: @@ -132,40 +134,24 @@ spec: steps: - name: build-bundles image: quay.io/redhat-appstudio/appstudio-utils:{{ revision }} + env: + - name: REVISION + value: "$(params.revision)" + - name: GIT_URL + value: "$(params.git-url)" # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. script: |- #!/usr/bin/env bash - set -euo pipefail - BUNDLES=( + export BUNDLES=( $(workspaces.artifacts.path)/source/task-bundle-list $(workspaces.artifacts.path)/source/pipeline-bundle-list ) - touch ${BUNDLES[@]} - echo "Bundles to be added:" - cat ${BUNDLES[@]} - BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) - - # The OPA data bundle is tagged with the current timestamp. This has two main - # advantages. First, it prevents the image from accidentally not having any tags, - # and getting garbage collected. Second, it helps us create a timeline of the - # changes done to the data over time. - TAG="$(date '+%s')" - DATA_BUNDLE_REPO='quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles' - # Update the OPA data bundle. - ec track bundle --debug \ - --input "oci:${DATA_BUNDLE_REPO}:latest" \ - --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ - --timeout "15m0s" \ - --freshen \ - --prune \ - ${BUNDLES_PARAM[@]} + .tekton/scripts/build-acceptable-bundles.sh - # To facilitate usage in some contexts, tag the image with the floating "latest" tag. - skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" volumeMounts: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/.tekton/scripts/build-acceptable-bundles.sh new file mode 100755 index 0000000000..e8538860f7 --- /dev/null +++ b/.tekton/scripts/build-acceptable-bundles.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# helps with debugging +DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles}" +BUNDLES=${BUNDLES:-()} + +# store a list of changed task files +task_records=() +# loop over all changed files +for path in $(git diff-tree -c --name-only --no-commit-id -r ${REVISION}); do + # check that the file modified is the task file + if [[ "${path}" == task/*/*/*.yaml ]]; then + IFS='/' read -r -a path_array <<< "${path}" + dir_name_after_task="${path_array[1]}" + file_name=$(basename "${path_array[-1]}" ".yaml") + + if [[ "${dir_name_after_task}" == "${file_name}" ]]; then + # GIT_URL is the repo_url from PAC (https://hostname/org/repo) + task_records+=("git+${GIT_URL}.git//${path}@${REVISION}") + fi + fi +done + +echo "${task_records[@]}" + +touch ${BUNDLES[@]} +echo "Bundles to be added:" +cat ${BUNDLES[@]} + +# The OPA data bundle is tagged with the current timestamp. This has two main +# advantages. First, it prevents the image from accidentally not having any tags, +# and getting garbage collected. Second, it helps us create a timeline of the +# changes done to the data over time. +TAG="$(date '+%s')" + +# task_records can be empty if a task wasn't changed +TASK_PARAM=() +if [ "${#task_records[@]}" -gt 0 ]; then + TASK_PARAM=($(printf "%s\n" "${task_records[@]}" | awk '{ print "--git=" $0 }')) +fi + +BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) + +PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") +ec track bundle --debug \ + --input "oci:${DATA_BUNDLE_REPO}:latest" \ + --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ + --timeout "15m0s" \ + --freshen \ + --prune \ + ${PARAMS[@]} + +# To facilitate usage in some contexts, tag the image with the floating "latest" tag. +skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" From 117850230cc180562e2fd6116b2c7858c8f20bab Mon Sep 17 00:00:00 2001 From: Andrew McNamara Date: Wed, 27 Mar 2024 11:32:07 -0400 Subject: [PATCH 2/3] Removing unneeded tasks from FBC pipeline (#865) This is follow-on work from the following PRs: - https://github.com/release-engineering/rhtap-ec-policy/pull/19 - https://github.com/redhat-appstudio/e2e-tests/pull/1065 Signed-off-by: arewm --- pipelines/fbc-builder/patch.yaml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/pipelines/fbc-builder/patch.yaml b/pipelines/fbc-builder/patch.yaml index 8a131d3fb1..9f139a8ce4 100644 --- a/pipelines/fbc-builder/patch.yaml +++ b/pipelines/fbc-builder/patch.yaml @@ -28,6 +28,35 @@ value: "$(params.image-expires-after)" - name: COMMIT_SHA value: "$(tasks.clone-repository.results.commit)" +# Remove tasks +# Example - yq .spec.tasks.[].name ../build-definitions/pipelines/template-build/template-build.yaml | nl -v 0 +# to compute offsets +# 0 init +# 1 clone-repository +# 2 prefetch-dependencies +# 3 build-container +# 4 build-source-image +# 5 deprecated-base-image-check +# 6 clair-scan +# 7 ecosystem-cert-preflight-checks +# 8 sast-snyk-check +# 9 clamav-scan +# 10 sbom-json-check +- op: replace + path: /spec/tasks/3/runAfter/0 + value: clone-repository +- op: remove + path: /spec/tasks/9 # clamav-scan +- op: remove + path: /spec/tasks/8 # sast-snyk-check +- op: remove + path: /spec/tasks/7 # ecosystem-cert-preflight-checks +- op: remove + path: /spec/tasks/6 # clair-scan +- op: remove + path: /spec/tasks/4 # build-source-image +- op: remove + path: /spec/tasks/2 # prefetch-dependencies - op: add path: /spec/tasks/- value: @@ -88,6 +117,3 @@ workspaces: - name: workspace workspace: workspace -# - op: remove -# # build-source-image as source images are not needed for FBC components -# path: /spec/tasks/4 From 00103229f3d5f5e76eb17e1d2dbc045e5866c1ab Mon Sep 17 00:00:00 2001 From: Joseph Stuart Date: Thu, 28 Mar 2024 02:27:56 -0500 Subject: [PATCH 3/3] Revert "Get changed task files for git resolvers" (#910) --- .tekton/push.yaml | 34 +++++++++---- .tekton/scripts/build-acceptable-bundles.sh | 55 --------------------- 2 files changed, 24 insertions(+), 65 deletions(-) delete mode 100755 .tekton/scripts/build-acceptable-bundles.sh diff --git a/.tekton/push.yaml b/.tekton/push.yaml index eaeab8c126..a957555a9d 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -34,10 +34,8 @@ spec: value: $(params.git-url) - name: revision value: "$(params.revision)" - # the task "build-acceptable-bundles" uses git diff-tree which needs history to compare the current - # revision to, so this must be set to 0 or > 1 for the task to work - name: depth - value: "2" + value: "0" taskRef: name: git-clone workspaces: @@ -134,24 +132,40 @@ spec: steps: - name: build-bundles image: quay.io/redhat-appstudio/appstudio-utils:{{ revision }} - env: - - name: REVISION - value: "$(params.revision)" - - name: GIT_URL - value: "$(params.git-url)" # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. script: |- #!/usr/bin/env bash + set -euo pipefail - export BUNDLES=( + BUNDLES=( $(workspaces.artifacts.path)/source/task-bundle-list $(workspaces.artifacts.path)/source/pipeline-bundle-list ) + touch ${BUNDLES[@]} + echo "Bundles to be added:" + cat ${BUNDLES[@]} + BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) + + # The OPA data bundle is tagged with the current timestamp. This has two main + # advantages. First, it prevents the image from accidentally not having any tags, + # and getting garbage collected. Second, it helps us create a timeline of the + # changes done to the data over time. + TAG="$(date '+%s')" + DATA_BUNDLE_REPO='quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles' - .tekton/scripts/build-acceptable-bundles.sh + # Update the OPA data bundle. + ec track bundle --debug \ + --input "oci:${DATA_BUNDLE_REPO}:latest" \ + --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ + --timeout "15m0s" \ + --freshen \ + --prune \ + ${BUNDLES_PARAM[@]} + # To facilitate usage in some contexts, tag the image with the floating "latest" tag. + skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" volumeMounts: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/.tekton/scripts/build-acceptable-bundles.sh deleted file mode 100755 index e8538860f7..0000000000 --- a/.tekton/scripts/build-acceptable-bundles.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# helps with debugging -DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles}" -BUNDLES=${BUNDLES:-()} - -# store a list of changed task files -task_records=() -# loop over all changed files -for path in $(git diff-tree -c --name-only --no-commit-id -r ${REVISION}); do - # check that the file modified is the task file - if [[ "${path}" == task/*/*/*.yaml ]]; then - IFS='/' read -r -a path_array <<< "${path}" - dir_name_after_task="${path_array[1]}" - file_name=$(basename "${path_array[-1]}" ".yaml") - - if [[ "${dir_name_after_task}" == "${file_name}" ]]; then - # GIT_URL is the repo_url from PAC (https://hostname/org/repo) - task_records+=("git+${GIT_URL}.git//${path}@${REVISION}") - fi - fi -done - -echo "${task_records[@]}" - -touch ${BUNDLES[@]} -echo "Bundles to be added:" -cat ${BUNDLES[@]} - -# The OPA data bundle is tagged with the current timestamp. This has two main -# advantages. First, it prevents the image from accidentally not having any tags, -# and getting garbage collected. Second, it helps us create a timeline of the -# changes done to the data over time. -TAG="$(date '+%s')" - -# task_records can be empty if a task wasn't changed -TASK_PARAM=() -if [ "${#task_records[@]}" -gt 0 ]; then - TASK_PARAM=($(printf "%s\n" "${task_records[@]}" | awk '{ print "--git=" $0 }')) -fi - -BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) - -PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") -ec track bundle --debug \ - --input "oci:${DATA_BUNDLE_REPO}:latest" \ - --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ - --timeout "15m0s" \ - --freshen \ - --prune \ - ${PARAMS[@]} - -# To facilitate usage in some contexts, tag the image with the floating "latest" tag. -skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest"