From 0edc312d0be1aadaf4b796ac473bc88b5dbc654f Mon Sep 17 00:00:00 2001 From: tkdchen Date: Mon, 25 Mar 2024 20:17:36 +0800 Subject: [PATCH 1/8] Update test repos for source container build (#905) Names of the test repos created in tha past match the cleanup rule defined in e2e-tests, which causes those repos were removed from redhat-appstudio-qe organization. As of this update, the test repos have been recovered with new names already. Signed-off-by: Chenxiong Qi --- .tekton/tasks/e2e-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/tasks/e2e-test.yaml b/.tekton/tasks/e2e-test.yaml index 04de907609..6cb0df377a 100644 --- a/.tekton/tasks/e2e-test.yaml +++ b/.tekton/tasks/e2e-test.yaml @@ -31,7 +31,7 @@ spec: - name: APP_SUFFIX value: "$(params.app_suffix)" - name: COMPONENT_REPO_URLS - value: "https://github.com/redhat-appstudio-qe/devfile-sample-python-basic,https://github.com/redhat-appstudio-qe/retrodep,https://github.com/cachito-testing/pip-e2e-test,https://github.com/redhat-appstudio-qe/fbc-sample-repo,https://github.com/redhat-appstudio-qe/nodejs-no-dockerfile,https://github.com/redhat-appstudio-qe/maven-hello-world,https://github.com/redhat-appstudio-qe/e2e-tests-parent-image-with-both-tag-digest,https://github.com/redhat-appstudio-qe/e2e-tests-parent-image-with-digest-only,https://github.com/redhat-appstudio-qe/e2e-tests-use-latest-parent-image,https://github.com/redhat-appstudio-qe/e2e-tests-parent-image-from-registry-rh-io" + value: "https://github.com/redhat-appstudio-qe/devfile-sample-python-basic,https://github.com/redhat-appstudio-qe/retrodep,https://github.com/cachito-testing/pip-e2e-test,https://github.com/redhat-appstudio-qe/fbc-sample-repo,https://github.com/redhat-appstudio-qe/nodejs-no-dockerfile,https://github.com/redhat-appstudio-qe/maven-hello-world,https://github.com/redhat-appstudio-qe/source-build-parent-image-with-digest-only,https://github.com/redhat-appstudio-qe/source-build-parent-image-with-both-tag-digest,https://github.com/redhat-appstudio-qe/source-build-use-latest-parent-image,https://github.com/redhat-appstudio-qe/source-build-parent-image-from-registry-rh-io" - name: QUAY_E2E_ORGANIZATION value: redhat-appstudio - name: E2E_APPLICATIONS_NAMESPACE From fd5815a110e54ae17de6396306ed62ca1beaf476 Mon Sep 17 00:00:00 2001 From: Adam Cmiel Date: Wed, 27 Mar 2024 08:03:33 +0100 Subject: [PATCH 2/8] e2e-test: Set ginkgo timeout to 2 hours (#901) In d07462a6068ca7f6f086c2b734c2cdfb97e902e5, the timeout for the test pipelineRun was increased to 2 hours (see the commit message for reasoning.) However, ginkgo has its own timeout which defaults to 1 hour. Increase to 2 hours correspondingly. Signed-off-by: Adam Cmiel --- .tekton/tasks/e2e-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tekton/tasks/e2e-test.yaml b/.tekton/tasks/e2e-test.yaml index 6cb0df377a..88e6e63245 100644 --- a/.tekton/tasks/e2e-test.yaml +++ b/.tekton/tasks/e2e-test.yaml @@ -23,7 +23,8 @@ spec: # against build-definitions to update this tag args: [ "--ginkgo.label-filter=build-templates-e2e", - "--ginkgo.no-color" + "--ginkgo.no-color", + "--ginkgo.timeout=2h" ] securityContext: runAsUser: 1000 From fcab3e95dd23bb4398062f4e2373fae4ff4873a8 Mon Sep 17 00:00:00 2001 From: Sergii Kabashniuk Date: Wed, 27 Mar 2024 11:14:06 +0200 Subject: [PATCH 3/8] Update ignore (#907) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..69f039aada --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.vscode/ +.idea/ +pipeline-bundle-list +task-bundle-list From d834455004932eb9c991f19e50c54f9ce0849490 Mon Sep 17 00:00:00 2001 From: Joe Stuart Date: Thu, 14 Mar 2024 14:16:57 -0500 Subject: [PATCH 4/8] Get changed task files for git resolvers After a merge, collect any changed task files then add them to a data bundle for policy evaluation --- .tekton/push.yaml | 34 ++++--------- .tekton/scripts/build-acceptable-bundles.sh | 55 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 24 deletions(-) create mode 100755 .tekton/scripts/build-acceptable-bundles.sh diff --git a/.tekton/push.yaml b/.tekton/push.yaml index a957555a9d..eaeab8c126 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -34,8 +34,10 @@ spec: value: $(params.git-url) - name: revision value: "$(params.revision)" + # the task "build-acceptable-bundles" uses git diff-tree which needs history to compare the current + # revision to, so this must be set to 0 or > 1 for the task to work - name: depth - value: "0" + value: "2" taskRef: name: git-clone workspaces: @@ -132,40 +134,24 @@ spec: steps: - name: build-bundles image: quay.io/redhat-appstudio/appstudio-utils:{{ revision }} + env: + - name: REVISION + value: "$(params.revision)" + - name: GIT_URL + value: "$(params.git-url)" # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. script: |- #!/usr/bin/env bash - set -euo pipefail - BUNDLES=( + export BUNDLES=( $(workspaces.artifacts.path)/source/task-bundle-list $(workspaces.artifacts.path)/source/pipeline-bundle-list ) - touch ${BUNDLES[@]} - echo "Bundles to be added:" - cat ${BUNDLES[@]} - BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) - - # The OPA data bundle is tagged with the current timestamp. This has two main - # advantages. First, it prevents the image from accidentally not having any tags, - # and getting garbage collected. Second, it helps us create a timeline of the - # changes done to the data over time. - TAG="$(date '+%s')" - DATA_BUNDLE_REPO='quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles' - # Update the OPA data bundle. - ec track bundle --debug \ - --input "oci:${DATA_BUNDLE_REPO}:latest" \ - --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ - --timeout "15m0s" \ - --freshen \ - --prune \ - ${BUNDLES_PARAM[@]} + .tekton/scripts/build-acceptable-bundles.sh - # To facilitate usage in some contexts, tag the image with the floating "latest" tag. - skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" volumeMounts: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/.tekton/scripts/build-acceptable-bundles.sh new file mode 100755 index 0000000000..e8538860f7 --- /dev/null +++ b/.tekton/scripts/build-acceptable-bundles.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# helps with debugging +DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles}" +BUNDLES=${BUNDLES:-()} + +# store a list of changed task files +task_records=() +# loop over all changed files +for path in $(git diff-tree -c --name-only --no-commit-id -r ${REVISION}); do + # check that the file modified is the task file + if [[ "${path}" == task/*/*/*.yaml ]]; then + IFS='/' read -r -a path_array <<< "${path}" + dir_name_after_task="${path_array[1]}" + file_name=$(basename "${path_array[-1]}" ".yaml") + + if [[ "${dir_name_after_task}" == "${file_name}" ]]; then + # GIT_URL is the repo_url from PAC (https://hostname/org/repo) + task_records+=("git+${GIT_URL}.git//${path}@${REVISION}") + fi + fi +done + +echo "${task_records[@]}" + +touch ${BUNDLES[@]} +echo "Bundles to be added:" +cat ${BUNDLES[@]} + +# The OPA data bundle is tagged with the current timestamp. This has two main +# advantages. First, it prevents the image from accidentally not having any tags, +# and getting garbage collected. Second, it helps us create a timeline of the +# changes done to the data over time. +TAG="$(date '+%s')" + +# task_records can be empty if a task wasn't changed +TASK_PARAM=() +if [ "${#task_records[@]}" -gt 0 ]; then + TASK_PARAM=($(printf "%s\n" "${task_records[@]}" | awk '{ print "--git=" $0 }')) +fi + +BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) + +PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") +ec track bundle --debug \ + --input "oci:${DATA_BUNDLE_REPO}:latest" \ + --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ + --timeout "15m0s" \ + --freshen \ + --prune \ + ${PARAMS[@]} + +# To facilitate usage in some contexts, tag the image with the floating "latest" tag. +skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" From 117850230cc180562e2fd6116b2c7858c8f20bab Mon Sep 17 00:00:00 2001 From: Andrew McNamara Date: Wed, 27 Mar 2024 11:32:07 -0400 Subject: [PATCH 5/8] Removing unneeded tasks from FBC pipeline (#865) This is follow-on work from the following PRs: - https://github.com/release-engineering/rhtap-ec-policy/pull/19 - https://github.com/redhat-appstudio/e2e-tests/pull/1065 Signed-off-by: arewm --- pipelines/fbc-builder/patch.yaml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/pipelines/fbc-builder/patch.yaml b/pipelines/fbc-builder/patch.yaml index 8a131d3fb1..9f139a8ce4 100644 --- a/pipelines/fbc-builder/patch.yaml +++ b/pipelines/fbc-builder/patch.yaml @@ -28,6 +28,35 @@ value: "$(params.image-expires-after)" - name: COMMIT_SHA value: "$(tasks.clone-repository.results.commit)" +# Remove tasks +# Example - yq .spec.tasks.[].name ../build-definitions/pipelines/template-build/template-build.yaml | nl -v 0 +# to compute offsets +# 0 init +# 1 clone-repository +# 2 prefetch-dependencies +# 3 build-container +# 4 build-source-image +# 5 deprecated-base-image-check +# 6 clair-scan +# 7 ecosystem-cert-preflight-checks +# 8 sast-snyk-check +# 9 clamav-scan +# 10 sbom-json-check +- op: replace + path: /spec/tasks/3/runAfter/0 + value: clone-repository +- op: remove + path: /spec/tasks/9 # clamav-scan +- op: remove + path: /spec/tasks/8 # sast-snyk-check +- op: remove + path: /spec/tasks/7 # ecosystem-cert-preflight-checks +- op: remove + path: /spec/tasks/6 # clair-scan +- op: remove + path: /spec/tasks/4 # build-source-image +- op: remove + path: /spec/tasks/2 # prefetch-dependencies - op: add path: /spec/tasks/- value: @@ -88,6 +117,3 @@ workspaces: - name: workspace workspace: workspace -# - op: remove -# # build-source-image as source images are not needed for FBC components -# path: /spec/tasks/4 From 00103229f3d5f5e76eb17e1d2dbc045e5866c1ab Mon Sep 17 00:00:00 2001 From: Joseph Stuart Date: Thu, 28 Mar 2024 02:27:56 -0500 Subject: [PATCH 6/8] Revert "Get changed task files for git resolvers" (#910) --- .tekton/push.yaml | 34 +++++++++---- .tekton/scripts/build-acceptable-bundles.sh | 55 --------------------- 2 files changed, 24 insertions(+), 65 deletions(-) delete mode 100755 .tekton/scripts/build-acceptable-bundles.sh diff --git a/.tekton/push.yaml b/.tekton/push.yaml index eaeab8c126..a957555a9d 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -34,10 +34,8 @@ spec: value: $(params.git-url) - name: revision value: "$(params.revision)" - # the task "build-acceptable-bundles" uses git diff-tree which needs history to compare the current - # revision to, so this must be set to 0 or > 1 for the task to work - name: depth - value: "2" + value: "0" taskRef: name: git-clone workspaces: @@ -134,24 +132,40 @@ spec: steps: - name: build-bundles image: quay.io/redhat-appstudio/appstudio-utils:{{ revision }} - env: - - name: REVISION - value: "$(params.revision)" - - name: GIT_URL - value: "$(params.git-url)" # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent # also per direction from Ralph Bean, we want to use image digest based tags to use a cue to automation like dependabot or renovatebot to periodially submit pull requests that update the digest as new images are released. script: |- #!/usr/bin/env bash + set -euo pipefail - export BUNDLES=( + BUNDLES=( $(workspaces.artifacts.path)/source/task-bundle-list $(workspaces.artifacts.path)/source/pipeline-bundle-list ) + touch ${BUNDLES[@]} + echo "Bundles to be added:" + cat ${BUNDLES[@]} + BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) + + # The OPA data bundle is tagged with the current timestamp. This has two main + # advantages. First, it prevents the image from accidentally not having any tags, + # and getting garbage collected. Second, it helps us create a timeline of the + # changes done to the data over time. + TAG="$(date '+%s')" + DATA_BUNDLE_REPO='quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles' - .tekton/scripts/build-acceptable-bundles.sh + # Update the OPA data bundle. + ec track bundle --debug \ + --input "oci:${DATA_BUNDLE_REPO}:latest" \ + --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ + --timeout "15m0s" \ + --freshen \ + --prune \ + ${BUNDLES_PARAM[@]} + # To facilitate usage in some contexts, tag the image with the floating "latest" tag. + skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" volumeMounts: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/.tekton/scripts/build-acceptable-bundles.sh deleted file mode 100755 index e8538860f7..0000000000 --- a/.tekton/scripts/build-acceptable-bundles.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# helps with debugging -DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles}" -BUNDLES=${BUNDLES:-()} - -# store a list of changed task files -task_records=() -# loop over all changed files -for path in $(git diff-tree -c --name-only --no-commit-id -r ${REVISION}); do - # check that the file modified is the task file - if [[ "${path}" == task/*/*/*.yaml ]]; then - IFS='/' read -r -a path_array <<< "${path}" - dir_name_after_task="${path_array[1]}" - file_name=$(basename "${path_array[-1]}" ".yaml") - - if [[ "${dir_name_after_task}" == "${file_name}" ]]; then - # GIT_URL is the repo_url from PAC (https://hostname/org/repo) - task_records+=("git+${GIT_URL}.git//${path}@${REVISION}") - fi - fi -done - -echo "${task_records[@]}" - -touch ${BUNDLES[@]} -echo "Bundles to be added:" -cat ${BUNDLES[@]} - -# The OPA data bundle is tagged with the current timestamp. This has two main -# advantages. First, it prevents the image from accidentally not having any tags, -# and getting garbage collected. Second, it helps us create a timeline of the -# changes done to the data over time. -TAG="$(date '+%s')" - -# task_records can be empty if a task wasn't changed -TASK_PARAM=() -if [ "${#task_records[@]}" -gt 0 ]; then - TASK_PARAM=($(printf "%s\n" "${task_records[@]}" | awk '{ print "--git=" $0 }')) -fi - -BUNDLES_PARAM=($(cat ${BUNDLES[@]} | awk '{ print "--bundle=" $0 }')) - -PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") -ec track bundle --debug \ - --input "oci:${DATA_BUNDLE_REPO}:latest" \ - --output "oci:${DATA_BUNDLE_REPO}:${TAG}" \ - --timeout "15m0s" \ - --freshen \ - --prune \ - ${PARAMS[@]} - -# To facilitate usage in some contexts, tag the image with the floating "latest" tag. -skopeo copy "docker://${DATA_BUNDLE_REPO}:${TAG}" "docker://${DATA_BUNDLE_REPO}:latest" From 70c12c660d3c99b9f69f594075801835861bca8b Mon Sep 17 00:00:00 2001 From: "rh-tap-build-team[bot]" <127938674+rh-tap-build-team[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:12:40 +0200 Subject: [PATCH 7/8] update .tekton/tasks/e2e-test.yaml (#897) Co-authored-by: rh-tap-build-team[bot] <127938674+rh-tap-build-team[bot]@users.noreply.github.com> --- .tekton/tasks/e2e-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/tasks/e2e-test.yaml b/.tekton/tasks/e2e-test.yaml index 88e6e63245..c514c7a3dd 100644 --- a/.tekton/tasks/e2e-test.yaml +++ b/.tekton/tasks/e2e-test.yaml @@ -18,7 +18,7 @@ spec: type: string steps: - name: e2e-test - image: quay.io/redhat-appstudio/e2e-tests:13ba743f0ae3fe85502a598ea2a9d7da91164b61 + image: quay.io/redhat-appstudio/e2e-tests:b5c5f1cf9bf9f641859aadb8f80eded96c70785c # a la infra-deployment updates, when PRs merge in e2e-tests, PRs will be opened # against build-definitions to update this tag args: [ From 7e2b2a14cb9145bf3aa3f54010fe43e5fcb44197 Mon Sep 17 00:00:00 2001 From: John Duimovich Date: Thu, 28 Mar 2024 10:52:51 -0400 Subject: [PATCH 8/8] add secret param to pipeline to allow gitlab/github (#911) - defaults remain the same so existing installer will work with github --- pipelines/docker-build-rhtap/patch.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pipelines/docker-build-rhtap/patch.yaml b/pipelines/docker-build-rhtap/patch.yaml index 362e7c7b5b..36e7df74fc 100644 --- a/pipelines/docker-build-rhtap/patch.yaml +++ b/pipelines/docker-build-rhtap/patch.yaml @@ -14,6 +14,13 @@ name: stackrox-secret type: string default: "rox-api-token" +- op: add + path: /spec/params/- + value: + name: gitops-auth-secret-name + type: string + default: gitops-auth-secret + description: "Secret name to enable this pipeline to update the gitops repo with the new image. " - op: add path: /spec/params/- value: @@ -141,6 +148,8 @@ value: $(params.git-url)-gitops - name: image value: $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - name: gitops-auth-secret-name + value: $(params.gitops-auth-secret-name) runAfter: - build-container when: