From 43f2f6c8a161e58bf0d89e2f6bc319d16daddec9 Mon Sep 17 00:00:00 2001 From: Nate Marsella Date: Fri, 6 Dec 2024 14:43:04 -0500 Subject: [PATCH] feat(KONFLUX-3935) Add pruning check to FBC pipeline --- CODEOWNERS | 17 +- pipelines/fbc-builder/README.md | 16 +- pipelines/fbc-builder/patch.yaml | 22 ++ renovate.json | 1 + .../0.1/README.md | 29 +++ .../0.1/USAGE.md | 10 + .../0.1/fbc-target-index-pruning-check.yaml | 199 ++++++++++++++++++ task/validate-fbc/0.1/README.md | 1 + task/validate-fbc/0.1/validate-fbc.yaml | 3 + 9 files changed, 288 insertions(+), 10 deletions(-) create mode 100644 task/fbc-target-index-pruning-check/0.1/README.md create mode 100644 task/fbc-target-index-pruning-check/0.1/USAGE.md create mode 100644 task/fbc-target-index-pruning-check/0.1/fbc-target-index-pruning-check.yaml diff --git a/CODEOWNERS b/CODEOWNERS index d2e69a6fe1..9a17607a6b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -51,14 +51,15 @@ /.tekton/tasks/ec-checks.yaml @konflux-ci/ec # renovate groupName=integration -/task/clair-scan @konflux-ci/integration-service-maintainers -/task/clamav-scan @konflux-ci/integration-service-maintainers -/task/deprecated-image-check @konflux-ci/integration-service-maintainers -/task/fbc-related-image-check @konflux-ci/integration-service-maintainers -/task/fbc-validation @konflux-ci/integration-service-maintainers -/task/inspect-image @konflux-ci/integration-service-maintainers -/task/sbom-json-check @konflux-ci/integration-service-maintainers -/task/validate-fbc @konflux-ci/integration-service-maintainers +/task/clair-scan @konflux-ci/integration-service-maintainers +/task/clamav-scan @konflux-ci/integration-service-maintainers +/task/deprecated-image-check @konflux-ci/integration-service-maintainers +/task/fbc-related-image-check @konflux-ci/integration-service-maintainers +/task/fbc-target-index-pruning-check @konflux-ci/integration-service-maintainers +/task/fbc-validation @konflux-ci/integration-service-maintainers +/task/inspect-image @konflux-ci/integration-service-maintainers +/task/sbom-json-check @konflux-ci/integration-service-maintainers +/task/validate-fbc @konflux-ci/integration-service-maintainers # renovate groupName=integration /task/coverity-availability-check @konflux-ci/integration-service-maintainers @kdudka diff --git a/pipelines/fbc-builder/README.md b/pipelines/fbc-builder/README.md index c2510879fd..d566d7b3f6 100644 --- a/pipelines/fbc-builder/README.md +++ b/pipelines/fbc-builder/README.md @@ -82,6 +82,13 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito |IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-image-index.results.IMAGE_URL)'| |POLICY_DIR| Path to directory containing Conftest policies.| /project/repository/| | |POLICY_NAMESPACE| Namespace for Conftest policy.| required_checks| | +### fbc-target-index-pruning-check:0.1 task parameters +|name|description|default value|already set by| +|---|---|---|---| +|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'| +|IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-image-index.results.IMAGE_URL)'| +|OCP_VERSION| OCP version.| None| '$(tasks.validate-fbc.results.OCP_VERSION)'| +|TARGET_INDEX| Image name of target index, minus tag.| registry.redhat.io/redhat/redhat-operator-index| 'registry.redhat.io/redhat/redhat-operator-index'| ### git-clone-oci-ta:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| @@ -148,9 +155,9 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito |name|description|used in params (taskname:taskrefversion:taskparam) |---|---|---| |IMAGES| List of all referenced image manifests| | -|IMAGE_DIGEST| Digest of the image just built| deprecated-base-image-check:0.4:IMAGE_DIGEST ; validate-fbc:0.1:IMAGE_DIGEST| +|IMAGE_DIGEST| Digest of the image just built| deprecated-base-image-check:0.4:IMAGE_DIGEST ; validate-fbc:0.1:IMAGE_DIGEST ; fbc-target-index-pruning-check:0.1:IMAGE_DIGEST| |IMAGE_REF| Image reference of the built image containing both the repository and the digest| | -|IMAGE_URL| Image repository and tag where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; apply-tags:0.1:IMAGE ; validate-fbc:0.1:IMAGE_URL| +|IMAGE_URL| Image repository and tag where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; apply-tags:0.1:IMAGE ; validate-fbc:0.1:IMAGE_URL ; fbc-target-index-pruning-check:0.1:IMAGE_URL| |SBOM_BLOB_URL| Reference of SBOM blob digest to enable digest-based verification from provenance| | ### buildah-remote-oci-ta:0.2 task results |name|description|used in params (taskname:taskrefversion:taskparam) @@ -166,6 +173,10 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito |---|---|---| |IMAGES_PROCESSED| Images processed in the task.| | |TEST_OUTPUT| Tekton task test output.| | +### fbc-target-index-pruning-check:0.1 task results +|name|description|used in params (taskname:taskrefversion:taskparam) +|---|---|---| +|TEST_OUTPUT| Tekton task test output.| | ### git-clone-oci-ta:0.1 task results |name|description|used in params (taskname:taskrefversion:taskparam) |---|---|---| @@ -189,6 +200,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito |name|description|used in params (taskname:taskrefversion:taskparam) |---|---|---| |IMAGES_PROCESSED| Images processed in the task.| | +|OCP_VERSION| OCP version derived from base image.| fbc-target-index-pruning-check:0.1:OCP_VERSION| |RELATED_IMAGES_DIGEST| Digest for attached json file containing related images| | |RELATED_IMAGE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the related images for the FBC fragment.| | |TEST_OUTPUT| Tekton task test output.| | diff --git a/pipelines/fbc-builder/patch.yaml b/pipelines/fbc-builder/patch.yaml index 670b45f63d..345ee62a30 100644 --- a/pipelines/fbc-builder/patch.yaml +++ b/pipelines/fbc-builder/patch.yaml @@ -98,3 +98,25 @@ value: $(tasks.build-image-index.results.IMAGE_URL) - name: IMAGE_DIGEST value: $(tasks.build-image-index.results.IMAGE_DIGEST) +- op: add + path: /spec/tasks/- + value: + name: fbc-target-index-pruning-check + when: + - input: $(params.skip-checks) + operator: in + values: ["false"] + runAfter: + - validate-fbc + taskRef: + name: fbc-target-index-pruning-check + version: "0.1" + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: TARGET_INDEX + value: registry.redhat.io/redhat/redhat-operator-index + - name: OCP_VERSION + value: $(tasks.validate-fbc.results.OCP_VERSION) diff --git a/renovate.json b/renovate.json index 98931fc932..86aaa74949 100644 --- a/renovate.json +++ b/renovate.json @@ -85,6 +85,7 @@ "task/coverity-availability-check/**", "task/deprecated-image-check/**", "task/fbc-related-image-check/**", + "task/fbc-target-index-pruning-check/**", "task/fbc-validation/**", "task/inspect-image/**", "task/sast-coverity-check-oci-ta/**", diff --git a/task/fbc-target-index-pruning-check/0.1/README.md b/task/fbc-target-index-pruning-check/0.1/README.md new file mode 100644 index 0000000000..70c7b98ed8 --- /dev/null +++ b/task/fbc-target-index-pruning-check/0.1/README.md @@ -0,0 +1,29 @@ +# fbc-target-index-pruning-check task + +## Description: +Ensures file-based catalog (FBC) components do not remove released versions of operators from the production catalog. + +For further information on how to use the task, see the USAGE.md file. + +## Params: + +| name | description | default value | +|--------------|----------------------------------|---------| +| IMAGE_URL | Fully qualified image name. | | +| IMAGE_DIGEST | Image digest. | | +| TARGET_IMAGE | Image name of target index, minus tag. | `registry.redhat.io/redhat/redhat-operator-index` | +| OCP_VERSION | OCP version of FBC image. | | + +## Results: + +| name | description | +|--------------------|---------------------------| +| TEST_OUTPUT | Tekton task test output. | + +## Source repository for image: +https://github.com/konflux-ci/konflux-test + +## Additional links: +https://olm.operatorframework.io/docs/reference/file-based-catalogs/ +https://github.com/containers/skopeo +https://docs.openshift.com/container-platform/4.12/cli_reference/opm/cli-opm-install.html diff --git a/task/fbc-target-index-pruning-check/0.1/USAGE.md b/task/fbc-target-index-pruning-check/0.1/USAGE.md new file mode 100644 index 0000000000..7d9f43b52c --- /dev/null +++ b/task/fbc-target-index-pruning-check/0.1/USAGE.md @@ -0,0 +1,10 @@ +# fbc-target-index-pruning-check task + +### Purpose: +- This task ensures file-based catalog (FBC) components do not remove previously released versions of operators from a target catalog, specified +in the `TARGET_INDEX` parameter, which by default points to the production Red Hat catalog `registry.redhat.io/redhat/redhat-operator-index`. + +### What this check does: +- Runs `opm render` on both FBC fragment and TARGET_INDEX:OCP_VERSION images. +- Compares the channel data of the FBC fragment and target index. +- Checks if the FBC fragment will remove channels or channel entries previously added to the target index. diff --git a/task/fbc-target-index-pruning-check/0.1/fbc-target-index-pruning-check.yaml b/task/fbc-target-index-pruning-check/0.1/fbc-target-index-pruning-check.yaml new file mode 100644 index 0000000000..724e252d1b --- /dev/null +++ b/task/fbc-target-index-pruning-check/0.1/fbc-target-index-pruning-check.yaml @@ -0,0 +1,199 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "konflux" + name: fbc-target-index-pruning-check +spec: + description: >- + Ensures file-based catalog (FBC) components do not remove versions of operators already added to a released catalog. + params: + - name: IMAGE_URL + description: Fully qualified image name. + - name: IMAGE_DIGEST + description: Image digest. + - name: TARGET_INDEX + description: Image name of target index, minus tag. + default: registry.redhat.io/redhat/redhat-operator-index + - name: OCP_VERSION + description: OCP version. + results: + - name: TEST_OUTPUT + description: Tekton task test output. + steps: + - name: check-if-fragment-prunes-target-index + image: quay.io/redhat-appstudio/konflux-test:v1.4.9@sha256:eee855e60b437d9a55a30e63f2eb7f95d9fd6d3b111c32cac8730c9b7a071394 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + workingDir: /var/workdir/fbc-pruning + env: + - name: IMAGE_URL + value: $(params.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(params.IMAGE_DIGEST) + - name: TARGET_INDEX + value: $(params.TARGET_INDEX) + - name: OCP_VERSION + value: $(params.OCP_VERSION) + securityContext: + runAsUser: 0 + capabilities: + add: + - SETFCAP + computeResources: + limits: + memory: 4Gi + requests: + memory: 512Mi + cpu: 10m + script: | + #!/usr/bin/env bash + set -euo pipefail + # shellcheck source=/dev/null + source /utils.sh + trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT + + IMAGE_URL="${IMAGE_URL}@${IMAGE_DIGEST}" + # Given a tag and a the digest in the IMAGE_URL we opt to use the digest alone + # this is because containers/image currently doesn't support image references + # that contain both. See https://github.com/containers/image/issues/1736 + if [[ "${IMAGE_URL}" == *":"*"@"* ]]; then + IMAGE_URL="${IMAGE_URL/:*@/@}" + fi + + ### Check if TARGET_INDEX is defined + if [ -z "${TARGET_INDEX}" ]; then + echo "TARGET_INDEX is not defined." + note="Task $(context.task.name) failed: The TARGET_INDEX is not defined." + TEST_OUTPUT=$(make_result_json -r ERROR -t "$note") + echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)" + exit 0 + fi + + ### Check if OCP_VERSION is defined + if [ -z "${OCP_VERSION}" ]; then + echo "OCP_VERSION is not defined." + note="Task $(context.task.name) failed: The OCP_VERSION is not defined." + TEST_OUTPUT=$(make_result_json -r ERROR -t "$note") + echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)" + exit 0 + fi + + ### Run opm render for FBC fragment and target index + rendered_fbc_image=/tmp/opm-render-fbc-fragment.json + rendered_target_index=/tmp/opm-render-target-index.json + + echo "Rendering FBC image: ${IMAGE_URL}" + opm render "${IMAGE_URL}" | tr -d '\000-\031' > "${rendered_fbc_image}" + if [[ ! -f "${rendered_fbc_image}" ]]; then + note="Task $(context.task.name) failed: Unable to render the fragment FBC image: ${IMAGE_URL}" + echo "${note}" + TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}") + exit 0 + fi + + target_index_pullspec="${TARGET_INDEX}:v${OCP_VERSION}" + echo "Rendering target index: ${target_index_pullspec}" + opm render "${target_index_pullspec}" | tr -d '\000-\031' > "${rendered_target_index}" + if [[ ! -f "${rendered_target_index}" ]]; then + note="Task $(context.task.name) failed: Unable to render the fragment target index image: ${IMAGE_URL}" + echo "${note}" + TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}") + exit 0 + fi + + failure_num=0 + TESTPASSED=true + + fbc_channels=/tmp/olm-channels-fbc-image.json + ndx_channels=/tmp/olm-channels-target-index.json + + ### Filter out channels and channel entries from FBC fragment render + jq -s 'map(select(.schema == "olm.channel")) | reduce .[] as $obj ([]; . += [{package: $obj.package, channel: $obj.name, entries: [$obj.entries[].name]}])' "${rendered_fbc_image}" > "${fbc_channels}" + + echo "" + echo "Channels defined in FBC fragment:" + jq '.' "${fbc_channels}" + echo "" + + ### Filter out channels and channel entries from target index render + jq -s 'map(select(.schema == "olm.channel")) | reduce .[] as $obj ([]; . += [{package: $obj.package, channel: $obj.name, entries: [$obj.entries[].name]}])' "${rendered_target_index}" > "${ndx_channels}" + + ### Get the package(s) the fragment is configuring + mapfile -t fbc_pkgs < <(jq -r '.[].package ' "${fbc_channels}" | sort -u) + if (( ${#fbc_pkgs[@]} < 1 )); then + note="Task $(context.task.name) failed: No OLM packages detected in FBC fragment." + echo "${note}" + TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}") + exit 0 + fi + + ### Get packages in target index + mapfile -t ndx_pkgs < <(jq -r '.[].package ' "${ndx_channels}" | sort -u) + if [[ ${#ndx_pkgs[@]} -lt 1 ]]; then + note="Task $(context.task.name) failed: No OLM packages detected in target index." + echo "${note}" + TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}") + exit 0 + fi + + ### Test packages in the FBC fragment that already exist in the target index. + pkgs_to_test=() + for pkg in "${fbc_pkgs[@]}"; do + if echo "${ndx_pkgs[@]}" | grep -Fwq "${pkg}"; then + pkgs_to_test+=("${pkg}") + fi + done + + if (( ${#pkgs_to_test[@]} > 0 )); then + for pkg in "${pkgs_to_test[@]}"; do + channels_to_test=() + mapfile -t fbc_channel_names < <(jq -r --arg p "${pkg}" '.[] | select(.package == $p) | .channel' ${fbc_channels}) + mapfile -t ndx_channel_names < <(jq -r --arg p "${pkg}" '.[] | select(.package == $p) | .channel' ${ndx_channels}) + + ### Check for removed channels + for chan in "${ndx_channel_names[@]}"; do + if echo "${fbc_channel_names[@]}" | grep -Fwq "${chan}"; then + channels_to_test+=("${chan}") + else + echo "!FAILURE! - FBC fragment prunes entire ${pkg}.${chan} channel." + TESTPASSED=false + failure_num=$((failure_num + 1)) + fi + done + + ### Check each channel for removed entries + if (( ${#channels_to_test[@]} > 0 )); then + for chan in "${channels_to_test[@]}"; do + echo "" + echo "TARGET INDEX ${pkg}.${chan} channel:" + jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c)' "${ndx_channels}" + echo "" + mapfile -t ndx_entries < <(jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c) | .entries[]' "${ndx_channels}") + mapfile -t fbc_entries < <(jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c) | .entries[]' "${fbc_channels}") + + for entry in "${ndx_entries[@]}"; do + if ! echo "${fbc_entries[@]}" | grep -Fwq "${entry}"; then + echo "!FAILURE! - FBC fragment prunes ${entry} from ${pkg}.${chan} channel." + failure_num=$((failure_num + 1)) + TESTPASSED=false + fi + done + done + fi + done + else + echo "FBC fragment is not modifying any existing channels in the target index." + fi + + note="Task $(context.task.name) completed: Check result for task result." + if [[ $TESTPASSED == false ]]; then + ERROR_OUTPUT=$(make_result_json -r FAILURE -f $failure_num -s 0 -t "${note}") + echo "${ERROR_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)" + else + TEST_OUTPUT=$(make_result_json -r SUCCESS -s 1 -t "${note}") + echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)" + fi diff --git a/task/validate-fbc/0.1/README.md b/task/validate-fbc/0.1/README.md index 5d472eb4d8..08a11772c7 100644 --- a/task/validate-fbc/0.1/README.md +++ b/task/validate-fbc/0.1/README.md @@ -16,4 +16,5 @@ Ensures file-based catalog (FBC) components are uniquely linted for proper const |TEST_OUTPUT|Tekton task test output.| |RELATED_IMAGES_DIGEST|Digest for attached json file containing related images| |IMAGES_PROCESSED|Images processed in the task.| +|OCP_VERSION|OCP version of FBC image.| diff --git a/task/validate-fbc/0.1/validate-fbc.yaml b/task/validate-fbc/0.1/validate-fbc.yaml index 73d416a36a..547c14a548 100644 --- a/task/validate-fbc/0.1/validate-fbc.yaml +++ b/task/validate-fbc/0.1/validate-fbc.yaml @@ -36,6 +36,8 @@ spec: description: Digest for attached json file containing related images - name: IMAGES_PROCESSED description: Images processed in the task. + - name: OCP_VERSION + description: OCP version derived from base image. volumes: - name: shared emptyDir: {} @@ -386,6 +388,7 @@ spec: # extracts major digits and filters out any leading alphabetic characters, for e.g. 'v4' --> '4' OCP_VER_MAJOR=$(echo "${OCP_VER_FROM_BASE}" | cut -d '.' -f 1 | sed "s/^[a-zA-Z]*//") OCP_VER_MINOR=$(echo "${OCP_VER_FROM_BASE}" | cut -d '.' -f 2) + echo -n "${OCP_VER_MAJOR}.${OCP_VER_MINOR}" > "$(results.OCP_VERSION.path)" RUN_OCP_VERSION_VALIDATION="false" digits_regex='^[0-9]*$'