feat(KONFLUX-3935) Add pruning check to FBC pipeline

CODEOWNERS

@@ -51,14 +51,15 @@
 /.tekton/tasks/ec-checks.yaml       @konflux-ci/ec
 
 # renovate groupName=integration
-/task/clair-scan                @konflux-ci/integration-service-maintainers
-/task/clamav-scan               @konflux-ci/integration-service-maintainers
-/task/deprecated-image-check    @konflux-ci/integration-service-maintainers
-/task/fbc-related-image-check   @konflux-ci/integration-service-maintainers
-/task/fbc-validation            @konflux-ci/integration-service-maintainers
-/task/inspect-image             @konflux-ci/integration-service-maintainers
-/task/sbom-json-check           @konflux-ci/integration-service-maintainers
-/task/validate-fbc              @konflux-ci/integration-service-maintainers
+/task/clair-scan                      @konflux-ci/integration-service-maintainers
+/task/clamav-scan                     @konflux-ci/integration-service-maintainers
+/task/deprecated-image-check          @konflux-ci/integration-service-maintainers
+/task/fbc-related-image-check         @konflux-ci/integration-service-maintainers
+/task/fbc-target-index-pruning-check  @konflux-ci/integration-service-maintainers
+/task/fbc-validation                  @konflux-ci/integration-service-maintainers
+/task/inspect-image                   @konflux-ci/integration-service-maintainers
+/task/sbom-json-check                 @konflux-ci/integration-service-maintainers
+/task/validate-fbc                    @konflux-ci/integration-service-maintainers
 
 # renovate groupName=integration
 /task/coverity-availability-check           @konflux-ci/integration-service-maintainers @kdudka

pipelines/fbc-builder/README.md

@@ -82,6 +82,13 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 |POLICY_DIR| Path to directory containing Conftest policies.| /project/repository/| |
 |POLICY_NAMESPACE| Namespace for Conftest policy.| required_checks| |
+### fbc-target-index-pruning-check:0.1 task parameters
+|name|description|default value|already set by|
+|---|---|---|---|
+|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
+|OCP_VERSION| OCP version.| None| '$(tasks.validate-fbc.results.OCP_VERSION)'|
+|TARGET_INDEX| Image name of target index, minus tag.| registry.redhat.io/redhat/redhat-operator-index| 'registry.redhat.io/redhat/redhat-operator-index'|
 ### git-clone-oci-ta:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
@@ -148,9 +155,9 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
 |IMAGES| List of all referenced image manifests| |
-|IMAGE_DIGEST| Digest of the image just built| deprecated-base-image-check:0.4:IMAGE_DIGEST ; validate-fbc:0.1:IMAGE_DIGEST|
+|IMAGE_DIGEST| Digest of the image just built| deprecated-base-image-check:0.4:IMAGE_DIGEST ; validate-fbc:0.1:IMAGE_DIGEST ; fbc-target-index-pruning-check:0.1:IMAGE_DIGEST|
 |IMAGE_REF| Image reference of the built image containing both the repository and the digest| |
-|IMAGE_URL| Image repository and tag where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; apply-tags:0.1:IMAGE ; validate-fbc:0.1:IMAGE_URL|
+|IMAGE_URL| Image repository and tag where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; apply-tags:0.1:IMAGE ; validate-fbc:0.1:IMAGE_URL ; fbc-target-index-pruning-check:0.1:IMAGE_URL|
 |SBOM_BLOB_URL| Reference of SBOM blob digest to enable digest-based verification from provenance| |
 ### buildah-remote-oci-ta:0.2 task results
 |name|description|used in params (taskname:taskrefversion:taskparam)
@@ -166,6 +173,10 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |---|---|---|
 |IMAGES_PROCESSED| Images processed in the task.| |
 |TEST_OUTPUT| Tekton task test output.| |
+### fbc-target-index-pruning-check:0.1 task results
+|name|description|used in params (taskname:taskrefversion:taskparam)
+|---|---|---|
+|TEST_OUTPUT| Tekton task test output.| |
 ### git-clone-oci-ta:0.1 task results
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
@@ -189,6 +200,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
 |IMAGES_PROCESSED| Images processed in the task.| |
+|OCP_VERSION| OCP version derived from base image.| fbc-target-index-pruning-check:0.1:OCP_VERSION|
 |RELATED_IMAGES_DIGEST| Digest for attached json file containing related images| |
 |RELATED_IMAGE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the related images for the FBC fragment.| |
 |TEST_OUTPUT| Tekton task test output.| |

pipelines/fbc-builder/patch.yaml

@@ -98,3 +98,25 @@
       value: $(tasks.build-image-index.results.IMAGE_URL)
     - name: IMAGE_DIGEST
       value: $(tasks.build-image-index.results.IMAGE_DIGEST)
+- op: add
+  path: /spec/tasks/-
+  value:
+    name: fbc-target-index-pruning-check
+    when:
+    - input: $(params.skip-checks)
+      operator: in
+      values: ["false"]
+    runAfter:
+      - validate-fbc
+    taskRef:
+      name: fbc-target-index-pruning-check
+      version: "0.1"
+    params:
+    - name: IMAGE_URL
+      value: $(tasks.build-image-index.results.IMAGE_URL)
+    - name: IMAGE_DIGEST
+      value: $(tasks.build-image-index.results.IMAGE_DIGEST)
+    - name: TARGET_INDEX
+      value: registry.redhat.io/redhat/redhat-operator-index
+    - name: OCP_VERSION
+      value: $(tasks.validate-fbc.results.OCP_VERSION)

renovate.json

@@ -85,6 +85,7 @@
         "task/coverity-availability-check/**",
         "task/deprecated-image-check/**",
         "task/fbc-related-image-check/**",
+        "task/fbc-target-index-pruning-check/**",
         "task/fbc-validation/**",
         "task/inspect-image/**",
         "task/sast-coverity-check-oci-ta/**",

task/fbc-target-index-pruning-check/0.1/README.md

@@ -0,0 +1,29 @@
+# fbc-target-index-pruning-check task
+
+## Description:
+Ensures file-based catalog (FBC) components do not remove released versions of operators from the production catalog.
+
+For further information on how to use the task, see the USAGE.md file.
+
+## Params:
+
+| name         | description                      | default value |
+|--------------|----------------------------------|---------|
+| IMAGE_URL    | Fully qualified image name.      | |
+| IMAGE_DIGEST | Image digest.                    | |
+| TARGET_IMAGE | Image name of target index, minus tag. | `registry.redhat.io/redhat/redhat-operator-index` |
+| OCP_VERSION  | OCP version of FBC image. | |
+
+## Results:
+
+| name               | description               |
+|--------------------|---------------------------|
+| TEST_OUTPUT  | Tekton task test output.  |
+
+## Source repository for image:
+https://github.com/konflux-ci/konflux-test
+
+## Additional links:
+https://olm.operatorframework.io/docs/reference/file-based-catalogs/
+https://github.com/containers/skopeo
+https://docs.openshift.com/container-platform/4.12/cli_reference/opm/cli-opm-install.html

task/fbc-target-index-pruning-check/0.1/USAGE.md

@@ -0,0 +1,10 @@
+# fbc-target-index-pruning-check task
+
+### Purpose:
+- This task ensures file-based catalog (FBC) components do not remove previously released versions of operators from a target catalog, specified
+in the `TARGET_INDEX` parameter, which by default points to the production Red Hat catalog `registry.redhat.io/redhat/redhat-operator-index`.
+
+### What this check does:
+- Runs `opm render` on both FBC fragment and TARGET_INDEX:OCP_VERSION images.
+- Compares the channel data of the FBC fragment and target index.
+- Checks if the FBC fragment will remove channels or channel entries previously added to the target index.

task/fbc-target-index-pruning-check/0.1/fbc-target-index-pruning-check.yaml

@@ -0,0 +1,199 @@
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.12.1"
+    tekton.dev/tags: "konflux"
+  name: fbc-target-index-pruning-check
+spec:
+  description: >-
+    Ensures file-based catalog (FBC) components do not remove versions of operators already added to a released catalog.
+  params:
+    - name: IMAGE_URL
+      description: Fully qualified image name.
+    - name: IMAGE_DIGEST
+      description: Image digest.
+    - name: TARGET_INDEX
+      description: Image name of target index, minus tag.
+      default: registry.redhat.io/redhat/redhat-operator-index
+    - name: OCP_VERSION
+      description: OCP version.
+  results:
+    - name: TEST_OUTPUT
+      description: Tekton task test output.
+  steps:
+    - name: check-if-fragment-prunes-target-index
+      image: quay.io/redhat-appstudio/konflux-test:v1.4.9@sha256:eee855e60b437d9a55a30e63f2eb7f95d9fd6d3b111c32cac8730c9b7a071394
+      # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
+      # the cluster will set imagePullPolicy to IfNotPresent
+      workingDir: /var/workdir/fbc-pruning
+      env:
+        - name: IMAGE_URL
+          value: $(params.IMAGE_URL)
+        - name: IMAGE_DIGEST
+          value: $(params.IMAGE_DIGEST)
+        - name: TARGET_INDEX
+          value: $(params.TARGET_INDEX)
+        - name: OCP_VERSION
+          value: $(params.OCP_VERSION)
+      securityContext:
+        runAsUser: 0
+        capabilities:
+          add:
+            - SETFCAP
+      computeResources:
+        limits:
+          memory: 4Gi
+        requests:
+          memory: 512Mi
+          cpu: 10m
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+        # shellcheck source=/dev/null
+        source /utils.sh
+        trap 'handle_error $(results.TEST_OUTPUT.path)' EXIT
+
+        IMAGE_URL="${IMAGE_URL}@${IMAGE_DIGEST}"
+        # Given a tag and a the digest in the IMAGE_URL we opt to use the digest alone
+        # this is because containers/image currently doesn't support image references
+        # that contain both. See https://github.com/containers/image/issues/1736
+        if [[ "${IMAGE_URL}" == *":"*"@"* ]]; then
+          IMAGE_URL="${IMAGE_URL/:*@/@}"
+        fi
+
+        ### Check if TARGET_INDEX is defined
+        if [ -z "${TARGET_INDEX}" ]; then
+          echo "TARGET_INDEX is not defined."
+          note="Task $(context.task.name) failed: The TARGET_INDEX is not defined."
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "$note")
+          echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)"
+          exit 0
+        fi
+
+        ### Check if OCP_VERSION is defined
+        if [ -z "${OCP_VERSION}" ]; then
+          echo "OCP_VERSION is not defined."
+          note="Task $(context.task.name) failed: The OCP_VERSION is not defined."
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "$note")
+          echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)"
+          exit 0
+        fi
+
+        ### Run opm render for FBC fragment and target index
+        rendered_fbc_image=/tmp/opm-render-fbc-fragment.json
+        rendered_target_index=/tmp/opm-render-target-index.json
+
+        echo "Rendering FBC image:    ${IMAGE_URL}"
+        opm render "${IMAGE_URL}" | tr -d '\000-\031' > "${rendered_fbc_image}"
+        if [[ ! -f "${rendered_fbc_image}" ]]; then
+          note="Task $(context.task.name) failed: Unable to render the fragment FBC image: ${IMAGE_URL}"
+          echo "${note}"
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}")
+          exit 0
+        fi
+
+        target_index_pullspec="${TARGET_INDEX}:v${OCP_VERSION}"
+        echo "Rendering target index: ${target_index_pullspec}"
+        opm render "${target_index_pullspec}" | tr -d '\000-\031' > "${rendered_target_index}"
+        if [[ ! -f "${rendered_target_index}" ]]; then
+          note="Task $(context.task.name) failed: Unable to render the fragment target index image: ${IMAGE_URL}"
+          echo "${note}"
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}")
+          exit 0
+        fi
+
+        failure_num=0
+        TESTPASSED=true
+
+        fbc_channels=/tmp/olm-channels-fbc-image.json
+        ndx_channels=/tmp/olm-channels-target-index.json
+
+        ### Filter out channels and channel entries from FBC fragment render
+        jq -s 'map(select(.schema == "olm.channel")) | reduce .[] as $obj ([]; . += [{package: $obj.package, channel: $obj.name, entries: [$obj.entries[].name]}])' "${rendered_fbc_image}" > "${fbc_channels}"
+
+        echo ""
+        echo "Channels defined in FBC fragment:"
+        jq '.' "${fbc_channels}"
+        echo ""
+
+        ### Filter out channels and channel entries from target index render
+        jq -s 'map(select(.schema == "olm.channel")) | reduce .[] as $obj ([]; . += [{package: $obj.package, channel: $obj.name, entries: [$obj.entries[].name]}])' "${rendered_target_index}" > "${ndx_channels}"
+
+        ### Get the package(s) the fragment is configuring
+        mapfile -t fbc_pkgs < <(jq -r '.[].package ' "${fbc_channels}" | sort -u)
+        if (( ${#fbc_pkgs[@]} < 1 )); then
+          note="Task $(context.task.name) failed: No OLM packages detected in FBC fragment."
+          echo "${note}"
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}")
+          exit 0
+        fi
+
+        ### Get packages in target index
+        mapfile -t ndx_pkgs < <(jq -r '.[].package ' "${ndx_channels}" | sort -u)
+        if [[ ${#ndx_pkgs[@]} -lt 1 ]]; then
+          note="Task $(context.task.name) failed: No OLM packages detected in target index."
+          echo "${note}"
+          TEST_OUTPUT=$(make_result_json -r ERROR -t "${note}")
+          exit 0
+        fi
+
+        ### Test packages in the FBC fragment that already exist in the target index.
+        pkgs_to_test=()
+        for pkg in "${fbc_pkgs[@]}"; do
+          if echo "${ndx_pkgs[@]}" | grep -Fwq "${pkg}"; then
+            pkgs_to_test+=("${pkg}")
+          fi
+        done
+
+        if (( ${#pkgs_to_test[@]} > 0 )); then
+          for pkg in "${pkgs_to_test[@]}"; do
+            channels_to_test=()
+            mapfile -t fbc_channel_names < <(jq -r --arg p "${pkg}" '.[] | select(.package == $p) | .channel' ${fbc_channels})
+            mapfile -t ndx_channel_names < <(jq -r --arg p "${pkg}" '.[] | select(.package == $p) | .channel' ${ndx_channels})
+
+            ### Check for removed channels
+            for chan in "${ndx_channel_names[@]}"; do
+              if echo "${fbc_channel_names[@]}" | grep -Fwq "${chan}"; then
+                channels_to_test+=("${chan}")
+              else
+                echo "!FAILURE! - FBC fragment prunes entire ${pkg}.${chan} channel."
+                TESTPASSED=false
+                failure_num=$((failure_num + 1))
+              fi
+            done
+
+            ### Check each channel for removed entries
+            if (( ${#channels_to_test[@]} > 0 )); then
+              for chan in "${channels_to_test[@]}"; do
+                echo ""
+                echo "TARGET INDEX ${pkg}.${chan} channel:"
+                jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c)' "${ndx_channels}"
+                echo ""
+                mapfile -t ndx_entries < <(jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c) | .entries[]' "${ndx_channels}")
+                mapfile -t fbc_entries < <(jq -r --arg p "${pkg}" --arg c "${chan}" '.[] | select(.package == $p and .channel == $c) | .entries[]' "${fbc_channels}")
+
+                for entry in "${ndx_entries[@]}"; do
+                  if ! echo "${fbc_entries[@]}" | grep -Fwq "${entry}"; then
+                    echo "!FAILURE! - FBC fragment prunes ${entry} from ${pkg}.${chan} channel."
+                    failure_num=$((failure_num + 1))
+                    TESTPASSED=false
+                  fi
+                done
+              done
+            fi
+          done
+        else
+          echo "FBC fragment is not modifying any existing channels in the target index."
+        fi
+
+        note="Task $(context.task.name) completed: Check result for task result."
+        if [[ $TESTPASSED == false ]]; then
+          ERROR_OUTPUT=$(make_result_json -r FAILURE -f $failure_num -s 0 -t "${note}")
+          echo "${ERROR_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)"
+        else
+          TEST_OUTPUT=$(make_result_json -r SUCCESS -s 1 -t "${note}")
+          echo "${TEST_OUTPUT}" | tee "$(results.TEST_OUTPUT.path)"
+        fi

task/validate-fbc/0.1/README.md

@@ -16,4 +16,5 @@ Ensures file-based catalog (FBC) components are uniquely linted for proper const
 |TEST_OUTPUT|Tekton task test output.|
 |RELATED_IMAGES_DIGEST|Digest for attached json file containing related images|
 |IMAGES_PROCESSED|Images processed in the task.|
+|OCP_VERSION|OCP version of FBC image.|
 

task/validate-fbc/0.1/validate-fbc.yaml

@@ -36,6 +36,8 @@ spec:
       description: Digest for attached json file containing related images
     - name: IMAGES_PROCESSED
       description: Images processed in the task.
+    - name: OCP_VERSION
+      description: OCP version derived from base image.
   volumes:
     - name: shared
       emptyDir: {}
@@ -386,6 +388,7 @@ spec:
         # extracts major digits and filters out any leading alphabetic characters, for e.g. 'v4' --> '4'
         OCP_VER_MAJOR=$(echo "${OCP_VER_FROM_BASE}" | cut -d '.' -f 1 | sed "s/^[a-zA-Z]*//")
         OCP_VER_MINOR=$(echo "${OCP_VER_FROM_BASE}" | cut -d '.' -f 2)
+        echo -n "${OCP_VER_MAJOR}.${OCP_VER_MINOR}" > "$(results.OCP_VERSION.path)"
 
         RUN_OCP_VERSION_VALIDATION="false"
         digits_regex='^[0-9]*$'