Support matrix results in build-image-manifest

Using matrix[1] to fan out TaskRuns on a list of parameter combinations
greatly improves the maintainability of the Pipeline definition. For
example adding or removing support for a platform is a couple of lines
changed, compared to maintaining several copies of the task.

The results of the matrix-run Tasks can only be referenced in whole,
that is as an array of results that contains all the results of the
single result of one of the matrix-run Tasks.

In our case we have the `IMAGE_URL` and `IMAGE_DIGEST` results from the
single buildah Task invocation, and if run as a matrix the results can
be obtained by accessing the `IMAGE_DIGEST[*]` array and th
`IMAGE_DIGEST[*]` array separately.

Given that, when aggregating the results for the `IMAGES` parameter of
the build-image-manifest Task, some of the elements might not contain
the digests.

For example if the parameters are passed as:

    - $(tasks.build-container-amd64.results.IMAGE_URL)@$(tasks.build-container-amd64.results.IMAGE_DIGEST)
    - $(tasks.build-container-multiarch.results.IMAGE_URL[*])
    - $(tasks.build-container-multiarch.results.IMAGE_DIGEST[*])

The variable substitution would provide the args value as:

    - registry.io/repository/image-amd64@sha256:...
    - registry.io/repository/image-arm64
    - registry.io/repository/image-ppc64le
    - ...
    - @sha256:[arm64]
    - @sha256:[ppc64le]
    - ...

This change tries to support that case. When an argument is encountered
that does not contain the digest, the digest is read from the trailing
list of digests. The order of the elements from the results of the
matrix-run Tasks is deterministic -- sorted by TaskRun name; so given
the convention of digests following the image references this should be
fairly safe.

[1] https://tekton.dev/docs/pipelines/matrix/

task/build-image-manifest/0.1/build-image-manifest.yaml

@@ -69,17 +69,62 @@ spec:
 
       sed -i 's/^\s*short-name-mode\s*=\s*.*/short-name-mode = "disabled"/' /etc/containers/registries.conf
 
-      buildah manifest create "$IMAGE"
-      for i in $@
-      do
-        TOADD="$i"
-        if [[ $(echo $i | tr -cd ":" | wc -c) == 2 ]]; then
-          #we need to remove the tag, and just reference the digest
-          #as tag + digest is not supported
-          TOADD="$(echo $i | cut -d: -f1)@sha256:$(echo $i | cut -d: -f3)"
+      buildah manifest create "${IMAGE}"
+
+      # collect arguments in two arrays, one containing images with digests and
+      # one containing only digests, it is expected that all images without
+      # digests are followed with their digests in the same exact order, this
+      # aligns with how Tekton results from matrix Tasks are provided, e.g.
+      # given args as:
+      # - $(tasks.build-container-amd64.results.IMAGE_URL)@$(tasks.build-container-amd64.results.IMAGE_DIGEST)
+      # - $(tasks.build-container-multiarch.results.IMAGE_URL[*])
+      # - $(tasks.build-container-multiarch.results.IMAGE_DIGEST[*])
+      # the substitution should provide the args value as:
+      # - registry.io/repository/image-amd64@sha256:...
+      # - registry.io/repository/image-arm64
+      # - registry.io/repository/image-ppc64le
+      # - ...
+      # - @sha256:[arm64]
+      # - @sha256:[ppc64le]
+      # - ...
+      # The order is guaranteed because Tekton sorts by TaskRun name before
+      # performing the substitution
+      images=()
+      digests=()
+      for i in "$@"; do
+        [[ "$i" == @* ]] && digests+=("$i") || images+=("$i")
+      done
+
+      declare -i digest_idx=0
+      for i in "${images[@]}"; do
+        TOADD="${i}"
+        # check if the image reference contains both the tag and the digest,
+        # making sure that any port in the authority part of the image reference
+        # URI is not considered to be part of the tag by looking at the colon
+        # present after the slash
+        if [[ "${TOADD}" == */*:*@sha*:* ]]; then
+          # we need to remove the tag, and just reference the digest
+          # as tag + digest is not supported
+          # first substitution removes the suffix starting from the last colon
+          # before the "@sha*" sequence; the second substitution removes
+          # everything up to the "@" character
+          TOADD="${TOADD%:*@sha*}@${TOADD#*@}"
+        elif [[ ! "${TOADD}" == */*@sha*:* ]]; then
+          # the digest was not provided
+
+          # strip the tag if it is present, tag needs to be after the slash
+          # character so we do not remove from the port in the authority part of
+          # the URI
+          if [[ "${TOADD}" == */*:* ]]; then
+            TOADD="${TOADD%:*}"
+          fi
+
+          # expect the next digest argument to contain the digest for this image
+          TOADD="${TOADD}${digests[digest_idx]}"
+          digest_idx=$((digest_idx+1))
         fi
-        echo "Adding $TOADD"
-        buildah manifest add $IMAGE "docker://$TOADD"
+        echo "Adding ${TOADD}"
+        buildah manifest add "${IMAGE}" "docker://${TOADD}"
       done
 
       status=-1
@@ -90,17 +135,17 @@ spec:
         [ "$run" -gt 1 ] && sleep $sleep_sec
         echo "Pushing image to registry"
         buildah manifest push \
-          --tls-verify=$TLSVERIFY \
-          --digestfile image-digest $IMAGE \
-          docker://$IMAGE && break || status=$?
+          --tls-verify="${TLSVERIFY}" \
+          --digestfile image-digest "${IMAGE}" \
+          "docker://${IMAGE}" && break || status=$?
       done
       if [ "$status" -ne 0 ]; then
           echo "Failed to push image to registry after ${max_run} tries"
           exit 1
       fi
 
-      cat image-digest | tee $(results.IMAGE_DIGEST.path)
-      echo -n "$IMAGE" | tee $(results.IMAGE_URL.path)
+      cat image-digest | tee "$(results.IMAGE_DIGEST.path)"
+      echo -n "${IMAGE}" | tee "$(results.IMAGE_URL.path)"
     securityContext:
       capabilities:
         add:

task/build-image-manifest/0.1/spec/build_image_manifest_spec.sh

@@ -0,0 +1,114 @@
+#!/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+eval "$(shellspec - -c) exit 1"
+
+task_path=build-image-manifest.yaml
+
+if [[ -f ../build-image-manifest.yaml ]]; then
+    task_path="../build-image-manifest.yaml"
+fi
+
+# Extract the script so we can test it
+script="$(mktemp --tmpdir script_XXXXXXXXXX.sh)"
+chmod +x "${script}"
+yq -r '.spec.steps[0].script' "${task_path}"  > "${script}"
+trap 'rm -f "${script}"' EXIT
+
+Describe "build-image-manifest task"
+    Mock chown
+        chown_args="$*"
+        %preserve chown_args
+    End
+
+    Mock sed
+        args=("$@")
+        /usr/bin/sed "${args[@]::${#args[@]}-1}" "${registries_conf}"
+    End
+
+    Mock buildah
+        echo buildah "$@"
+        args=("$@")
+        if [[ "${args[1]}" == "push" ]]; then
+            echo "sha256:manifest_digest" > image-digest
+        fi
+    End
+
+    Mock results.IMAGE_DIGEST.path
+        echo "${digest_file}"
+    End
+
+    Mock results.IMAGE_URL.path
+        echo "${image_file}"
+    End
+
+    setup() {
+        export registries_conf="$(mktemp --tmpdir registries_XXXXXXXXXX.conf)"
+        echo 'short-name-mode = something' > "${registries_conf}"
+
+        export digest_file="$(mktemp --tmpdir digest_XXXXXXXXXX.txt)"
+        export image_file="$(mktemp --tmpdir digest_XXXXXXXXXX.txt)"
+
+        export IMAGE=registry.io/repository/image:tag
+        export TLSVERIFY=true
+    }
+
+    cleanup() {
+        rm -f "${registries_conf}" "${digest_file}" "${image_file}" image-digest
+    }
+
+    BeforeEach setup
+    AfterEach cleanup
+
+    It "strips tags from image references"
+        When call "${script}" registry.io/repository/image-amd64:tag@sha:abc
+        The variable chown_args should eq "root:root /var/lib/containers"
+        The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"'
+        The output should eq 'buildah manifest create registry.io/repository/image:tag
+Adding registry.io/repository/image-amd64@sha:abc
+buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc
+Pushing image to registry
+buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag
+sha256:manifest_digest
+registry.io/repository/image:tag'
+    End
+
+    It "supports digests following image references"
+        When call "${script}" registry.io/repository/image-amd64 registry.io/repository/image-arm64:tag registry.io:12345/repository/image-ppc64le @sha:abc @sha:def @sha:ghi
+        The variable chown_args should eq "root:root /var/lib/containers"
+        The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"'
+        The output should eq 'buildah manifest create registry.io/repository/image:tag
+Adding registry.io/repository/image-amd64@sha:abc
+buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc
+Adding registry.io/repository/image-arm64@sha:def
+buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-arm64@sha:def
+Adding registry.io:12345/repository/image-ppc64le@sha:ghi
+buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-ppc64le@sha:ghi
+Pushing image to registry
+buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag
+sha256:manifest_digest
+registry.io/repository/image:tag'
+    End
+
+    It "supports mixed image references"
+        When call "${script}" registry.io/repository/image-amd64@sha:abc registry.io/repository/image-arm64 registry.io:12345/repository/image-ppc64le@sha:ghi registry.io:12345/repository/image-s390x @sha:def @sha:jkl
+        The variable chown_args should eq "root:root /var/lib/containers"
+        The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"'
+        The output should eq 'buildah manifest create registry.io/repository/image:tag
+Adding registry.io/repository/image-amd64@sha:abc
+buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc
+Adding registry.io/repository/image-arm64@sha:def
+buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-arm64@sha:def
+Adding registry.io:12345/repository/image-ppc64le@sha:ghi
+buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-ppc64le@sha:ghi
+Adding registry.io:12345/repository/image-s390x@sha:jkl
+buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-s390x@sha:jkl
+Pushing image to registry
+buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag
+sha256:manifest_digest
+registry.io/repository/image:tag'
+    End
+End