From 3f1f0d4cdb3a21247ef5f7712f6085c4eed09bfb Mon Sep 17 00:00:00 2001 From: Zoran Regvart Date: Wed, 29 May 2024 14:07:04 +0200 Subject: [PATCH] Support matrix results in build-image-manifest Using matrix[1] to fan out TaskRuns on a list of parameter combinations greatly improves the maintainability of the Pipeline definition. For example adding or removing support for a platform is a couple of lines changed, compared to maintaining several copies of the task. The results of the matrix-run Tasks can only be referenced in whole, that is as an array of results that contains all the results of the single result of one of the matrix-run Tasks. In our case we have the `IMAGE_URL` and `IMAGE_DIGEST` results from the single buildah Task invocation, and if run as a matrix the results can be obtained by accessing the `IMAGE_DIGEST[*]` array and th `IMAGE_DIGEST[*]` array separately. Given that, when aggregating the results for the `IMAGES` parameter of the build-image-manifest Task, some of the elements might not contain the digests. For example if the parameters are passed as: - $(tasks.build-container-amd64.results.IMAGE_URL)@$(tasks.build-container-amd64.results.IMAGE_DIGEST) - $(tasks.build-container-multiarch.results.IMAGE_URL[*]) - $(tasks.build-container-multiarch.results.IMAGE_DIGEST[*]) The variable substitution would provide the args value as: - registry.io/repository/image-amd64@sha256:... - registry.io/repository/image-arm64 - registry.io/repository/image-ppc64le - ... - @sha256:[arm64] - @sha256:[ppc64le] - ... This change tries to support that case. When an argument is encountered that does not contain the digest, the digest is read from the trailing list of digests. The order of the elements from the results of the matrix-run Tasks is deterministic -- sorted by TaskRun name; so given the convention of digests following the image references this should be fairly safe. [1] https://tekton.dev/docs/pipelines/matrix/ --- .../0.1/build-image-manifest.yaml | 75 +++++++++--- .../0.1/spec/build_image_manifest_spec.sh | 114 ++++++++++++++++++ 2 files changed, 174 insertions(+), 15 deletions(-) create mode 100644 task/build-image-manifest/0.1/spec/build_image_manifest_spec.sh diff --git a/task/build-image-manifest/0.1/build-image-manifest.yaml b/task/build-image-manifest/0.1/build-image-manifest.yaml index 2967c78248..6c15aa98e3 100644 --- a/task/build-image-manifest/0.1/build-image-manifest.yaml +++ b/task/build-image-manifest/0.1/build-image-manifest.yaml @@ -69,17 +69,62 @@ spec: sed -i 's/^\s*short-name-mode\s*=\s*.*/short-name-mode = "disabled"/' /etc/containers/registries.conf - buildah manifest create "$IMAGE" - for i in $@ - do - TOADD="$i" - if [[ $(echo $i | tr -cd ":" | wc -c) == 2 ]]; then - #we need to remove the tag, and just reference the digest - #as tag + digest is not supported - TOADD="$(echo $i | cut -d: -f1)@sha256:$(echo $i | cut -d: -f3)" + buildah manifest create "${IMAGE}" + + # collect arguments in two arrays, one containing images with digests and + # one containing only digests, it is expected that all images without + # digests are followed with their digests in the same exact order, this + # aligns with how Tekton results from matrix Tasks are provided, e.g. + # given args as: + # - $(tasks.build-container-amd64.results.IMAGE_URL)@$(tasks.build-container-amd64.results.IMAGE_DIGEST) + # - $(tasks.build-container-multiarch.results.IMAGE_URL[*]) + # - $(tasks.build-container-multiarch.results.IMAGE_DIGEST[*]) + # the substitution should provide the args value as: + # - registry.io/repository/image-amd64@sha256:... + # - registry.io/repository/image-arm64 + # - registry.io/repository/image-ppc64le + # - ... + # - @sha256:[arm64] + # - @sha256:[ppc64le] + # - ... + # The order is guaranteed because Tekton sorts by TaskRun name before + # performing the substitution + images=() + digests=() + for i in "$@"; do + [[ "$i" == @* ]] && digests+=("$i") || images+=("$i") + done + + declare -i digest_idx=0 + for i in "${images[@]}"; do + TOADD="${i}" + # check if the image reference contains both the tag and the digest, + # making sure that any port in the authority part of the image reference + # URI is not considered to be part of the tag by looking at the colon + # present after the slash + if [[ "${TOADD}" == */*:*@sha*:* ]]; then + # we need to remove the tag, and just reference the digest + # as tag + digest is not supported + # first substitution removes the suffix starting from the last colon + # before the "@sha*" sequence; the second substitution removes + # everything up to the "@" character + TOADD="${TOADD%:*@sha*}@${TOADD#*@}" + elif [[ ! "${TOADD}" == */*@sha*:* ]]; then + # the digest was not provided + + # strip the tag if it is present, tag needs to be after the slash + # character so we do not remove from the port in the authority part of + # the URI + if [[ "${TOADD}" == */*:* ]]; then + TOADD="${TOADD%:*}" + fi + + # expect the next digest argument to contain the digest for this image + TOADD="${TOADD}${digests[digest_idx]}" + digest_idx=$((digest_idx+1)) fi - echo "Adding $TOADD" - buildah manifest add $IMAGE "docker://$TOADD" + echo "Adding ${TOADD}" + buildah manifest add "${IMAGE}" "docker://${TOADD}" done status=-1 @@ -90,17 +135,17 @@ spec: [ "$run" -gt 1 ] && sleep $sleep_sec echo "Pushing image to registry" buildah manifest push \ - --tls-verify=$TLSVERIFY \ - --digestfile image-digest $IMAGE \ - docker://$IMAGE && break || status=$? + --tls-verify="${TLSVERIFY}" \ + --digestfile image-digest "${IMAGE}" \ + "docker://${IMAGE}" && break || status=$? done if [ "$status" -ne 0 ]; then echo "Failed to push image to registry after ${max_run} tries" exit 1 fi - cat image-digest | tee $(results.IMAGE_DIGEST.path) - echo -n "$IMAGE" | tee $(results.IMAGE_URL.path) + cat image-digest | tee "$(results.IMAGE_DIGEST.path)" + echo -n "${IMAGE}" | tee "$(results.IMAGE_URL.path)" securityContext: capabilities: add: diff --git a/task/build-image-manifest/0.1/spec/build_image_manifest_spec.sh b/task/build-image-manifest/0.1/spec/build_image_manifest_spec.sh new file mode 100644 index 0000000000..c02af7926b --- /dev/null +++ b/task/build-image-manifest/0.1/spec/build_image_manifest_spec.sh @@ -0,0 +1,114 @@ +#!/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +eval "$(shellspec - -c) exit 1" + +task_path=build-image-manifest.yaml + +if [[ -f ../build-image-manifest.yaml ]]; then + task_path="../build-image-manifest.yaml" +fi + +# Extract the script so we can test it +script="$(mktemp --tmpdir script_XXXXXXXXXX.sh)" +chmod +x "${script}" +yq -r '.spec.steps[0].script' "${task_path}" > "${script}" +trap 'rm -f "${script}"' EXIT + +Describe "build-image-manifest task" + Mock chown + chown_args="$*" + %preserve chown_args + End + + Mock sed + args=("$@") + /usr/bin/sed "${args[@]::${#args[@]}-1}" "${registries_conf}" + End + + Mock buildah + echo buildah "$@" + args=("$@") + if [[ "${args[1]}" == "push" ]]; then + echo "sha256:manifest_digest" > image-digest + fi + End + + Mock results.IMAGE_DIGEST.path + echo "${digest_file}" + End + + Mock results.IMAGE_URL.path + echo "${image_file}" + End + + setup() { + export registries_conf="$(mktemp --tmpdir registries_XXXXXXXXXX.conf)" + echo 'short-name-mode = something' > "${registries_conf}" + + export digest_file="$(mktemp --tmpdir digest_XXXXXXXXXX.txt)" + export image_file="$(mktemp --tmpdir digest_XXXXXXXXXX.txt)" + + export IMAGE=registry.io/repository/image:tag + export TLSVERIFY=true + } + + cleanup() { + rm -f "${registries_conf}" "${digest_file}" "${image_file}" image-digest + } + + BeforeEach setup + AfterEach cleanup + + It "strips tags from image references" + When call "${script}" registry.io/repository/image-amd64:tag@sha:abc + The variable chown_args should eq "root:root /var/lib/containers" + The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"' + The output should eq 'buildah manifest create registry.io/repository/image:tag +Adding registry.io/repository/image-amd64@sha:abc +buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc +Pushing image to registry +buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag +sha256:manifest_digest +registry.io/repository/image:tag' + End + + It "supports digests following image references" + When call "${script}" registry.io/repository/image-amd64 registry.io/repository/image-arm64:tag registry.io:12345/repository/image-ppc64le @sha:abc @sha:def @sha:ghi + The variable chown_args should eq "root:root /var/lib/containers" + The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"' + The output should eq 'buildah manifest create registry.io/repository/image:tag +Adding registry.io/repository/image-amd64@sha:abc +buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc +Adding registry.io/repository/image-arm64@sha:def +buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-arm64@sha:def +Adding registry.io:12345/repository/image-ppc64le@sha:ghi +buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-ppc64le@sha:ghi +Pushing image to registry +buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag +sha256:manifest_digest +registry.io/repository/image:tag' + End + + It "supports mixed image references" + When call "${script}" registry.io/repository/image-amd64@sha:abc registry.io/repository/image-arm64 registry.io:12345/repository/image-ppc64le@sha:ghi registry.io:12345/repository/image-s390x @sha:def @sha:jkl + The variable chown_args should eq "root:root /var/lib/containers" + The contents of file "${registries_conf}" should eq 'short-name-mode = "disabled"' + The output should eq 'buildah manifest create registry.io/repository/image:tag +Adding registry.io/repository/image-amd64@sha:abc +buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-amd64@sha:abc +Adding registry.io/repository/image-arm64@sha:def +buildah manifest add registry.io/repository/image:tag docker://registry.io/repository/image-arm64@sha:def +Adding registry.io:12345/repository/image-ppc64le@sha:ghi +buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-ppc64le@sha:ghi +Adding registry.io:12345/repository/image-s390x@sha:jkl +buildah manifest add registry.io/repository/image:tag docker://registry.io:12345/repository/image-s390x@sha:jkl +Pushing image to registry +buildah manifest push --tls-verify=true --digestfile image-digest registry.io/repository/image:tag docker://registry.io/repository/image:tag +sha256:manifest_digest +registry.io/repository/image:tag' + End +End