From b42536d52ca8488dab17329ccbec6dc368027779 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 22 Oct 2024 16:13:38 -0400 Subject: [PATCH] feat(KONFLUX-4892): supply layer size information to pyxis This is meant to be paired with a change in the step image: https://github.com/konflux-ci/release-service-utils/pull/284 If the layers of the image are found to be gzip compressed, then pull them, decompress them, and include information about their decompressed digests and sizes in the manifest provided to the pyxis script. Signed-off-by: Ralph Bean --- tasks/create-pyxis-image/README.md | 2 + .../create-pyxis-image.yaml | 55 ++++++++++- tasks/create-pyxis-image/tests/mocks.sh | 15 ++- ...eate-pyxis-image-dockerfile-not-found.yaml | 4 +- ...yxis-image-fail-dockerfile-not-pulled.yaml | 2 +- ...mage-multi-containerimages-multi-arch.yaml | 4 +- ...-image-multi-containerimages-one-arch.yaml | 4 +- ...s-image-one-containerimage-multi-arch.yaml | 4 +- ...xis-image-one-containerimage-one-arch.yaml | 4 +- ...eate-pyxis-image-rhpush-and-commontag.yaml | 4 +- ...reate-pyxis-image-with-gzipped-layers.yaml | 97 +++++++++++++++++++ 11 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml diff --git a/tasks/create-pyxis-image/README.md b/tasks/create-pyxis-image/README.md index 5315b2141..9285a52e7 100644 --- a/tasks/create-pyxis-image/README.md +++ b/tasks/create-pyxis-image/README.md @@ -17,6 +17,8 @@ The relative path of the pyxis.json file in the data workspace is output as a ta | isLatest | If set to true, the images will have a latest tag added with their Pyxis entries | Yes | false | | rhPush | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes | false | | snapshotPath | Path to the JSON string of the mapped Snapshot spec in the data workspace | No | - | +## Changes in 3.4.0 +* Added support for uploading uncompressed layer information to pyxis ## Changes in 3.3.4 * Properly expand platform variables for oras args diff --git a/tasks/create-pyxis-image/create-pyxis-image.yaml b/tasks/create-pyxis-image/create-pyxis-image.yaml index 9dc751a1f..4b291d435 100644 --- a/tasks/create-pyxis-image/create-pyxis-image.yaml +++ b/tasks/create-pyxis-image/create-pyxis-image.yaml @@ -4,7 +4,7 @@ kind: Task metadata: name: create-pyxis-image labels: - app.kubernetes.io/version: "3.3.4" + app.kubernetes.io/version: "3.4.0" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release @@ -48,7 +48,7 @@ spec: description: The relative path in the workspace to the stored pyxis data json steps: - name: create-pyxis-image - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 env: - name: pyxisCert valueFrom: @@ -156,11 +156,58 @@ spec: ORAS_ARGS+=(--platform "$OS/$ARCH") fi + # Save the OCI manifest locally, to pass to a script to create the pyxis entry + MANIFEST_FILE="$(workspaces.data.path)/$(dirname "$(params.snapshotPath)")/oras-manifest-fetch.json" oras manifest fetch \ --registry-config "$AUTH_FILE" \ "${ORAS_ARGS[@]}" \ "${PULLSPEC}" \ - | tee /tmp/oras-manifest-fetch.json + | tee "${MANIFEST_FILE}" + + # Augment that manifest with further information about the layers, decompressed + # This requires pulling the layers to decompress and then measure them + while IFS= read -r BLOB_DETAIL; + do + BLOB_TYPE=$(jq -r '.mediaType' <<< "$BLOB_DETAIL") + BLOB_DIGEST=$(jq -r '.digest' <<< "$BLOB_DETAIL") + + # Normal images will always have the layers compressed. + # If they are not compressed, this will not save the + # uncompressed data in Pyxis. + # + # It's also possible that the layers are compressed with + # some scheme other than gzip. In that case, the + # uncompressed layer information will also not be saved in + # Pyxis. + # https://github.com/konflux-ci/build-definitions/issues/1264 + + if [[ "$BLOB_TYPE" =~ ^.*\.gzip$|^.*\+gzip$ ]]; then + BLOB_FILE="/tmp/oras-blob-fetch-${BLOB_DIGEST}" + BLOB_PULLSPEC="${PULLSPEC%%@*}@${BLOB_DIGEST}" + + # Save the blob + oras blob fetch \ + --registry-config "$AUTH_FILE" \ + --output "${BLOB_FILE}.gz" \ + "${BLOB_PULLSPEC}" + + # Decompress it + gunzip "${BLOB_FILE}.gz" + + # Measure it + EXPANDED_DIGEST=$(sha256sum "${BLOB_FILE}" | cut -d " " -f 1) + EXPANDED_SIZE=$(wc --bytes "${BLOB_FILE}" | awk '{print $1}' | tr -d '\n') + + # Append this information to the parsed_data manifest + jq \ + '.uncompressed_layers += [{"digest": "'"$EXPANDED_DIGEST"'", "size": '"$EXPANDED_SIZE"'}]' \ + "${MANIFEST_FILE}" > "${MANIFEST_FILE}.tmp" + mv "${MANIFEST_FILE}.tmp" "${MANIFEST_FILE}" + + # Clean up, in case we're dealing with large images + rm "/tmp/oras-blob-fetch-${BLOB_DIGEST}" + fi + done <<< "$(jq -c '.layers[]' "${MANIFEST_FILE}")" PYXIS_CERT_PATH=/tmp/crt PYXIS_KEY_PATH=/tmp/key create_container_image \ --pyxis-url $PYXIS_URL \ @@ -168,7 +215,7 @@ spec: --tags "$TAGS" \ --is-latest "$(params.isLatest)" \ --verbose \ - --oras-manifest-fetch "/tmp/oras-manifest-fetch.json" \ + --oras-manifest-fetch "${MANIFEST_FILE}" \ --name "$REPOSITORY" \ --media-type "$MEDIA_TYPE" \ --digest "$DIGEST" \ diff --git a/tasks/create-pyxis-image/tests/mocks.sh b/tasks/create-pyxis-image/tests/mocks.sh index 086c550ce..26b6052bf 100644 --- a/tasks/create-pyxis-image/tests/mocks.sh +++ b/tasks/create-pyxis-image/tests/mocks.sh @@ -9,7 +9,7 @@ function create_container_image() { # e.g. 0001, 0002, 0003... echo The image id is $(awk 'END{printf("%04i", NR)}' $(workspaces.data.path)/mock_create_container_image.txt) - if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch /tmp/oras-manifest-fetch.json --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]] + if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch "*" --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]] then echo Error: Unexpected call echo Mock create_container_image called with: $* @@ -35,7 +35,7 @@ function skopeo() { echo '{"mediaType": "application/vnd.oci.image.index.v1+json"}' elif [[ "$*" == "inspect --raw docker://"* ]] || [[ "$*" == "inspect --no-tags --override-os linux --override-arch "*" docker://"* ]] then - echo '{"mediaType": "my_media_type"}' + echo '{"mediaType": "my_media_type+gzip"}' else echo Mock skopeo called with: $* if [[ "$*" != "inspect --no-tags docker://"* ]] @@ -64,9 +64,18 @@ function oras() { if [[ "$*" == "manifest fetch --registry-config"*.dockerfile ]] then echo '{"layers": [{"annotations": {"org.opencontainers.image.title": "Dockerfile.custom"}}]}' + elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-beef.gz"* ]] + then + echo -n 'H4sIAAAAAAAAA0vKzEssqlRISSxJVEjPTy1WyEgtSgUAXVhZVxUAAAA=' | base64 -d > /tmp/oras-blob-fetch-beef.gz + elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-pork.gz"* ]] + then + echo -n 'H4sIAAAAAAAAA8vNL0pVSEksSQQA2pxWLAkAAAA=' | base64 -d > /tmp/oras-blob-fetch-pork.gz + elif [[ "$*" == "manifest fetch --registry-config"*image-with-gzipped-layers* ]] + then + echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+gzip", "digest": "beef"}, {"mediaType": "blob+gzip", "digest": "pork"}]}' elif [[ "$*" == "manifest fetch --registry-config"* ]] then - echo '{"mediaType": "my_media_type"}' + echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+other", "digest": "tofu"}]}' elif [[ "$*" == "pull --registry-config"*dockerfile-not-found:sha256-*.dockerfile* ]] then echo Mock oras called with: $* diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml index 169d0a7a5..352800aad 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml @@ -20,7 +20,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux @@ -70,7 +70,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml index 3fb5d8905..1c45032ae 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml @@ -22,7 +22,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml index a19352b4e..26afb3158 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -85,7 +85,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml index 45c9254cf..373353781 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -85,7 +85,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml index ea9820397..ec9033e28 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -69,7 +69,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml index ebb132a50..546a7ff3e 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml @@ -18,7 +18,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -68,7 +68,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml index 58a05ab90..823ed7753 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -66,7 +66,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml new file mode 100644 index 000000000..fa1592b47 --- /dev/null +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-create-pyxis-image-with-gzipped-layers +spec: + description: | + Run the create-pyxis-image task with an image that has gzipped layers, to + see that we reported the uncompressed digests correctly + workspaces: + - name: tests-workspace + tasks: + - name: setup + workspaces: + - name: data + workspace: tests-workspace + taskSpec: + workspaces: + - name: data + steps: + - name: setup-values + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 + script: | + #!/usr/bin/env bash + set -eux + + cat > "$(workspaces.data.path)"/mapped_snapshot.json << EOF + { + "application": "myapp", + "components": [ + { + "name": "comp", + "containerImage": "source@sha256:mydigest", + "repository": "registry.io/image-with-gzipped-layers", + "tags": [ + "testtag" + ] + } + ] + } + EOF + - name: run-task + taskRef: + name: create-pyxis-image + params: + - name: pyxisSecret + value: test-create-pyxis-image-cert + - name: server + value: stage + - name: snapshotPath + value: mapped_snapshot.json + workspaces: + - name: data + workspace: tests-workspace + runAfter: + - setup + - name: check-result + params: + - name: pyxisDataPath + value: $(tasks.run-task.results.pyxisDataPath) + workspaces: + - name: data + workspace: tests-workspace + taskSpec: + params: + - name: pyxisDataPath + type: string + workspaces: + - name: data + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 + script: | + #!/usr/bin/env bash + set -eux + + if [ "$(wc -l < "$(workspaces.data.path)"/mock_create_container_image.txt)" != 1 ]; then + echo Error: create_container_image was expected to be called 1 time. Actual calls: + cat "$(workspaces.data.path)/mock_create_container_image.txt" + exit 1 + fi + + if [ "$(wc -l < "$(workspaces.data.path)"/mock_oras.txt)" != 5 ]; then + echo Error: oras was expected to be called 4 times. Actual calls: + cat "$(workspaces.data.path)/mock_oras.txt" + exit 1 + fi + + [ "$(head -n 1 < "$(workspaces.data.path)"/mock_skopeo.txt)" \ + = "inspect --raw docker://registry.io/image-with-gzipped-layers@sha256:mydigest" ] + + # check that the size of the decompressed layers is as expected + jq -e '.uncompressed_layers[0].size == 21' "$(workspaces.data.path)/oras-manifest-fetch.json" + jq -e '.uncompressed_layers[1].size == 9' "$(workspaces.data.path)/oras-manifest-fetch.json" + + runAfter: + - run-task