diff --git a/tasks/create-pyxis-image/README.md b/tasks/create-pyxis-image/README.md index 5315b2141..9285a52e7 100644 --- a/tasks/create-pyxis-image/README.md +++ b/tasks/create-pyxis-image/README.md @@ -17,6 +17,8 @@ The relative path of the pyxis.json file in the data workspace is output as a ta | isLatest | If set to true, the images will have a latest tag added with their Pyxis entries | Yes | false | | rhPush | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes | false | | snapshotPath | Path to the JSON string of the mapped Snapshot spec in the data workspace | No | - | +## Changes in 3.4.0 +* Added support for uploading uncompressed layer information to pyxis ## Changes in 3.3.4 * Properly expand platform variables for oras args diff --git a/tasks/create-pyxis-image/create-pyxis-image.yaml b/tasks/create-pyxis-image/create-pyxis-image.yaml index 9dc751a1f..4b291d435 100644 --- a/tasks/create-pyxis-image/create-pyxis-image.yaml +++ b/tasks/create-pyxis-image/create-pyxis-image.yaml @@ -4,7 +4,7 @@ kind: Task metadata: name: create-pyxis-image labels: - app.kubernetes.io/version: "3.3.4" + app.kubernetes.io/version: "3.4.0" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release @@ -48,7 +48,7 @@ spec: description: The relative path in the workspace to the stored pyxis data json steps: - name: create-pyxis-image - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 env: - name: pyxisCert valueFrom: @@ -156,11 +156,58 @@ spec: ORAS_ARGS+=(--platform "$OS/$ARCH") fi + # Save the OCI manifest locally, to pass to a script to create the pyxis entry + MANIFEST_FILE="$(workspaces.data.path)/$(dirname "$(params.snapshotPath)")/oras-manifest-fetch.json" oras manifest fetch \ --registry-config "$AUTH_FILE" \ "${ORAS_ARGS[@]}" \ "${PULLSPEC}" \ - | tee /tmp/oras-manifest-fetch.json + | tee "${MANIFEST_FILE}" + + # Augment that manifest with further information about the layers, decompressed + # This requires pulling the layers to decompress and then measure them + while IFS= read -r BLOB_DETAIL; + do + BLOB_TYPE=$(jq -r '.mediaType' <<< "$BLOB_DETAIL") + BLOB_DIGEST=$(jq -r '.digest' <<< "$BLOB_DETAIL") + + # Normal images will always have the layers compressed. + # If they are not compressed, this will not save the + # uncompressed data in Pyxis. + # + # It's also possible that the layers are compressed with + # some scheme other than gzip. In that case, the + # uncompressed layer information will also not be saved in + # Pyxis. + # https://github.com/konflux-ci/build-definitions/issues/1264 + + if [[ "$BLOB_TYPE" =~ ^.*\.gzip$|^.*\+gzip$ ]]; then + BLOB_FILE="/tmp/oras-blob-fetch-${BLOB_DIGEST}" + BLOB_PULLSPEC="${PULLSPEC%%@*}@${BLOB_DIGEST}" + + # Save the blob + oras blob fetch \ + --registry-config "$AUTH_FILE" \ + --output "${BLOB_FILE}.gz" \ + "${BLOB_PULLSPEC}" + + # Decompress it + gunzip "${BLOB_FILE}.gz" + + # Measure it + EXPANDED_DIGEST=$(sha256sum "${BLOB_FILE}" | cut -d " " -f 1) + EXPANDED_SIZE=$(wc --bytes "${BLOB_FILE}" | awk '{print $1}' | tr -d '\n') + + # Append this information to the parsed_data manifest + jq \ + '.uncompressed_layers += [{"digest": "'"$EXPANDED_DIGEST"'", "size": '"$EXPANDED_SIZE"'}]' \ + "${MANIFEST_FILE}" > "${MANIFEST_FILE}.tmp" + mv "${MANIFEST_FILE}.tmp" "${MANIFEST_FILE}" + + # Clean up, in case we're dealing with large images + rm "/tmp/oras-blob-fetch-${BLOB_DIGEST}" + fi + done <<< "$(jq -c '.layers[]' "${MANIFEST_FILE}")" PYXIS_CERT_PATH=/tmp/crt PYXIS_KEY_PATH=/tmp/key create_container_image \ --pyxis-url $PYXIS_URL \ @@ -168,7 +215,7 @@ spec: --tags "$TAGS" \ --is-latest "$(params.isLatest)" \ --verbose \ - --oras-manifest-fetch "/tmp/oras-manifest-fetch.json" \ + --oras-manifest-fetch "${MANIFEST_FILE}" \ --name "$REPOSITORY" \ --media-type "$MEDIA_TYPE" \ --digest "$DIGEST" \ diff --git a/tasks/create-pyxis-image/tests/mocks.sh b/tasks/create-pyxis-image/tests/mocks.sh index 086c550ce..26b6052bf 100644 --- a/tasks/create-pyxis-image/tests/mocks.sh +++ b/tasks/create-pyxis-image/tests/mocks.sh @@ -9,7 +9,7 @@ function create_container_image() { # e.g. 0001, 0002, 0003... echo The image id is $(awk 'END{printf("%04i", NR)}' $(workspaces.data.path)/mock_create_container_image.txt) - if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch /tmp/oras-manifest-fetch.json --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]] + if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch "*" --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]] then echo Error: Unexpected call echo Mock create_container_image called with: $* @@ -35,7 +35,7 @@ function skopeo() { echo '{"mediaType": "application/vnd.oci.image.index.v1+json"}' elif [[ "$*" == "inspect --raw docker://"* ]] || [[ "$*" == "inspect --no-tags --override-os linux --override-arch "*" docker://"* ]] then - echo '{"mediaType": "my_media_type"}' + echo '{"mediaType": "my_media_type+gzip"}' else echo Mock skopeo called with: $* if [[ "$*" != "inspect --no-tags docker://"* ]] @@ -64,9 +64,18 @@ function oras() { if [[ "$*" == "manifest fetch --registry-config"*.dockerfile ]] then echo '{"layers": [{"annotations": {"org.opencontainers.image.title": "Dockerfile.custom"}}]}' + elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-beef.gz"* ]] + then + echo -n 'H4sIAAAAAAAAA0vKzEssqlRISSxJVEjPTy1WyEgtSgUAXVhZVxUAAAA=' | base64 -d > /tmp/oras-blob-fetch-beef.gz + elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-pork.gz"* ]] + then + echo -n 'H4sIAAAAAAAAA8vNL0pVSEksSQQA2pxWLAkAAAA=' | base64 -d > /tmp/oras-blob-fetch-pork.gz + elif [[ "$*" == "manifest fetch --registry-config"*image-with-gzipped-layers* ]] + then + echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+gzip", "digest": "beef"}, {"mediaType": "blob+gzip", "digest": "pork"}]}' elif [[ "$*" == "manifest fetch --registry-config"* ]] then - echo '{"mediaType": "my_media_type"}' + echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+other", "digest": "tofu"}]}' elif [[ "$*" == "pull --registry-config"*dockerfile-not-found:sha256-*.dockerfile* ]] then echo Mock oras called with: $* diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml index 169d0a7a5..352800aad 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml @@ -20,7 +20,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux @@ -70,7 +70,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml index 3fb5d8905..1c45032ae 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml @@ -22,7 +22,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env sh set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml index a19352b4e..26afb3158 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -85,7 +85,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml index 45c9254cf..373353781 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -85,7 +85,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml index ea9820397..ec9033e28 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -69,7 +69,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml index ebb132a50..546a7ff3e 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml @@ -18,7 +18,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -68,7 +68,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml index 58a05ab90..823ed7753 100644 --- a/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml @@ -19,7 +19,7 @@ spec: - name: data steps: - name: setup-values - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux @@ -66,7 +66,7 @@ spec: - name: data steps: - name: check-result - image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 script: | #!/usr/bin/env bash set -eux diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml new file mode 100644 index 000000000..fa1592b47 --- /dev/null +++ b/tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-create-pyxis-image-with-gzipped-layers +spec: + description: | + Run the create-pyxis-image task with an image that has gzipped layers, to + see that we reported the uncompressed digests correctly + workspaces: + - name: tests-workspace + tasks: + - name: setup + workspaces: + - name: data + workspace: tests-workspace + taskSpec: + workspaces: + - name: data + steps: + - name: setup-values + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 + script: | + #!/usr/bin/env bash + set -eux + + cat > "$(workspaces.data.path)"/mapped_snapshot.json << EOF + { + "application": "myapp", + "components": [ + { + "name": "comp", + "containerImage": "source@sha256:mydigest", + "repository": "registry.io/image-with-gzipped-layers", + "tags": [ + "testtag" + ] + } + ] + } + EOF + - name: run-task + taskRef: + name: create-pyxis-image + params: + - name: pyxisSecret + value: test-create-pyxis-image-cert + - name: server + value: stage + - name: snapshotPath + value: mapped_snapshot.json + workspaces: + - name: data + workspace: tests-workspace + runAfter: + - setup + - name: check-result + params: + - name: pyxisDataPath + value: $(tasks.run-task.results.pyxisDataPath) + workspaces: + - name: data + workspace: tests-workspace + taskSpec: + params: + - name: pyxisDataPath + type: string + workspaces: + - name: data + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77 + script: | + #!/usr/bin/env bash + set -eux + + if [ "$(wc -l < "$(workspaces.data.path)"/mock_create_container_image.txt)" != 1 ]; then + echo Error: create_container_image was expected to be called 1 time. Actual calls: + cat "$(workspaces.data.path)/mock_create_container_image.txt" + exit 1 + fi + + if [ "$(wc -l < "$(workspaces.data.path)"/mock_oras.txt)" != 5 ]; then + echo Error: oras was expected to be called 4 times. Actual calls: + cat "$(workspaces.data.path)/mock_oras.txt" + exit 1 + fi + + [ "$(head -n 1 < "$(workspaces.data.path)"/mock_skopeo.txt)" \ + = "inspect --raw docker://registry.io/image-with-gzipped-layers@sha256:mydigest" ] + + # check that the size of the decompressed layers is as expected + jq -e '.uncompressed_layers[0].size == 21' "$(workspaces.data.path)/oras-manifest-fetch.json" + jq -e '.uncompressed_layers[1].size == 9' "$(workspaces.data.path)/oras-manifest-fetch.json" + + runAfter: + - run-task