Skip to content

Commit

Permalink
feat(KONFLUX-4892): supply layer size information to pyxis
Browse files Browse the repository at this point in the history
This is meant to be paired with a change in the step image:
konflux-ci/release-service-utils#284

If the layers of the image are found to be gzip compressed, then pull
them, decompress them, and include information about their decompressed
digests and sizes in the manifest provided to the pyxis script.

Signed-off-by: Ralph Bean <[email protected]>
  • Loading branch information
ralphbean committed Oct 30, 2024
1 parent fdc2026 commit 19e62fe
Show file tree
Hide file tree
Showing 11 changed files with 175 additions and 20 deletions.
2 changes: 2 additions & 0 deletions tasks/create-pyxis-image/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ The relative path of the pyxis.json file in the data workspace is output as a ta
| isLatest | If set to true, the images will have a latest tag added with their Pyxis entries | Yes | false |
| rhPush | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes | false |
| snapshotPath | Path to the JSON string of the mapped Snapshot spec in the data workspace | No | - |
## Changes in 3.4.0
* Added support for uploading uncompressed layer information to pyxis

## Changes in 3.3.4
* Properly expand platform variables for oras args
Expand Down
55 changes: 51 additions & 4 deletions tasks/create-pyxis-image/create-pyxis-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ kind: Task
metadata:
name: create-pyxis-image
labels:
app.kubernetes.io/version: "3.3.4"
app.kubernetes.io/version: "3.4.0"
annotations:
tekton.dev/pipelines.minVersion: "0.12.1"
tekton.dev/tags: release
Expand Down Expand Up @@ -48,7 +48,7 @@ spec:
description: The relative path in the workspace to the stored pyxis data json
steps:
- name: create-pyxis-image
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
env:
- name: pyxisCert
valueFrom:
Expand Down Expand Up @@ -156,19 +156,66 @@ spec:
ORAS_ARGS+=(--platform "$OS/$ARCH")
fi
# Save the OCI manifest locally, to pass to a script to create the pyxis entry
MANIFEST_FILE="$(workspaces.data.path)/$(dirname "$(params.snapshotPath)")/oras-manifest-fetch.json"
oras manifest fetch \
--registry-config "$AUTH_FILE" \
"${ORAS_ARGS[@]}" \
"${PULLSPEC}" \
| tee /tmp/oras-manifest-fetch.json
| tee "${MANIFEST_FILE}"
# Augment that manifest with further information about the layers, decompressed
# This requires pulling the layers to decompress and then measure them
while IFS= read -r BLOB_DETAIL;
do
BLOB_TYPE=$(jq -r '.mediaType' <<< "$BLOB_DETAIL")
BLOB_DIGEST=$(jq -r '.digest' <<< "$BLOB_DETAIL")
# Normal images will always have the layers compressed.
# If they are not compressed, this will not save the
# uncompressed data in Pyxis.
#
# It's also possible that the layers are compressed with
# some scheme other than gzip. In that case, the
# uncompressed layer information will also not be saved in
# Pyxis.
# https://github.com/konflux-ci/build-definitions/issues/1264
if [[ "$BLOB_TYPE" =~ ^.*\.gzip$|^.*\+gzip$ ]]; then
BLOB_FILE="/tmp/oras-blob-fetch-${BLOB_DIGEST}"
BLOB_PULLSPEC="${PULLSPEC%%@*}@${BLOB_DIGEST}"
# Save the blob
oras blob fetch \
--registry-config "$AUTH_FILE" \
--output "${BLOB_FILE}.gz" \
"${BLOB_PULLSPEC}"
# Decompress it
gunzip "${BLOB_FILE}.gz"
# Measure it
EXPANDED_DIGEST=$(sha256sum "${BLOB_FILE}" | cut -d " " -f 1)
EXPANDED_SIZE=$(wc --bytes "${BLOB_FILE}" | awk '{print $1}' | tr -d '\n')
# Append this information to the parsed_data manifest
jq \
'.uncompressed_layers += [{"digest": "'"$EXPANDED_DIGEST"'", "size": '"$EXPANDED_SIZE"'}]' \
"${MANIFEST_FILE}" > "${MANIFEST_FILE}.tmp"
mv "${MANIFEST_FILE}.tmp" "${MANIFEST_FILE}"
# Clean up, in case we're dealing with large images
rm "/tmp/oras-blob-fetch-${BLOB_DIGEST}"
fi
done <<< "$(jq -c '.layers[]' "${MANIFEST_FILE}")"
PYXIS_CERT_PATH=/tmp/crt PYXIS_KEY_PATH=/tmp/key create_container_image \
--pyxis-url $PYXIS_URL \
--certified "$(params.certified)" \
--tags "$TAGS" \
--is-latest "$(params.isLatest)" \
--verbose \
--oras-manifest-fetch "/tmp/oras-manifest-fetch.json" \
--oras-manifest-fetch "${MANIFEST_FILE}" \
--name "$REPOSITORY" \
--media-type "$MEDIA_TYPE" \
--digest "$DIGEST" \
Expand Down
15 changes: 12 additions & 3 deletions tasks/create-pyxis-image/tests/mocks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ function create_container_image() {
# e.g. 0001, 0002, 0003...
echo The image id is $(awk 'END{printf("%04i", NR)}' $(workspaces.data.path)/mock_create_container_image.txt)

if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch /tmp/oras-manifest-fetch.json --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]]
if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch "*" --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]]
then
echo Error: Unexpected call
echo Mock create_container_image called with: $*
Expand All @@ -35,7 +35,7 @@ function skopeo() {
echo '{"mediaType": "application/vnd.oci.image.index.v1+json"}'
elif [[ "$*" == "inspect --raw docker://"* ]] || [[ "$*" == "inspect --no-tags --override-os linux --override-arch "*" docker://"* ]]
then
echo '{"mediaType": "my_media_type"}'
echo '{"mediaType": "my_media_type+gzip"}'
else
echo Mock skopeo called with: $*
if [[ "$*" != "inspect --no-tags docker://"* ]]
Expand Down Expand Up @@ -64,9 +64,18 @@ function oras() {
if [[ "$*" == "manifest fetch --registry-config"*.dockerfile ]]
then
echo '{"layers": [{"annotations": {"org.opencontainers.image.title": "Dockerfile.custom"}}]}'
elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-beef.gz"* ]]
then
echo -n 'H4sIAAAAAAAAA0vKzEssqlRISSxJVEjPTy1WyEgtSgUAXVhZVxUAAAA=' | base64 -d > /tmp/oras-blob-fetch-beef.gz
elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-pork.gz"* ]]
then
echo -n 'H4sIAAAAAAAAA8vNL0pVSEksSQQA2pxWLAkAAAA=' | base64 -d > /tmp/oras-blob-fetch-pork.gz
elif [[ "$*" == "manifest fetch --registry-config"*image-with-gzipped-layers* ]]
then
echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+gzip", "digest": "beef"}, {"mediaType": "blob+gzip", "digest": "pork"}]}'
elif [[ "$*" == "manifest fetch --registry-config"* ]]
then
echo '{"mediaType": "my_media_type"}'
echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+other", "digest": "tofu"}]}'
elif [[ "$*" == "pull --registry-config"*dockerfile-not-found:sha256-*.dockerfile* ]]
then
echo Mock oras called with: $*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env sh
set -eux
Expand Down Expand Up @@ -70,7 +70,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env sh
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env sh
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down Expand Up @@ -85,7 +85,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down Expand Up @@ -85,7 +85,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down Expand Up @@ -69,7 +69,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down Expand Up @@ -68,7 +68,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down Expand Up @@ -66,7 +66,7 @@ spec:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
---
apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
name: test-create-pyxis-image-with-gzipped-layers
spec:
description: |
Run the create-pyxis-image task with an image that has gzipped layers, to
see that we reported the uncompressed digests correctly
workspaces:
- name: tests-workspace
tasks:
- name: setup
workspaces:
- name: data
workspace: tests-workspace
taskSpec:
workspaces:
- name: data
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
cat > "$(workspaces.data.path)"/mapped_snapshot.json << EOF
{
"application": "myapp",
"components": [
{
"name": "comp",
"containerImage": "source@sha256:mydigest",
"repository": "registry.io/image-with-gzipped-layers",
"tags": [
"testtag"
]
}
]
}
EOF
- name: run-task
taskRef:
name: create-pyxis-image
params:
- name: pyxisSecret
value: test-create-pyxis-image-cert
- name: server
value: stage
- name: snapshotPath
value: mapped_snapshot.json
workspaces:
- name: data
workspace: tests-workspace
runAfter:
- setup
- name: check-result
params:
- name: pyxisDataPath
value: $(tasks.run-task.results.pyxisDataPath)
workspaces:
- name: data
workspace: tests-workspace
taskSpec:
params:
- name: pyxisDataPath
type: string
workspaces:
- name: data
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
script: |
#!/usr/bin/env bash
set -eux
if [ "$(wc -l < "$(workspaces.data.path)"/mock_create_container_image.txt)" != 1 ]; then
echo Error: create_container_image was expected to be called 1 time. Actual calls:
cat "$(workspaces.data.path)/mock_create_container_image.txt"
exit 1
fi
if [ "$(wc -l < "$(workspaces.data.path)"/mock_oras.txt)" != 5 ]; then
echo Error: oras was expected to be called 4 times. Actual calls:
cat "$(workspaces.data.path)/mock_oras.txt"
exit 1
fi
[ "$(head -n 1 < "$(workspaces.data.path)"/mock_skopeo.txt)" \
= "inspect --raw docker://registry.io/image-with-gzipped-layers@sha256:mydigest" ]
# check that the size of the decompressed layers is as expected
jq -e '.uncompressed_layers[0].size == 21' "$(workspaces.data.path)/oras-manifest-fetch.json"
jq -e '.uncompressed_layers[1].size == 9' "$(workspaces.data.path)/oras-manifest-fetch.json"
runAfter:
- run-task

0 comments on commit 19e62fe

Please sign in to comment.