From 45c51cbdc3b15093044e822c79ada2cdd2df6681 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 24 Jun 2024 09:49:37 -0400 Subject: [PATCH] Refactor oci-copy task Update this to produce a single OCI artifact with a layer for each file, rather than an image index with an OCI artifact for each file. --- task/oci-copy-oci-ta/0.1/README.md | 1 - task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml | 46 +++++++---------- task/oci-copy/0.1/README.md | 7 ++- task/oci-copy/0.1/oci-copy.yaml | 49 +++++++------------ 4 files changed, 41 insertions(+), 62 deletions(-) diff --git a/task/oci-copy-oci-ta/0.1/README.md b/task/oci-copy-oci-ta/0.1/README.md index 5302d745b3..65cd28337b 100644 --- a/task/oci-copy-oci-ta/0.1/README.md +++ b/task/oci-copy-oci-ta/0.1/README.md @@ -12,7 +12,6 @@ Given a file in the user's source directory, copy content from arbitrary urls in ## Results |name|description| |---|---| -|IMAGES|URIs for all image manifests published, for signing| |IMAGE_DIGEST|Digest of the artifact just pushed| |IMAGE_URL|Repository where the artifact was pushed| diff --git a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml index a9cef1085d..a37030e6ea 100644 --- a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml +++ b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml @@ -25,8 +25,6 @@ spec: the application source code. type: string results: - - name: IMAGES - description: URIs for all image manifests published, for signing - name: IMAGE_DIGEST description: Digest of the artifact just pushed - name: IMAGE_URL @@ -62,8 +60,7 @@ spec: set -eu set -o pipefail - SOURCE_CODE_DIR=source - oci_copy_file_path="$(pwd)/$SOURCE_CODE_DIR/$OCI_COPY_FILE" + oci_copy_file_path="$(pwd)/source/$OCI_COPY_FILE" mkdir -p /var/workdir/vars/ @@ -83,7 +80,7 @@ spec: cat /var/workdir/vars/$filename done - name: oci-copy - image: quay.io/redhat-appstudio/buildah:v1.35.4@sha256:3d3575bb7d0df64abcf1f22f06e82101a945d03317db1f3caac12814f796d01c + image: quay.io/konflux-ci/oras:latest@sha256:04cbd5efbcf1a38944b050e3adbc3071218643f6aa92e95d5d25b173574bca5e workingDir: /var/workdir volumeMounts: - mountPath: /var/lib/containers @@ -92,8 +89,6 @@ spec: set -eu set -o pipefail - buildah manifest create $IMAGE - for varfile in /var/workdir/vars/*; do echo "Reading $varfile" source $varfile @@ -104,21 +99,27 @@ spec: echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST" echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check - echo "Building manifest of type $OCI_ARTIFACT_TYPE from $OCI_FILENAME" - buildah manifest add $IMAGE --artifact --artifact-type $OCI_ARTIFACT_TYPE $OCI_FILENAME + echo "Appending to arguments for $OCI_FILENAME of type $OCI_ARTIFACT_TYPE" + args+=("${OCI_FILENAME}:${OCI_ARTIFACT_TYPE}") done - echo "Pushing conents to $IMAGE" - buildah manifest push \ - --digestfile /var/workdir/image-digest \ - --authfile $HOME/.docker/config.json \ - --all \ - $IMAGE + if [ -z "${args}" ]; then + echo "No files found. Something is very wrong. Skipping upload." + exit 1 + fi + + echo "Extracting artifact_type" + ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type') + + echo "Selecting auth for $IMAGE" + select-oci-auth $IMAGE >auth.json + + echo "Pushing contents to ${IMAGE}" + oras push --no-tty --registry-config auth.json --artifact-type ${ARTIFACT_TYPE} "${IMAGE}" "${args[@]}" - IMAGE_INDEX_DIGEST=$(cat "/var/workdir"/image-digest) + IMAGE_INDEX_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}") echo -n "$IMAGE_INDEX_DIGEST" | tee "$(results.IMAGE_DIGEST.path)" echo -n "$IMAGE" | tee "$(results.IMAGE_URL.path)" - buildah manifest inspect $IMAGE@$IMAGE_INDEX_DIGEST | tee "/var/workdir"/image-manifests computeResources: limits: memory: 1Gi @@ -129,17 +130,6 @@ spec: capabilities: add: - SETFCAP - - name: artifact-manifest-generate - image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430 - workingDir: /var/workdir - script: | - MANIFEST_DIGESTS=$(cat "/var/workdir"/image-manifests | yq -r '.manifests[].digest') - for MANIFEST_DIGEST in $MANIFEST_DIGESTS; do - reference="$IMAGE@$MANIFEST_DIGEST, " - echo "Writing ${reference} to $(results.IMAGES.path)" - echo -n "${reference}" >>$(results.IMAGES.path) - done - echo -n "$IMAGE@$(cat $(results.IMAGE_DIGEST.path))" >>$(results.IMAGES.path) - name: sbom-generate image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430 workingDir: /var/workdir diff --git a/task/oci-copy/0.1/README.md b/task/oci-copy/0.1/README.md index 07fb360922..bb2826b124 100644 --- a/task/oci-copy/0.1/README.md +++ b/task/oci-copy/0.1/README.md @@ -17,7 +17,6 @@ It is not to be considered safe for general use as it cannot provide a high degr |---|---| |IMAGE_DIGEST|Digest of the image just built| |IMAGE_URL|Image repository where the built image was pushed| -|IMAGES|A comma separate list of all the individual image manifests produced| ## Workspaces |name|description|optional| @@ -30,8 +29,12 @@ JSON schema for the `oci-copy.yaml` file. ```json { "type": "object", - "required": ["artifacts"], + "required": ["artifacts", "artifact_type"], "properties": { + "artifact_type": { + "description": "Artifact type to be applied to the top-level OCI artifact, i.e. `application/x-mlmodel`", + "type": "string" + }, "artifacts": { "type": "array", "items": { diff --git a/task/oci-copy/0.1/oci-copy.yaml b/task/oci-copy/0.1/oci-copy.yaml index da97f87f2d..189083a80c 100644 --- a/task/oci-copy/0.1/oci-copy.yaml +++ b/task/oci-copy/0.1/oci-copy.yaml @@ -23,8 +23,6 @@ spec: name: IMAGE_DIGEST - description: Repository where the artifact was pushed name: IMAGE_URL - - description: URIs for all image manifests published, for signing - name: IMAGES stepTemplate: env: - name: OCI_COPY_FILE @@ -41,8 +39,7 @@ spec: set -eu set -o pipefail - SOURCE_CODE_DIR=source - oci_copy_file_path="$(pwd)/$SOURCE_CODE_DIR/$OCI_COPY_FILE" + oci_copy_file_path="$(pwd)/source/$OCI_COPY_FILE" mkdir -p /var/workdir/vars/ @@ -62,9 +59,8 @@ spec: cat /var/workdir/vars/$filename done workingDir: $(workspaces.source.path) - - name: oci-copy - image: quay.io/redhat-appstudio/buildah:v1.35.4@sha256:3d3575bb7d0df64abcf1f22f06e82101a945d03317db1f3caac12814f796d01c + image: quay.io/konflux-ci/oras:latest@sha256:04cbd5efbcf1a38944b050e3adbc3071218643f6aa92e95d5d25b173574bca5e computeResources: limits: memory: 1Gi @@ -79,8 +75,6 @@ spec: set -eu set -o pipefail - buildah manifest create $IMAGE - for varfile in /var/workdir/vars/*; do echo "Reading $varfile" source $varfile @@ -91,38 +85,31 @@ spec: echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST" echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check - echo "Building manifest of type $OCI_ARTIFACT_TYPE from $OCI_FILENAME" - buildah manifest add $IMAGE --artifact --artifact-type $OCI_ARTIFACT_TYPE $OCI_FILENAME + echo "Appending to arguments for $OCI_FILENAME of type $OCI_ARTIFACT_TYPE" + args+=("${OCI_FILENAME}:${OCI_ARTIFACT_TYPE}") done - echo "Pushing conents to $IMAGE" - buildah manifest push \ - --digestfile $(workspaces.source.path)/image-digest \ - --authfile $HOME/.docker/config.json \ - --all \ - $IMAGE + if [ -z "${args}" ]; then + echo "No files found. Something is very wrong. Skipping upload." + exit 1; + fi + + echo "Extracting artifact_type" + ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type') - IMAGE_INDEX_DIGEST=$(cat "$(workspaces.source.path)"/image-digest) + echo "Selecting auth for $IMAGE" + select-oci-auth $IMAGE > auth.json + + echo "Pushing contents to ${IMAGE}" + oras push --no-tty --registry-config auth.json --artifact-type ${ARTIFACT_TYPE} "${IMAGE}" "${args[@]}" + + IMAGE_INDEX_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}") echo -n "$IMAGE_INDEX_DIGEST" | tee "$(results.IMAGE_DIGEST.path)" echo -n "$IMAGE" | tee "$(results.IMAGE_URL.path)" - buildah manifest inspect $IMAGE@$IMAGE_INDEX_DIGEST | tee "$(workspaces.source.path)"/image-manifests volumeMounts: - mountPath: /var/lib/containers name: varlibcontainers workingDir: $(workspaces.source.path) - - - name: artifact-manifest-generate - image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430 - script: | - MANIFEST_DIGESTS=$(cat "$(workspaces.source.path)"/image-manifests | yq -r '.manifests[].digest') - for MANIFEST_DIGEST in $MANIFEST_DIGESTS; do - reference="$IMAGE@$MANIFEST_DIGEST, " - echo "Writing ${reference} to $(results.IMAGES.path)" - echo -n "${reference}" >> $(results.IMAGES.path) - done - echo -n "$IMAGE@$(cat $(results.IMAGE_DIGEST.path))" >> $(results.IMAGES.path) - workingDir: $(workspaces.source.path) - - name: sbom-generate image: quay.io/konflux-ci/yq:latest@sha256:974dea6375ee9df561ffd3baf994db2b61777a71f3bcf0050c5dca91ac9b3430 script: |