From d8df1d6f0306294fe4e2b623b75b45aa86d60a3d Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Fri, 5 Jul 2024 21:00:16 -0400
Subject: [PATCH 1/3] Refactor oci-copy to be more efficient

Originally, this task would download all artifacts requested in the
input file, check them all, and then upload them all to the registry in
one invocation of "oras push".

This had two problems. First, if "oras push" flaked out part way
through and the user needed to retry their pipeline, the entire download
section would need to be run again needlessly. Second, for extremely
large artifacts with lots of medium-sized files, an enormous PVC would
be needed to hold all of them between download and push to the registry.

The change here addresses both problems.

First, files are downloaded, checked, pushed to the registry and then
deleted from local storage - one at a time. This obviates the need for a
large volume to store all files at once, since only enough storage is
needed to store one file, not all of them.

Second, as files are considered, first the registry is checked to see if
the blob has already been pushed there. If it has, then skip the
download step. This has the effect of greatly improving the runtime for
artifacts where only one or two of many files have changed since the
last taskrun.
---
 task/oci-copy/0.1/oci-copy.yaml | 77 +++++++++++++++++++++++++--------
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/task/oci-copy/0.1/oci-copy.yaml b/task/oci-copy/0.1/oci-copy.yaml
index ab21991de3..7e6e714011 100644
--- a/task/oci-copy/0.1/oci-copy.yaml
+++ b/task/oci-copy/0.1/oci-copy.yaml
@@ -99,36 +99,75 @@ spec:
 
         set -u
 
+        echo "Selecting auth for $IMAGE"
+        select-oci-auth $IMAGE > auth.json
+
+        echo "Extracting artifact_type"
+        ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
+
+        REPO=$(echo ${IMAGE} | awk -F ':' '{print $1}')
+        echo "Found that ${REPO} is the repository for ${IMAGE}"
+
+        cat >artifact-manifest.json <<EOL
+        {
+          "schemaVersion": 2,
+          "mediaType": "application/vnd.oci.image.manifest.v1+json",
+          "artifactType": "${ARTIFACT_TYPE}",
+          "config": {
+            "mediaType": "application/vnd.oci.empty.v1+json",
+            "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
+            "size": 2,
+            "data": "e30="
+          },
+          "layers": [],
+          "annotations": {
+            "org.opencontainers.image.created": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          }
+        }
+        EOL
+
         for varfile in /var/workdir/vars/*; do
           echo "Reading $varfile"
           source $varfile
 
-          echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
-          curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
+          echo "Checking to see if blob $OCI_ARTIFACT_DIGEST exists"
+          if [[ $(oras blob fetch --registry-config auth.json --descriptor "${REPO}@sha256:${OCI_ARTIFACT_DIGEST}") ]]; then
+            echo "Blob for ${OCI_FILENAME} already exists in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}. Skipping download."
+          else
+            echo "Blob for ${OCI_FILENAME} does not yet exist in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}."
+            echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
+            curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
 
-          echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
-          echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
+            echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
+            echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
 
-          echo "Appending to arguments for $OCI_FILENAME of type $OCI_ARTIFACT_TYPE"
-          args+=("${OCI_FILENAME}:${OCI_ARTIFACT_TYPE}")
-        done
+            echo "Pushing blob of $OCI_FILENAME of type $OCI_ARTIFACT_TYPE"
+            oras blob push --registry-config auth.json ${REPO} --media-type ${OCI_ARTIFACT_TYPE} ${OCI_FILENAME}
 
-        if [ -z "${args}" ]; then
-          echo "No files found. Something is very wrong. Skipping upload."
-          exit 1;
-        fi
+            echo "Removing local copy of $OCI_FILENAME to save space."
+            rm ${OCI_FILENAME}
+          fi
 
-        echo "Extracting artifact_type"
-        ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
+          echo "Grabbing descriptor of blob from the registry"
+          oras blob fetch --registry-config auth.json --descriptor "${REPO}@sha256:${OCI_ARTIFACT_DIGEST}" > descriptor.json
 
-        echo "Selecting auth for $IMAGE"
-        select-oci-auth $IMAGE > auth.json
+          echo "Setting mediaType to ${OCI_ARTIFACT_TYPE}"
+          yq -oj -i '.mediaType = "'${OCI_ARTIFACT_TYPE}'"' descriptor.json
+
+          echo "Inserting org.opencontainers.image.title = ${OCI_FILENAME} annotation"
+          yq -oj -i '.annotations."org.opencontainers.image.title" = "'${OCI_FILENAME}'"' descriptor.json
+
+          echo "Appending blob descriptor for ${OCI_FILENAME} to the overall artifact manifest for ${IMAGE}"
+          yq -oj -i ".layers += $(cat descriptor.json)" artifact-manifest.json
+
+          echo "Done with ${OCI_FILENAME}."
+        done
 
-        echo "Pushing contents to ${IMAGE}"
-        oras push --no-tty --registry-config auth.json --artifact-type ${ARTIFACT_TYPE} "${IMAGE}" "${args[@]}"
+        echo "Pushing complete artifact manifest to ${IMAGE}"
+        oras manifest push --no-tty --registry-config auth.json "${IMAGE}" artifact-manifest.json
 
-        IMAGE_INDEX_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}")
-        echo -n "$IMAGE_INDEX_DIGEST" | tee "$(results.IMAGE_DIGEST.path)"
+        RESULTING_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}")
+        echo -n "$RESULTING_DIGEST" | tee "$(results.IMAGE_DIGEST.path)"
         echo -n "$IMAGE" | tee "$(results.IMAGE_URL.path)"
       volumeMounts:
         - mountPath: /var/lib/containers

From 0829517c59cfd3a9ff7c1a9ce4181889d5531b78 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Fri, 5 Jul 2024 21:05:56 -0400
Subject: [PATCH 2/3] chore: run hack/generate-ta-tasks.sh

---
 task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml | 77 ++++++++++++++-----
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
index f6b97faf03..e3be6485a7 100644
--- a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
+++ b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
@@ -114,36 +114,75 @@ spec:
 
         set -u
 
+        echo "Selecting auth for $IMAGE"
+        select-oci-auth $IMAGE >auth.json
+
+        echo "Extracting artifact_type"
+        ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
+
+        REPO=$(echo ${IMAGE} | awk -F ':' '{print $1}')
+        echo "Found that ${REPO} is the repository for ${IMAGE}"
+
+        cat >artifact-manifest.json <<EOL
+        {
+          "schemaVersion": 2,
+          "mediaType": "application/vnd.oci.image.manifest.v1+json",
+          "artifactType": "${ARTIFACT_TYPE}",
+          "config": {
+            "mediaType": "application/vnd.oci.empty.v1+json",
+            "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
+            "size": 2,
+            "data": "e30="
+          },
+          "layers": [],
+          "annotations": {
+            "org.opencontainers.image.created": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          }
+        }
+        EOL
+
         for varfile in /var/workdir/vars/*; do
           echo "Reading $varfile"
           source $varfile
 
-          echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
-          curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
+          echo "Checking to see if blob $OCI_ARTIFACT_DIGEST exists"
+          if [[ $(oras blob fetch --registry-config auth.json --descriptor "${REPO}@sha256:${OCI_ARTIFACT_DIGEST}") ]]; then
+            echo "Blob for ${OCI_FILENAME} already exists in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}. Skipping download."
+          else
+            echo "Blob for ${OCI_FILENAME} does not yet exist in the registry at ${REPO}@sha256:${OCI_ARTIFACT_DIGEST}."
+            echo "Downloading $OCI_SOURCE to $OCI_FILENAME"
+            curl "${CURL_ARGS[@]}" --fail --silent --show-error --location $OCI_SOURCE -o $OCI_FILENAME
 
-          echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
-          echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
+            echo "Confirming that digest of $OCI_FILENAME matches expected $OCI_ARTIFACT_DIGEST"
+            echo "$OCI_ARTIFACT_DIGEST $OCI_FILENAME" | sha256sum --check
 
-          echo "Appending to arguments for $OCI_FILENAME of type $OCI_ARTIFACT_TYPE"
-          args+=("${OCI_FILENAME}:${OCI_ARTIFACT_TYPE}")
-        done
+            echo "Pushing blob of $OCI_FILENAME of type $OCI_ARTIFACT_TYPE"
+            oras blob push --registry-config auth.json ${REPO} --media-type ${OCI_ARTIFACT_TYPE} ${OCI_FILENAME}
 
-        if [ -z "${args}" ]; then
-          echo "No files found. Something is very wrong. Skipping upload."
-          exit 1
-        fi
+            echo "Removing local copy of $OCI_FILENAME to save space."
+            rm ${OCI_FILENAME}
+          fi
 
-        echo "Extracting artifact_type"
-        ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
+          echo "Grabbing descriptor of blob from the registry"
+          oras blob fetch --registry-config auth.json --descriptor "${REPO}@sha256:${OCI_ARTIFACT_DIGEST}" >descriptor.json
 
-        echo "Selecting auth for $IMAGE"
-        select-oci-auth $IMAGE >auth.json
+          echo "Setting mediaType to ${OCI_ARTIFACT_TYPE}"
+          yq -oj -i '.mediaType = "'${OCI_ARTIFACT_TYPE}'"' descriptor.json
+
+          echo "Inserting org.opencontainers.image.title = ${OCI_FILENAME} annotation"
+          yq -oj -i '.annotations."org.opencontainers.image.title" = "'${OCI_FILENAME}'"' descriptor.json
+
+          echo "Appending blob descriptor for ${OCI_FILENAME} to the overall artifact manifest for ${IMAGE}"
+          yq -oj -i ".layers += $(cat descriptor.json)" artifact-manifest.json
+
+          echo "Done with ${OCI_FILENAME}."
+        done
 
-        echo "Pushing contents to ${IMAGE}"
-        oras push --no-tty --registry-config auth.json --artifact-type ${ARTIFACT_TYPE} "${IMAGE}" "${args[@]}"
+        echo "Pushing complete artifact manifest to ${IMAGE}"
+        oras manifest push --no-tty --registry-config auth.json "${IMAGE}" artifact-manifest.json
 
-        IMAGE_INDEX_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}")
-        echo -n "$IMAGE_INDEX_DIGEST" | tee "$(results.IMAGE_DIGEST.path)"
+        RESULTING_DIGEST=$(oras resolve --registry-config auth.json "${IMAGE}")
+        echo -n "$RESULTING_DIGEST" | tee "$(results.IMAGE_DIGEST.path)"
         echo -n "$IMAGE" | tee "$(results.IMAGE_URL.path)"
       computeResources:
         limits:

From da59266c70b78cc0dddf8fa1255e32fcb486908f Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Mon, 8 Jul 2024 09:50:32 -0400
Subject: [PATCH 3/3] Improve method of deriving repo name from pullspec

Theoretically, this works if the IMAGE reference contains a port number.

Co-authored-by: Adam Cmiel <acmiel@redhat.com>
---
 task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml | 2 +-
 task/oci-copy/0.1/oci-copy.yaml               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
index e3be6485a7..0c4ebf28fe 100644
--- a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
+++ b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml
@@ -120,7 +120,7 @@ spec:
         echo "Extracting artifact_type"
         ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
 
-        REPO=$(echo ${IMAGE} | awk -F ':' '{print $1}')
+        REPO=${IMAGE%:*}
         echo "Found that ${REPO} is the repository for ${IMAGE}"
 
         cat >artifact-manifest.json <<EOL
diff --git a/task/oci-copy/0.1/oci-copy.yaml b/task/oci-copy/0.1/oci-copy.yaml
index 7e6e714011..23e3afb735 100644
--- a/task/oci-copy/0.1/oci-copy.yaml
+++ b/task/oci-copy/0.1/oci-copy.yaml
@@ -105,7 +105,7 @@ spec:
         echo "Extracting artifact_type"
         ARTIFACT_TYPE=$(cat "$(pwd)/source/$OCI_COPY_FILE" | yq '.artifact_type')
 
-        REPO=$(echo ${IMAGE} | awk -F ':' '{print $1}')
+        REPO=${IMAGE%:*}
         echo "Found that ${REPO} is the repository for ${IMAGE}"
 
         cat >artifact-manifest.json <<EOL