feat(RELEASE-1336): skip image layers in Pyxis with skipLayers (#730)

A new option is added to data file to be able to skip layer details when create Pyxis image. This was previously created as a one-off workaround to unblock bootc releases here: #684 Now it's an option that can be enabled as needed. For full explanation, see the jira and also the comment in the task script. Signed-off-by: Martin Malina <[email protected]>
konflux-ci · Dec 11, 2024 · 6bd8cf8 · 6bd8cf8
1 parent 5071da4
commit 6bd8cf8
Show file tree

Hide file tree

Showing 23 changed files with 238 additions and 13 deletions.
diff --git a/pipelines/rh-advisories/README.md b/pipelines/rh-advisories/README.md
@@ -23,6 +23,9 @@ the rh-push-to-registry-redhat-io pipeline.
 | taskGitUrl                      | The url to the git repo where the release-service-catalog tasks to be used are stored                                              | Yes      | https://github.com/konflux-ci/release-service-catalog.git |
 | taskGitRevision                 | The revision in the taskGitUrl repo to be used                                                                                     | No       | -                                                         |
 
+## Changes in 1.5.7
+* new mandatory parameter `dataPath` added to `create-pyxis-image` task
+
 ## Changes in 1.5.6
 * new mandatory parameter resultsDirPath added to run-file-updates task
 

diff --git a/pipelines/rh-advisories/rh-advisories.yaml b/pipelines/rh-advisories/rh-advisories.yaml
@@ -4,7 +4,7 @@ kind: Pipeline
 metadata:
   name: rh-advisories
   labels:
-    app.kubernetes.io/version: "1.5.6"
+    app.kubernetes.io/version: "1.5.7"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -422,6 +422,8 @@ spec:
           value: "true"
         - name: snapshotPath
           value: "$(tasks.collect-data.results.snapshotSpec)"
+        - name: dataPath
+          value: "$(tasks.collect-data.results.data)"
       workspaces:
         - name: data
           workspace: release-workspace

diff --git a/pipelines/rh-push-to-external-registry/README.md b/pipelines/rh-push-to-external-registry/README.md
@@ -19,6 +19,9 @@ Tekton pipeline to release Red Hat Snapshots to an external registry. This pipel
 | taskGitUrl              | The url to the git repo where the release-service-catalog tasks to be used are stored | Yes      | https://github.com/konflux-ci/release-service-catalog.git |
 | taskGitRevision         | The revision in the taskGitUrl repo to be used                                       | No        | -            |
 
+## Changes in 5.3.3
+* new mandatory parameter `dataPath` added to `create-pyxis-image` task
+
 ## Changes in 5.3.2
 * new mandatory parameter resultsDirPath added to run-file-updates task
 

diff --git a/pipelines/rh-push-to-external-registry/rh-push-to-external-registry.yaml b/pipelines/rh-push-to-external-registry/rh-push-to-external-registry.yaml
@@ -4,7 +4,7 @@ kind: Pipeline
 metadata:
   name: rh-push-to-external-registry
   labels:
-    app.kubernetes.io/version: "5.3.2"
+    app.kubernetes.io/version: "5.3.3"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -258,6 +258,8 @@ spec:
           value: $(tasks.collect-pyxis-params.results.secret)
         - name: snapshotPath
           value: "$(tasks.collect-data.results.snapshotSpec)"
+        - name: dataPath
+          value: "$(tasks.collect-data.results.data)"
       workspaces:
         - name: data
           workspace: release-workspace

diff --git a/pipelines/rh-push-to-registry-redhat-io/README.md b/pipelines/rh-push-to-registry-redhat-io/README.md
@@ -20,6 +20,9 @@ Tekton pipeline to release content to registry.redhat.io registry.
 | taskGitUrl                      | The url to the git repo where the release-service-catalog tasks to be used are stored                                              | Yes      | https://github.com/konflux-ci/release-service-catalog.git |
 | taskGitRevision                 | The revision in the taskGitUrl repo to be used                                                                                     | No       | -                                                         |
 
+## Changes in 4.5.6
+* new mandatory parameter `dataPath` added to `create-pyxis-image` task
+
 ## Changes in 4.5.5
 * new mandatory parameter resultsDirPath added to run-file-updates task
 

diff --git a/pipelines/rh-push-to-registry-redhat-io/rh-push-to-registry-redhat-io.yaml b/pipelines/rh-push-to-registry-redhat-io/rh-push-to-registry-redhat-io.yaml
@@ -4,7 +4,7 @@ kind: Pipeline
 metadata:
   name: rh-push-to-registry-redhat-io
   labels:
-    app.kubernetes.io/version: "4.5.5"
+    app.kubernetes.io/version: "4.5.6"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -376,6 +376,8 @@ spec:
           value: "true"
         - name: snapshotPath
           value: "$(tasks.collect-data.results.snapshotSpec)"
+        - name: dataPath
+          value: "$(tasks.collect-data.results.data)"
       workspaces:
         - name: data
           workspace: release-workspace

diff --git a/pipelines/rhtap-service-push/README.md b/pipelines/rhtap-service-push/README.md
@@ -21,6 +21,9 @@
 | taskGitUrl               | The url to the git repo where the release-service-catalog tasks to be used are stored   | Yes      | https://github.com/konflux-ci/release-service-catalog.git |
 | taskGitRevision          | The revision in the taskGitUrl repo to be used                                          | No       | -             |
 
+## Changes in 4.2.1
+* new mandatory parameter `dataPath` added to `create-pyxis-image` task
+
 ## Changes in 4.2.0
 * Add new reduce-snapshot task
 

diff --git a/pipelines/rhtap-service-push/rhtap-service-push.yaml b/pipelines/rhtap-service-push/rhtap-service-push.yaml
@@ -4,7 +4,7 @@ kind: Pipeline
 metadata:
   name: rhtap-service-push
   labels:
-    app.kubernetes.io/version: "4.2.0"
+    app.kubernetes.io/version: "4.2.1"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -278,6 +278,8 @@ spec:
           value: $(tasks.collect-pyxis-params.results.secret)
         - name: snapshotPath
           value: "$(tasks.collect-data.results.snapshotSpec)"
+        - name: dataPath
+          value: "$(tasks.collect-data.results.data)"
       workspaces:
         - name: data
           workspace: release-workspace

diff --git a/schema/dataKeys.json b/schema/dataKeys.json
@@ -315,6 +315,10 @@
         "skipRepoPublishing": {
           "type": "boolean",
           "description": "Skip setting the pyxis repo to published"
+        },
+        "skipLayers": {
+          "type": "boolean",
+          "description": "When creating ContainerImage in Pyxis, omit details about layers"
         }
       }
     },

diff --git a/tasks/create-pyxis-image/README.md b/tasks/create-pyxis-image/README.md
@@ -9,14 +9,20 @@ The relative path of the pyxis.json file in the data workspace is output as a ta
 
 ## Parameters
 
-| Name        | Description                                                                                                                                                                                                                                                                                                                                                                                                 | Optional | Default value |
-|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------|
-| server      | The server type to use. Options are 'production','production-internal,'stage-internal' and 'stage'.                                                                                                                                                                                                                                                                                                         | Yes      | production    |
-| pyxisSecret | The kubernetes secret to use to authenticate to Pyxis. It needs to contain two keys: key and cert                                                                                                                                                                                                                                                                                                           | No       | -             |
-| certified   | If set to true, the images will be marked as certified in their Pyxis entries                                                                                                                                                                                                                                                                                                                               | Yes      | false         |
-| isLatest    | If set to true, the images will have a latest tag added with their Pyxis entries                                                                                                                                                                                                                                                                                                                            | Yes      | false         |
-| rhPush      | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes      | false         |
+| Name         | Description                                                                                                                                                                                                                                                                                                                                                                                                 | Optional | Default value |
+| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------- |
+| server       | The server type to use. Options are 'production','production-internal,'stage-internal' and 'stage'.                                                                                                                                                                                                                                                                                                         | Yes      | production    |
+| pyxisSecret  | The kubernetes secret to use to authenticate to Pyxis. It needs to contain two keys: key and cert                                                                                                                                                                                                                                                                                                           | No       | -             |
+| certified    | If set to true, the images will be marked as certified in their Pyxis entries                                                                                                                                                                                                                                                                                                                               | Yes      | false         |
+| isLatest     | If set to true, the images will have a latest tag added with their Pyxis entries                                                                                                                                                                                                                                                                                                                            | Yes      | false         |
+| rhPush       | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes      | false         |
 | snapshotPath | Path to the JSON string of the mapped Snapshot spec in the data workspace                                                                                                                                                                                                                                                                                                                                   | No       | -             |
+| dataPath     | Path to the JSON string of the merged data to use in the data workspace                                                                                                                                                                                                                                                                                                                                     | No       |               |
+
+## Changes in 3.5.0
+* Added mandatory `dataPath` task parameter
+  * The data file can now contain `.pyxis.skipLayers` flag. If true, image layer information
+    is deleted before calling the `create_container_image` script
 
 ## Changes in 3.4.4
 * Bumped the utils image used in this task

diff --git a/tasks/create-pyxis-image/create-pyxis-image.yaml b/tasks/create-pyxis-image/create-pyxis-image.yaml
@@ -4,7 +4,7 @@ kind: Task
 metadata:
   name: create-pyxis-image
   labels:
-    app.kubernetes.io/version: "3.4.4"
+    app.kubernetes.io/version: "3.5.0"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -40,6 +40,9 @@ spec:
     - name: snapshotPath
       type: string
       description: Path to the JSON string of the mapped Snapshot spec in the data workspace
+    - name: dataPath
+      description: Path to the JSON string of the merged data to use in the data workspace
+      type: string
   workspaces:
     - name: data
       description: The workspace where the snapshot spec and data json files reside
@@ -91,6 +94,12 @@ spec:
             exit 1
         fi
 
+        DATA_FILE="$(workspaces.data.path)/$(params.dataPath)"
+        if [ ! -f "${DATA_FILE}" ] ; then
+            echo "No data JSON was provided."
+            exit 1
+        fi
+
         PYXIS_DATA_PATH="$(dirname "$(params.snapshotPath)")/pyxis.json"
         echo -n "${PYXIS_DATA_PATH}" > "$(results.pyxisDataPath.path)"
 
@@ -101,6 +110,9 @@ spec:
 
         AUTH_FILE=$(mktemp)
 
+        # Default to false
+        skipLayers="$(jq -r ".pyxis.skipLayers // false" "${DATA_FILE}")"
+
         COMPONENTS=$(jq '.components | length' "${SNAPSHOT_SPEC_FILE}")
         JSON_OUTPUT='{}'
         for (( i=0; i < COMPONENTS; i++ )); do
@@ -164,6 +176,16 @@ spec:
                   "${PULLSPEC}" \
                       | tee "${MANIFEST_FILE}"
 
+                # When building images without squashing, their final layer might always be bit-wise identical.
+                # This causes upload to pyxis to fail which cannot tolerate duplicate "top_layer_id" values.
+                # This flag allows to overcome this limitation by just deleting the layers so that no layer
+                # information is uploaded.
+                if [ "$skipLayers" = true ]; then
+                  echo ".pyxis.skipLayers is true in data file, so delete the layers"
+                  jq '.layers = []' "${MANIFEST_FILE}" > "${MANIFEST_FILE}.tmp"
+                  mv "${MANIFEST_FILE}.tmp"  "${MANIFEST_FILE}"
+                fi
+
                 # Augment that manifest with further information about the layers, decompressed
                 # This requires pulling the layers to decompress and then measure them
                 while IFS= read -r BLOB_DETAIL;

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml
@@ -40,6 +40,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -50,6 +55,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml
@@ -42,6 +42,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -52,6 +57,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-invalid-server.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-invalid-server.yaml
@@ -22,6 +22,8 @@ spec:
           value: test-create-pyxis-image-cert
         - name: snapshotPath
           value: ""
+        - name: dataPath
+          value: ""
       workspaces:
         - name: data
           workspace: tests-workspace
diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-no-snapshot-file.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-no-snapshot-file.yaml
@@ -20,6 +20,8 @@ spec:
           value: test-create-pyxis-image-cert
         - name: snapshotPath
           value: missing
+        - name: dataPath
+          value: ""
       workspaces:
         - name: data
           workspace: tests-workspace
diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml
@@ -55,6 +55,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -65,6 +70,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml
@@ -55,6 +55,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -65,6 +70,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-oci-artifact.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-oci-artifact.yaml
@@ -39,6 +39,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -49,6 +54,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml
@@ -39,6 +39,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -49,6 +54,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml
@@ -38,6 +38,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -48,6 +53,8 @@ spec:
           value: stage
         - name: snapshotPath
           value: mapped_snapshot.json
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace

diff --git a/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml b/tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml
@@ -40,6 +40,11 @@ spec:
                 ]
               }
               EOF
+
+              cat > "$(workspaces.data.path)/mydata.json" << EOF
+              {
+              }
+              EOF
     - name: run-task
       taskRef:
         name: create-pyxis-image
@@ -52,6 +57,8 @@ spec:
           value: mapped_snapshot.json
         - name: rhPush
           value: "true"
+        - name: dataPath
+          value: mydata.json
       workspaces:
         - name: data
           workspace: tests-workspace