Introduce trusted artifacts

Adds the use of trusted artifacts[1] to the Tasks and Pipelines. The `prefetch-dependencies` Task is no longer skipped based on the `hermetic` parameter -- it needs to run always to produce the trusted artifacts as the build* tasks depend on these, so if it were skipped the build* tasks would be skipped as well. With this the default value is set for the `*_ARTIFACT` parameters and the workspace will be unaltered by the `build-trusted-artifact` steps. This way it is simpler to opt-in to using trusted artifacts, and the default pipelines do so. Ref: https://issues.redhat.com/browse/EC-251 [1] https://github.com/redhat-appstudio/build-trusted-artifacts
konflux-ci · Jan 29, 2024 · a88f805 · a88f805
1 parent fc7e8fa
commit a88f805
Show file tree

Hide file tree

Showing 25 changed files with 299 additions and 36 deletions.
diff --git a/.github/workflows/check-buildah-remote.yaml b/.github/workflows/check-buildah-remote.yaml
@@ -1,5 +1,5 @@
 name: Validate PR - buildah-remote
-on:
+'on':
   pull_request:
     branches: [main]
 jobs:

diff --git a/.tekton/pull-request.yaml b/.tekton/pull-request.yaml
@@ -6,7 +6,7 @@ metadata:
   annotations:
     pipelinesascode.tekton.dev/on-event: "pull_request"
     pipelinesascode.tekton.dev/on-target-branch: "main"
-    pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.1/sast-snyk-check.yaml]"
+    pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/yaml-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.1/sast-snyk-check.yaml, .tekton/tasks/ec-checks.yaml]"
     pipelinesascode.tekton.dev/task-2: "yaml-lint"
     pipelinesascode.tekton.dev/max-keep-runs: "5"
 spec:
@@ -29,7 +29,9 @@ spec:
         taskRef:
           name: git-clone
         workspaces:
-          - name: output
+          - name: source
+            workspace: workspace
+          - name: artifacts
             workspace: workspace
         params:
           - name: url
@@ -58,7 +60,7 @@ spec:
         taskRef:
           name: sast-snyk-check
         workspaces:
-          - name: workspace
+          - name: artifacts
             workspace: workspace
       - name: build-container
         runAfter:
@@ -71,7 +73,7 @@ spec:
         taskRef:
           name: buildah
         workspaces:
-          - name: source
+          - name: artifacts
             workspace: workspace
       - name: check-partner-tasks
         runAfter:

diff --git a/.tekton/push.yaml b/.tekton/push.yaml
@@ -34,7 +34,9 @@ spec:
         taskRef:
           name: git-clone
         workspaces:
-          - name: output
+          - name: source
+            workspace: workspace
+          - name: artifacts
             workspace: workspace
       - name: build-container
         params:
@@ -47,7 +49,7 @@ spec:
         taskRef:
           name: buildah
         workspaces:
-          - name: source
+          - name: artifacts
             workspace: workspace
       - name: build-bundles
         params:

diff --git a/pipelines/docker-build-dance/patch.yaml b/pipelines/docker-build-dance/patch.yaml
@@ -84,7 +84,7 @@
     taskRef:
       name: acs-image-check
     workspaces:
-      - name: workspace
+      - name: source
         workspace: workspace
 - op: add
   path: /spec/tasks/-

diff --git a/pipelines/docker-build/patch.yaml b/pipelines/docker-build/patch.yaml
@@ -30,6 +30,10 @@
     value: "$(params.image-expires-after)"
   - name: COMMIT_SHA
     value: "$(tasks.clone-repository.results.commit)"
+  - name: SOURCE_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+  - name: CACHI2_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
 - op: add
   path: /spec/results/-
   value:

diff --git a/pipelines/fbc-builder/patch.yaml b/pipelines/fbc-builder/patch.yaml
@@ -28,6 +28,10 @@
     value: "$(params.image-expires-after)"
   - name: COMMIT_SHA
     value: "$(tasks.clone-repository.results.commit)"
+  - name: SOURCE_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+  - name: CACHI2_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
 - op: add
   path: /spec/tasks/-
   value:
@@ -70,7 +74,7 @@
     - name: BASE_IMAGE
       value: $(tasks.inspect-image.results.BASE_IMAGE)
     workspaces:
-      - name: workspace
+      - name: source
         workspace: workspace
 - op: add
   path: /spec/tasks/-
@@ -86,7 +90,7 @@
       name: fbc-related-image-check
       version: "0.1"
     workspaces:
-      - name: workspace
+      - name: source
         workspace: workspace
 # - op: remove
 #   # build-source-image as source images are not needed for FBC components

diff --git a/pipelines/java-builder/patch.yaml b/pipelines/java-builder/patch.yaml
@@ -24,6 +24,10 @@
     value: "$(params.image-expires-after)"
   - name: COMMIT_SHA
     value: "$(tasks.clone-repository.results.commit)"
+  - name: SOURCE_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+  - name: CACHI2_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
 - op: add
   path: /spec/results/-
   value:

diff --git a/pipelines/nodejs-builder/patch.yaml b/pipelines/nodejs-builder/patch.yaml
@@ -24,3 +24,7 @@
     value: "$(params.image-expires-after)"
   - name: COMMIT_SHA
     value: "$(tasks.clone-repository.results.commit)"
+  - name: SOURCE_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+  - name: CACHI2_ARTIFACT
+    value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
diff --git a/pipelines/prototypes/prototype-build-compliance.yaml b/pipelines/prototypes/prototype-build-compliance.yaml
@@ -49,7 +49,7 @@ spec:
       taskRef:
         name: git-clone
       workspaces:
-        - name: output
+        - name: source
           workspace: workspace
         - name: basic-auth
           workspace: git-auth

diff --git a/pipelines/tekton-bundle-builder/kustomization.yaml b/pipelines/tekton-bundle-builder/kustomization.yaml
@@ -22,6 +22,10 @@ patches:
         value: $(params.output-image)
       - name: CONTEXT
         value: $(params.path-context)
+      - name: SOURCE_ARTIFACT
+        value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+      - name: CACHI2_ARTIFACT
+        value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
     # Remove tasks that assume a binary image
     - op: remove
       path: /spec/tasks/9  # sbom-json-check

diff --git a/pipelines/template-build/template-build.yaml b/pipelines/template-build/template-build.yaml
@@ -81,18 +81,18 @@ spec:
         name: git-clone
         version: "0.1"
       workspaces:
-        - name: output
+        - name: source
           workspace: workspace
         - name: basic-auth
           workspace: git-auth
     - name: prefetch-dependencies
-      when:
-      - input: $(params.hermetic)
-        operator: in
-        values: ["true"]
       params:
         - name: input
           value: $(params.prefetch-input)
+        - name: hermetic
+          value: $(params.hermetic)
+        - name: SOURCE_ARTIFACT
+          value: $(tasks.clone-repository.results.SOURCE_ARTIFACT)
       runAfter:
         - clone-repository
       taskRef:
@@ -106,12 +106,15 @@ spec:
       - input: $(tasks.init.results.build)
         operator: in
         values: ["true"]
-      runAfter:
-        - prefetch-dependencies
+      params:
+        - name: SOURCE_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+        - name: CACHI2_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
       taskRef:
         name: $REPLACE_ME
       workspaces:
-        - name: source
+        - name: artifacts
           workspace: workspace
     - name: build-source-image
       when:
@@ -131,8 +134,12 @@ spec:
           value: "$(params.output-image)"
         - name: BASE_IMAGES
           value: "$(tasks.build-container.results.BASE_IMAGES_DIGESTS)"
+        - name: SOURCE_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+        - name: CACHI2_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
       workspaces:
-        - name: workspace
+        - name: artifacts
           workspace: workspace
     - name: deprecated-base-image-check
       when:
@@ -167,13 +174,18 @@ spec:
       - input: $(params.skip-checks)
         operator: in
         values: ["false"]
+      params:
+        - name: SOURCE_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
+        - name: CACHI2_ARTIFACT
+          value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
       runAfter:
         - clone-repository
       taskRef:
         name: sast-snyk-check
         version: "0.1"
       workspaces:
-      - name: workspace
+      - name: artifacts
         workspace: workspace
     - name: clamav-scan
       when:

diff --git a/task/buildah-10gb/0.1/patch.yaml b/task/buildah-10gb/0.1/patch.yaml
@@ -2,8 +2,8 @@
   path: /metadata/name
   value: buildah-10gb
 - op: replace
-  path: /spec/steps/0/computeResources/limits/memory
+  path: /spec/steps/1/computeResources/limits/memory
   value: 10Gi
 - op: replace
-  path: /spec/steps/0/computeResources/requests/memory
+  path: /spec/steps/1/computeResources/requests/memory
   value: 8Gi
diff --git a/task/buildah-6gb/0.1/patch.yaml b/task/buildah-6gb/0.1/patch.yaml
@@ -2,8 +2,8 @@
   path: /metadata/name
   value: buildah-6gb
 - op: replace
-  path: /spec/steps/0/computeResources/limits/memory
+  path: /spec/steps/1/computeResources/limits/memory
   value: 6Gi
 - op: replace
-  path: /spec/steps/0/computeResources/requests/memory
+  path: /spec/steps/1/computeResources/requests/memory
   value: 4Gi
diff --git a/task/buildah-8gb/0.1/patch.yaml b/task/buildah-8gb/0.1/patch.yaml
@@ -2,8 +2,8 @@
   path: /metadata/name
   value: buildah-8gb
 - op: replace
-  path: /spec/steps/0/computeResources/limits/memory
+  path: /spec/steps/1/computeResources/limits/memory
   value: 8Gi
 - op: replace
-  path: /spec/steps/0/computeResources/requests/memory
+  path: /spec/steps/1/computeResources/requests/memory
   value: 6Gi
diff --git a/task/buildah-remote/0.1/buildah-remote.yaml b/task/buildah-remote/0.1/buildah-remote.yaml
@@ -69,6 +69,14 @@ spec:
     description: Target path on the container in which yum repository files should
       be made available
     name: YUM_REPOS_D_TARGET
+  - default: ""
+    description: The source trusted artifact URI
+    name: SOURCE_ARTIFACT
+    type: string
+  - default: ""
+    description: The prefetched dependencies trusted artifact URI
+    name: CACHI2_ARTIFACT
+    type: string
   - description: The platform to build on
     name: PLATFORM
     type: string
@@ -115,6 +123,15 @@ spec:
     - name: BUILDER_IMAGE
       value: $(params.BUILDER_IMAGE)
   steps:
+  - args:
+    - use
+    - --store
+    - $(workspaces.artifacts.path)
+    - $(params.SOURCE_ARTIFACT)=$(workspaces.source.path)/source
+    - $(params.CACHI2_ARTIFACT)=$(workspaces.source.path)/cachi2
+    computeResources: {}
+    image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:dbcf0102d0e9f21c2bcb06dcaa9af680cb9151f1984f5cec1fb397e1356ae771
+    name: use-trusted-artifact
   - computeResources:
       limits:
         memory: 4Gi
@@ -155,6 +172,7 @@ spec:
       fi
 
       rsync -ra $(workspaces.source.path)/ "$SSH_HOST:$BUILD_DIR/workspaces/source/"
+      rsync -ra $(workspaces.artifacts.path)/ "$SSH_HOST:$BUILD_DIR/workspaces/artifacts/"
       rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
       rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/tekton-results/"
       cat >scripts/script-build.sh <<'REMOTESSHEOF'
@@ -276,11 +294,13 @@ spec:
        -e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
        -e COMMIT_SHA="$COMMIT_SHA" \
        -v "$BUILD_DIR/workspaces/source:$(workspaces.source.path):Z" \
+       -v "$BUILD_DIR/workspaces/artifacts:$(workspaces.artifacts.path):Z" \
        -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
        -v "$BUILD_DIR/tekton-results/:/tekton/results:Z" \
        -v $BUILD_DIR/scripts:/script:Z \
       --user=0  --rm  "$BUILDER_IMAGE" /script/script-build.sh
       rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/source/" "$(workspaces.source.path)/"
+      rsync -ra "$SSH_HOST:$BUILD_DIR/workspaces/artifacts/" "$(workspaces.artifacts.path)/"
       rsync -ra "$SSH_HOST:$BUILD_DIR/tekton-results/" "/tekton/results/"
       buildah pull oci:rhtap-final-image
       buildah images
@@ -452,3 +472,5 @@ spec:
   workspaces:
   - description: Workspace containing the source code to build.
     name: source
+  - description: The trusted artifact store
+    name: artifacts
diff --git a/task/buildah-rhtap/0.1/buildah-rhtap.yaml b/task/buildah-rhtap/0.1/buildah-rhtap.yaml
@@ -28,6 +28,14 @@ spec:
     description: Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry)
     name: TLSVERIFY
     type: string
+  - name: SOURCE_ARTIFACT
+    type: string
+    description: The source trusted artifact URI
+    default: ""
+  - name: CACHI2_ARTIFACT
+    type: string
+    description: The prefetched dependencies trusted artifact URI
+    default: ""
   results:
   - description: Digest of the image just built
     name: IMAGE_DIGEST
@@ -48,6 +56,14 @@ spec:
     - name: TLSVERIFY
       value: $(params.TLSVERIFY)
   steps:
+  - name: use-trusted-artifact
+    image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:dbcf0102d0e9f21c2bcb06dcaa9af680cb9151f1984f5cec1fb397e1356ae771
+    args:
+      - use
+      - --store
+      - $(workspaces.artifacts.path)
+      - $(params.SOURCE_ARTIFACT)=$(workspaces.source.path)/source
+      - $(params.CACHI2_ARTIFACT)=$(workspaces.source.path)/cachi2
   - name: build
     image: registry.access.redhat.com/ubi9/buildah@sha256:04fde77ea72c25b56efb3f71db809c5d7b09938130df2da9175a3c888b94043d
     script: |
@@ -172,3 +188,7 @@ spec:
   workspaces:
   - name: source
     description: Workspace containing the source code to build.
+    emptyDir: {}
+    optional: true
+  - name: artifacts
+    description: The trusted artifact store
diff --git a/task/buildah/0.1/buildah.yaml b/task/buildah/0.1/buildah.yaml
@@ -66,6 +66,14 @@ spec:
   - name: YUM_REPOS_D_TARGET
     description: Target path on the container in which yum repository files should be made available
     default: /etc/yum.repos.d
+  - name: SOURCE_ARTIFACT
+    type: string
+    description: The source trusted artifact URI
+    default: ""
+  - name: CACHI2_ARTIFACT
+    type: string
+    description: The prefetched dependencies trusted artifact URI
+    default: ""
   results:
   - description: Digest of the image just built
     name: IMAGE_DIGEST
@@ -105,6 +113,14 @@ spec:
     - name: YUM_REPOS_D_TARGET
       value: $(params.YUM_REPOS_D_TARGET)
   steps:
+  - name: use-trusted-artifact
+    image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:dbcf0102d0e9f21c2bcb06dcaa9af680cb9151f1984f5cec1fb397e1356ae771
+    args:
+      - use
+      - --store
+      - $(workspaces.artifacts.path)
+      - $(params.SOURCE_ARTIFACT)=$(workspaces.source.path)/source
+      - $(params.CACHI2_ARTIFACT)=$(workspaces.source.path)/cachi2
   - image: $(params.BUILDER_IMAGE)
     # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
     # the cluster will set imagePullPolicy to IfNotPresent; our default param above specifies a digest
@@ -392,3 +408,7 @@ spec:
   workspaces:
   - name: source
     description: Workspace containing the source code to build.
+    emptyDir: {}
+    optional: true
+  - name: artifacts
+    description: The trusted artifact store