Improve generation of base images SBOMs

most functional changes are in the related PR that updates the base_images_sbom_script.py konflux-ci/build-tasks-dockerfiles#191 Here, we are just updating on how we generate the inputs for this script. We are now passing the whole parsed Dockerfile in json format to that script, which allows us to better parse/detect base images. Also, the format of the /shared/base_images_digests file was changed. Previously we could rely on the order of the image references with the digests in the file. Now we need to provide a mapping from an image reference as it was used in the Dockerfile to the full image reference with digests that was used during build and generated by buildah. The mapping is done as: <image-reference-used-in-dockerfile> <full-image-reference-with-digest> Also, the sbom utility image has to be updated together in the same PR/commit, otherwise it would break konflux temporarily KFLUXBUGS-1718 Signed-off-by: mkosiarc <[email protected]>
konflux-ci · Dec 9, 2024 · eee3c4f · eee3c4f
1 parent 3def877
commit eee3c4f
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 33 deletions.
diff --git a/task/build-image-index/0.1/build-image-index.yaml b/task/build-image-index/0.1/build-image-index.yaml
@@ -163,7 +163,7 @@ spec:
         add:
           - SETFCAP
 
-  - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4
+  - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579
     name: create-sbom
     computeResources:
       limits:

diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
@@ -348,9 +348,9 @@ spec:
           BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
         done
 
+        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json
         BASE_IMAGES=$(
-          dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-            jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
         )
 
         BUILDAH_ARGS=()
@@ -530,11 +530,13 @@ spec:
 
         touch /shared/base_images_digests
         for image in $BASE_IMAGES; do
-          buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests
+          base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+          # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+          # if buildah did not use that particular image during build because it was skipped
+          if [ -n "$base_image_digest" ]; then
+            echo "$image $base_image_digest" >>/shared/base_images_digests
+          fi
         done
-
-        # Needed to generate base images SBOM
-        echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile
       computeResources:
         limits:
           cpu: "4"
@@ -651,7 +653,7 @@ spec:
       securityContext:
         runAsUser: 0
     - name: prepare-sboms
-      image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4
+      image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579
       workingDir: /var/workdir
       script: |
         echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json"
@@ -666,7 +668,7 @@ spec:
         echo "Adding base images data to sbom-cyclonedx.json"
         python3 /scripts/base_images_sbom_script.py \
           --sbom=sbom-cyclonedx.json \
-          --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+          --parsed-dockerfile=/shared/parsed_dockerfile.json \
           --base-images-digests=/shared/base_images_digests
 
         echo "Adding image reference to sbom"

diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml
@@ -382,9 +382,9 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+        jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -564,12 +564,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >>/shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile
-
       buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE"
       REMOTESSHEOF
       chmod +x scripts/script-build.sh
@@ -769,7 +771,7 @@ spec:
       requests:
         cpu: 100m
         memory: 256Mi
-    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4
+    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579
     name: prepare-sboms
     script: |
       #!/bin/bash
@@ -790,7 +792,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"

diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml
@@ -358,9 +358,10 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -542,12 +543,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >> /shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile
-
       buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE"
       REMOTESSHEOF
       chmod +x scripts/script-build.sh
@@ -749,7 +752,7 @@ spec:
       requests:
         cpu: 100m
         memory: 256Mi
-    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4
+    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579
     name: prepare-sboms
     script: |
       #!/bin/bash
@@ -770,7 +773,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"

diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml
@@ -279,9 +279,10 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -463,12 +464,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >> /shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile
-
     securityContext:
       capabilities:
         add:
@@ -599,7 +602,7 @@ spec:
       runAsUser: 0
 
   - name: prepare-sboms
-    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4
+    image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579
     computeResources:
       limits:
         memory: 512Mi
@@ -620,7 +623,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"