Improve generation of base images SBOMs

most functional changes are in the related PR that updates the base_images_sbom_script.py konflux-ci/build-tasks-dockerfiles#191 Here, we are just updating on how we generate the inputs for this script. We are now passing the whole parsed Dockerfile in json format to that script, which allows us to better parse/detect base images. Also, the format of the /shared/base_images_digests file was changed. Previously we could rely on the order of the image references with the digests in the file. Now we need to provide a mapping from an image reference as it was used in the Dockerfile to the full image reference with digests that was used during build and generated by buildah. The mapping is done as: <image-reference-used-in-dockerfile> <full-image-reference-with-digest> KFLUXBUGS-1718 Signed-off-by: mkosiarc <[email protected]>
konflux-ci · Dec 4, 2024 · af81c03 · af81c03
1 parent 4af967b
commit af81c03
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 28 deletions.
diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
@@ -348,9 +348,9 @@ spec:
           BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
         done
 
+        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json
         BASE_IMAGES=$(
-          dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-            jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
         )
 
         BUILDAH_ARGS=()
@@ -530,11 +530,13 @@ spec:
 
         touch /shared/base_images_digests
         for image in $BASE_IMAGES; do
-          buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests
+          base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+          # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+          # if buildah did not use that particular image during build because it was skipped
+          if [ -n "$base_image_digest" ]; then
+            echo "$image $base_image_digest" >>/shared/base_images_digests
+          fi
         done
-
-        # Needed to generate base images SBOM
-        echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile
       computeResources:
         limits:
           cpu: "4"
@@ -674,7 +676,7 @@ spec:
         echo "Adding base images data to sbom-cyclonedx.json"
         python3 /scripts/base_images_sbom_script.py \
           --sbom=sbom-cyclonedx.json \
-          --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+          --parsed-dockerfile=/shared/parsed_dockerfile.json \
           --base-images-digests=/shared/base_images_digests
 
         echo "Adding image reference to sbom"

diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml
@@ -382,9 +382,9 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+        jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -564,12 +564,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >>/shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile
-
       buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE"
       REMOTESSHEOF
       chmod +x scripts/script-build.sh
@@ -798,7 +800,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"

diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml
@@ -358,9 +358,10 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -542,12 +543,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >> /shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile
-
       buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE"
       REMOTESSHEOF
       chmod +x scripts/script-build.sh
@@ -778,7 +781,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"

diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml
@@ -279,9 +279,10 @@ spec:
         BUILD_ARG_FLAGS+=("--build-arg=$build_arg")
       done
 
+
+      dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json
       BASE_IMAGES=$(
-        dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" |
-          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)'
+          jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json
       )
 
       BUILDAH_ARGS=()
@@ -463,12 +464,14 @@ spec:
 
       touch /shared/base_images_digests
       for image in $BASE_IMAGES; do
-        buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests
+        base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image")
+        # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens
+        # if buildah did not use that particular image during build because it was skipped
+        if [ -n "$base_image_digest" ]; then
+          echo "$image $base_image_digest" >> /shared/base_images_digests
+        fi
       done
 
-      # Needed to generate base images SBOM
-      echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile
-
     securityContext:
       capabilities:
         add:
@@ -628,7 +631,7 @@ spec:
       echo "Adding base images data to sbom-cyclonedx.json"
       python3 /scripts/base_images_sbom_script.py \
         --sbom=sbom-cyclonedx.json \
-        --base-images-from-dockerfile=/shared/base_images_from_dockerfile \
+        --parsed-dockerfile=/shared/parsed_dockerfile.json \
         --base-images-digests=/shared/base_images_digests
 
       echo "Adding image reference to sbom"