From 935188ba5407da8bc641601c75da9808a291c1e7 Mon Sep 17 00:00:00 2001 From: mkosiarc Date: Tue, 13 Aug 2024 12:46:51 +0200 Subject: [PATCH] Merge sbom manipulation steps The steps merge-syft-sboms, merge-cachi2-sbom, create-purl-sbom, create-base-images-sbom were merged into single "prepare-sboms" step. This is done because the tekton results size for a task depends on the number of steps. By reducing the number of the steps we increase the limit. The scripts from the individual steps were moved to the build-tasks-dockerfiles repo that also contains the Dockerfile for the sbom-utility-scripts image. STONEBLD-2608 Signed-off-by: mkosiarc --- renovate.json | 3 +- task/buildah-oci-ta/0.2/buildah-oci-ta.yaml | 90 ++--------------- .../0.2/buildah-remote-oci-ta.yaml | 97 ++----------------- task/buildah-remote/0.2/buildah-remote.yaml | 97 ++----------------- task/buildah/0.2/buildah.yaml | 93 ++---------------- task/rpm-ostree/0.2/rpm-ostree.yaml | 4 +- 6 files changed, 42 insertions(+), 342 deletions(-) diff --git a/renovate.json b/renovate.json index a1ed52ae87..022bbff2f9 100644 --- a/renovate.json +++ b/renovate.json @@ -27,7 +27,8 @@ "quay.io/redhat-appstudio/hacbs-jvm-build-request-processor", "quay.io/redhat-appstudio/build-definitions-source-image-build-utils", "quay.io/redhat-appstudio/cosign", - "quay.io/redhat-appstudio/cachi2" + "quay.io/redhat-appstudio/cachi2", + "quay.io/redhat-appstudio/sbom-utility-scripts-image" ], "groupName": "build", "reviewers": ["mmorhun", "tkdchen", "rcerven", "mkosiarc", "brunoapimentel", "chmeliik"] diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml index 8842207481..f6db08c933 100644 --- a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml +++ b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml @@ -475,96 +475,24 @@ spec: memory: 256Mi securityContext: runAsUser: 0 - - name: merge-syft-sboms - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d + - name: prepare-sboms + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06 workingDir: /var/workdir script: | - #!/bin/python3 - import json + echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" + python3 /scripts/merge_syft_sboms.py - # load SBOMs - with open("./sbom-image.json") as f: - image_sbom = json.load(f) - - with open("./sbom-source.json") as f: - source_sbom = json.load(f) - - # fetch unique components from available SBOMs - def get_identifier(component): - return component["name"] + '@' + component.get("version", "") - - image_sbom_components = image_sbom.setdefault("components", []) - existing_components = [get_identifier(component) for component in image_sbom_components] - - source_sbom_components = source_sbom.get("components", []) - for component in source_sbom_components: - if get_identifier(component) not in existing_components: - image_sbom_components.append(component) - existing_components.append(get_identifier(component)) - - image_sbom_components.sort(key=lambda c: get_identifier(c)) - - # write the CycloneDX unified SBOM - with open("./sbom-cyclonedx.json", "w") as f: - json.dump(image_sbom, f, indent=4) - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - securityContext: - runAsUser: 0 - - name: merge-cachi2-sbom - image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad - workingDir: /var/workdir - script: | if [ -f "sbom-cachi2.json" ]; then echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json" - merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json + python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json mv sbom-temp.json sbom-cyclonedx.json - else - echo "Skipping step since no Cachi2 SBOM was produced" fi - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - securityContext: - runAsUser: 0 - - name: create-purl-sbom - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - workingDir: /var/workdir - script: | - #!/bin/python3 - import json - with open("./sbom-cyclonedx.json") as f: - cyclonedx_sbom = json.load(f) + echo "Creating sbom-purl.json" + python3 /scripts/create_purl_sbom.py - purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component] - purl_content = {"image_contents": {"dependencies": purls}} - - with open("sbom-purl.json", "w") as output_file: - json.dump(purl_content, output_file, indent=4) - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - securityContext: - runAsUser: 0 - - name: create-base-images-sbom - image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840 - workingDir: /var/workdir - script: | - python3 /app/base_images_sbom_script.py \ + echo "Adding base images data to sbom-cyclonedx.json" + python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ --base-images-digests=/shared/base_images_digests diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml index 5bdad24676..150d00c883 100644 --- a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml +++ b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml @@ -564,105 +564,28 @@ spec: requests: cpu: 100m memory: 256Mi - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - name: merge-syft-sboms - script: | - #!/bin/python3 - import json - - # load SBOMs - with open("./sbom-image.json") as f: - image_sbom = json.load(f) - - with open("./sbom-source.json") as f: - source_sbom = json.load(f) - - # fetch unique components from available SBOMs - def get_identifier(component): - return component["name"] + '@' + component.get("version", "") - - image_sbom_components = image_sbom.setdefault("components", []) - existing_components = [get_identifier(component) for component in image_sbom_components] - - source_sbom_components = source_sbom.get("components", []) - for component in source_sbom_components: - if get_identifier(component) not in existing_components: - image_sbom_components.append(component) - existing_components.append(get_identifier(component)) - - image_sbom_components.sort(key=lambda c: get_identifier(c)) - - # write the CycloneDX unified SBOM - with open("./sbom-cyclonedx.json", "w") as f: - json.dump(image_sbom, f, indent=4) - securityContext: - runAsUser: 0 - workingDir: /var/workdir - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad - name: merge-cachi2-sbom + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06 + name: prepare-sboms script: | #!/bin/bash set -e if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then export IMAGE="${IMAGE}-${PLATFORM##*/}" fi + echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" + python3 /scripts/merge_syft_sboms.py + if [ -f "sbom-cachi2.json" ]; then echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json" - merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json + python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json >sbom-temp.json mv sbom-temp.json sbom-cyclonedx.json - else - echo "Skipping step since no Cachi2 SBOM was produced" fi - securityContext: - runAsUser: 0 - workingDir: /var/workdir - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - name: create-purl-sbom - script: | - #!/bin/python3 - import json - - with open("./sbom-cyclonedx.json") as f: - cyclonedx_sbom = json.load(f) - purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component] - purl_content = {"image_contents": {"dependencies": purls}} + echo "Creating sbom-purl.json" + python3 /scripts/create_purl_sbom.py - with open("sbom-purl.json", "w") as output_file: - json.dump(purl_content, output_file, indent=4) - securityContext: - runAsUser: 0 - workingDir: /var/workdir - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840 - name: create-base-images-sbom - script: | - #!/bin/bash - set -e - if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then - export IMAGE="${IMAGE}-${PLATFORM##*/}" - fi - python3 /app/base_images_sbom_script.py \ + echo "Adding base images data to sbom-cyclonedx.json" + python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ --base-images-digests=/shared/base_images_digests diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml index aafc9baedb..5b89063723 100644 --- a/task/buildah-remote/0.2/buildah-remote.yaml +++ b/task/buildah-remote/0.2/buildah-remote.yaml @@ -546,105 +546,28 @@ spec: requests: cpu: 100m memory: 256Mi - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - name: merge-syft-sboms - script: | - #!/bin/python3 - import json - - # load SBOMs - with open("./sbom-image.json") as f: - image_sbom = json.load(f) - - with open("./sbom-source.json") as f: - source_sbom = json.load(f) - - # fetch unique components from available SBOMs - def get_identifier(component): - return component["name"] + '@' + component.get("version", "") - - image_sbom_components = image_sbom.setdefault("components", []) - existing_components = [get_identifier(component) for component in image_sbom_components] - - source_sbom_components = source_sbom.get("components", []) - for component in source_sbom_components: - if get_identifier(component) not in existing_components: - image_sbom_components.append(component) - existing_components.append(get_identifier(component)) - - image_sbom_components.sort(key=lambda c: get_identifier(c)) - - # write the CycloneDX unified SBOM - with open("./sbom-cyclonedx.json", "w") as f: - json.dump(image_sbom, f, indent=4) - securityContext: - runAsUser: 0 - workingDir: $(workspaces.source.path) - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad - name: merge-cachi2-sbom + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06 + name: prepare-sboms script: | #!/bin/bash set -e if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then export IMAGE="${IMAGE}-${PLATFORM##*/}" fi + echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" + python3 /scripts/merge_syft_sboms.py + if [ -f "sbom-cachi2.json" ]; then echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json" - merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json + python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json mv sbom-temp.json sbom-cyclonedx.json - else - echo "Skipping step since no Cachi2 SBOM was produced" fi - securityContext: - runAsUser: 0 - workingDir: $(workspaces.source.path) - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - name: create-purl-sbom - script: | - #!/bin/python3 - import json - - with open("./sbom-cyclonedx.json") as f: - cyclonedx_sbom = json.load(f) - purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component] - purl_content = {"image_contents": {"dependencies": purls}} + echo "Creating sbom-purl.json" + python3 /scripts/create_purl_sbom.py - with open("sbom-purl.json", "w") as output_file: - json.dump(purl_content, output_file, indent=4) - securityContext: - runAsUser: 0 - workingDir: $(workspaces.source.path) - - computeResources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840 - name: create-base-images-sbom - script: | - #!/bin/bash - set -e - if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then - export IMAGE="${IMAGE}-${PLATFORM##*/}" - fi - python3 /app/base_images_sbom_script.py \ + echo "Adding base images data to sbom-cyclonedx.json" + python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ --base-images-digests=/shared/base_images_digests diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml index 392618410b..4e25450487 100644 --- a/task/buildah/0.2/buildah.yaml +++ b/task/buildah/0.2/buildah.yaml @@ -423,8 +423,8 @@ spec: securityContext: runAsUser: 0 - - name: merge-syft-sboms - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d + - name: prepare-sboms + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:064eeec1cfb75abd21b533bf0d9870a7a3fb138f30192596dce28061798cbf06 computeResources: limits: memory: 512Mi @@ -433,95 +433,20 @@ spec: memory: 256Mi cpu: 100m script: | - #!/bin/python3 - import json + echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" + python3 /scripts/merge_syft_sboms.py - # load SBOMs - with open("./sbom-image.json") as f: - image_sbom = json.load(f) - - with open("./sbom-source.json") as f: - source_sbom = json.load(f) - - # fetch unique components from available SBOMs - def get_identifier(component): - return component["name"] + '@' + component.get("version", "") - - image_sbom_components = image_sbom.setdefault("components", []) - existing_components = [get_identifier(component) for component in image_sbom_components] - - source_sbom_components = source_sbom.get("components", []) - for component in source_sbom_components: - if get_identifier(component) not in existing_components: - image_sbom_components.append(component) - existing_components.append(get_identifier(component)) - - image_sbom_components.sort(key=lambda c: get_identifier(c)) - - # write the CycloneDX unified SBOM - with open("./sbom-cyclonedx.json", "w") as f: - json.dump(image_sbom, f, indent=4) - workingDir: $(workspaces.source.path) - securityContext: - runAsUser: 0 - - - name: merge-cachi2-sbom - image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad - computeResources: - limits: - memory: 512Mi - cpu: 200m - requests: - memory: 256Mi - cpu: 100m - script: | if [ -f "sbom-cachi2.json" ]; then echo "Merging contents of sbom-cachi2.json into sbom-cyclonedx.json" - merge_syft_sbom sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json + python3 /scripts/merge_cachi2_sboms.py sbom-cachi2.json sbom-cyclonedx.json > sbom-temp.json mv sbom-temp.json sbom-cyclonedx.json - else - echo "Skipping step since no Cachi2 SBOM was produced" fi - workingDir: $(workspaces.source.path) - securityContext: - runAsUser: 0 - - name: create-purl-sbom - image: registry.access.redhat.com/ubi9/python-39:1-192.1722518946@sha256:0176b477075984d5a502253f951d2502f0763c551275f9585ac515b9f241d73d - computeResources: - limits: - memory: 512Mi - cpu: 200m - requests: - memory: 256Mi - cpu: 100m - script: | - #!/bin/python3 - import json - - with open("./sbom-cyclonedx.json") as f: - cyclonedx_sbom = json.load(f) - - purls = [{"purl": component["purl"]} for component in cyclonedx_sbom.get("components", []) if "purl" in component] - purl_content = {"image_contents": {"dependencies": purls}} - - with open("sbom-purl.json", "w") as output_file: - json.dump(purl_content, output_file, indent=4) - workingDir: $(workspaces.source.path) - securityContext: - runAsUser: 0 + echo "Creating sbom-purl.json" + python3 /scripts/create_purl_sbom.py - - name: create-base-images-sbom - image: quay.io/redhat-appstudio/base-images-sbom-script@sha256:667669e3def018f9dbb8eaf8868887a40bc07842221e9a98f6787edcff021840 - computeResources: - limits: - memory: 512Mi - cpu: 200m - requests: - memory: 256Mi - cpu: 100m - script: | - python3 /app/base_images_sbom_script.py \ + echo "Adding base images data to sbom-cyclonedx.json" + python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ --base-images-digests=/shared/base_images_digests diff --git a/task/rpm-ostree/0.2/rpm-ostree.yaml b/task/rpm-ostree/0.2/rpm-ostree.yaml index ca472151f3..641351f3cb 100644 --- a/task/rpm-ostree/0.2/rpm-ostree.yaml +++ b/task/rpm-ostree/0.2/rpm-ostree.yaml @@ -212,12 +212,12 @@ spec: - mountPath: /var/lib/containers name: varlibcontainers - name: merge-cachi2-sbom - image: quay.io/redhat-appstudio/cachi2:0.9.1@sha256:df67f9e063b544a8c49a271359377fed560562615e0278f6d0b9a3485f3f8fad + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:53a3041dff341b7fd1765b9cc2c324625d19e804b2eaff10a6e6d9dcdbde3a91 script: | cachi2_sbom=./cachi2/output/bom.json if [ -f "$cachi2_sbom" ]; then echo "Merging contents of $cachi2_sbom into sbom-cyclonedx.json" - merge_syft_sbom "$cachi2_sbom" sbom-cyclonedx.json > sbom-temp.json + python3 /scripts/merge_cachi2_sboms.py "$cachi2_sbom" sbom-cyclonedx.json > sbom-temp.json mv sbom-temp.json sbom-cyclonedx.json else echo "Skipping step since no Cachi2 SBOM was produced"