From 8fc64933d7d536f752320a0ecd47fb356bdd2b28 Mon Sep 17 00:00:00 2001 From: "young.yang03" Date: Tue, 2 Jul 2024 10:29:53 +0800 Subject: [PATCH] Mbt upgrade SBOM file content (#1103) * modified: .circleci/config.yml modified: Dockerfile_mbtci_template modified: Makefile modified: internal/artifacts/sbom.go modified: internal/commands/commands.go * modified: internal/artifacts/sbom.go modified: internal/commands/commands.go * modified: internal/artifacts/sbom.go * modified: Dockerfile_mbtci_template modified: internal/artifacts/sbom.go * modified: internal/artifacts/project.go modified: internal/artifacts/project_test.go modified: internal/artifacts/sbom.go * modified: internal/artifacts/sbom.go * modified: Makefile --------- Co-authored-by: Young Yang --- .circleci/config.yml | 16 +- Dockerfile_mbtci_template | 12 +- Makefile | 25 +-- internal/artifacts/project.go | 2 +- internal/artifacts/project_test.go | 2 +- internal/artifacts/sbom.go | 241 ++++++++++++++++++++++++++++- internal/commands/commands.go | 29 ++-- 7 files changed, 279 insertions(+), 48 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fd4b447..f80bca16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,14 +136,14 @@ jobs: go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest cyclonedx-gomod version - run: - name: install cyclonedx-bom - command: | - CYCLONEDX_BOM_PACKAGE=cyclonedx-bom - CYCLONEDX_BOM_VERSION=0.0.9 - CYCLONEDX_BOM_BINARY=cyclonedx-bom - npm install ${CYCLONEDX_BOM_PACKAGE}@${CYCLONEDX_BOM_VERSION} --no-save - echo "${CYCLONEDX_BOM_BINARY} -h" - npx ${CYCLONEDX_BOM_BINARY} -h + name: install cyclonedx-npm + command: | + CYCLONEDX_NPM_PACKAGE=@cyclonedx/cyclonedx-npm + CYCLONEDX_NPM_VERSION=1.11.0 + CYCLONEDX_NPM_BINARY=cyclonedx-npm + npm install ${CYCLONEDX_NPM_PACKAGE}@${CYCLONEDX_NPM_VERSION} --no-save + echo "${CYCLONEDX_NPM_BINARY} -h" + npx ${CYCLONEDX_NPM_BINARY} -h - run: name: build mbt binary command: | diff --git a/Dockerfile_mbtci_template b/Dockerfile_mbtci_template index d6b07b61..7f9418a9 100644 --- a/Dockerfile_mbtci_template +++ b/Dockerfile_mbtci_template @@ -14,9 +14,9 @@ ARG CYCLONEDX_CLI_VERSION=0.24.2 ARG CYCLONEDX_CLI_BINARY=cyclonedx ARG CYCLONEDX_GOMOD_VERSION=1.4.0 ARG CYCLONEDX_GOMOD_BINARY=cyclonedx-gomod -ARG CYCLONEDX_BOM_PACKAGE=cyclonedx-bom -ARG CYCLONEDX_BOM_VERSION=0.0.9 -ARG CYCLONEDX_BOM_BINARY=cyclonedx-bom +ARG CYCLONEDX_NPM_PACKAGE=@cyclonedx/cyclonedx-npm +ARG CYCLONEDX_NPM_VERSION=1.11.0 +ARG CYCLONEDX_NPM_BINARY=cyclonedx-npm # Environment variables ENV PYTHON /usr/bin/python3 @@ -293,12 +293,6 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && echo "cyclonedx-gomod smoke tests!" \ && cyclonedx-gomod version -# Install cyclone-bom -RUN set -ex \ - && npm install --prefix /usr/local/ -g ${CYCLONEDX_BOM_PACKAGE}@${CYCLONEDX_BOM_VERSION} \ - && echo "cyclonedx-bom smoke tests!" \ - && npx ${CYCLONEDX_BOM_BINARY} -h - # Install curl and ca-certificates RUN set -ex \ && apt-get update \ diff --git a/Makefile b/Makefile index 99fca899..653f7a48 100644 --- a/Makefile +++ b/Makefile @@ -23,11 +23,10 @@ CYCLONEDX_CLI_VERSION = 0.24.2 CYCLONEDX_GOMOD_BINARY = cyclonedx-gomod CYCLONEDX_GOMOD_VERSION = latest -# cyclonedx-bom -CYCLONEDX_BOM_PACKAGE = cyclonedx-bom -CYCLONEDX_BOM_VERSION = 0.0.9 -CYCLONEDX_BOM_BINARY = cyclonedx-bom - +# cyclonedx_npm +CYCLONEDX_NPM_PACKAGE = @cyclonedx/cyclonedx-npm +CYCLONEDX_NPM_VERSION = 1.11.0 +CYCLONEDX_NPM_BINARY = cyclonedx-npm ifeq ($(OS),Windows_NT) CYCLONEDX_OS=win @@ -70,10 +69,10 @@ lint: # execute general tests tests: - go test -v -count=1 -timeout 30m ./... + go test -v -count=1 -timeout 60m ./... # check code coverage cover: - go test -v -coverprofile cover.out ./... -count=1 -timeout 30m + go test -v -coverprofile cover.out ./... -count=1 -timeout 60m go tool cover -html=cover.out -o cover.html open cover.html @@ -111,18 +110,20 @@ else cp $(CURDIR)/release/$(BINARY_NAME) $~/usr/local/bin/ endif -# use for local development - > install cyclonedx-gomod, cyclonedx-cli and cyclonedx-bom +# use for local development - > install cyclonedx-gomod, cyclonedx-cli and cyclonedx-npm install-cyclonedx: # install cyclonedx-gomod go install github.com/CycloneDX/cyclonedx-gomod/cmd/${CYCLONEDX_GOMOD_BINARY}@${CYCLONEDX_GOMOD_VERSION} echo "${CYCLONEDX_GOMOD_BINARY} version" ${CYCLONEDX_GOMOD_BINARY} version + # install cyclonedx-cli curl -fsSLO --compressed "https://github.com/CycloneDX/cyclonedx-cli/releases/download/v${CYCLONEDX_CLI_VERSION}/${CYCLONEDX_CLI_BINARY}-${CYCLONEDX_OS}-${CYCLONEDX_ARCH}${CYCLONEDX_BINARY_SUFFIX}" mv ${CYCLONEDX_CLI_BINARY}-${CYCLONEDX_OS}-${CYCLONEDX_ARCH}${CYCLONEDX_BINARY_SUFFIX} $(GOPATH)/bin/${CYCLONEDX_CLI_BINARY}${CYCLONEDX_BINARY_SUFFIX} echo "${CYCLONEDX_CLI_BINARY} version:" ${CYCLONEDX_CLI_BINARY} --version -# install cyclonedx-bom - npm install -g ${CYCLONEDX_BOM_PACKAGE}@${CYCLONEDX_BOM_VERSION} - echo "${CYCLONEDX_BOM_BINARY} -h" - npx ${CYCLONEDX_BOM_BINARY} -h \ No newline at end of file + +# install cyclonedx-npm + npm install -g ${CYCLONEDX_NPM_PACKAGE}@${CYCLONEDX_NPM_VERSION} + echo "${CYCLONEDX_NPM_BINARY} -h" + npx ${CYCLONEDX_NPM_BINARY} -h \ No newline at end of file diff --git a/internal/artifacts/project.go b/internal/artifacts/project.go index 7cdc9812..de553538 100644 --- a/internal/artifacts/project.go +++ b/internal/artifacts/project.go @@ -9,7 +9,7 @@ import ( "github.com/pkg/errors" - dir "github.com/SAP/cloud-mta-build-tool/internal/archive" + "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta-build-tool/internal/commands" "github.com/SAP/cloud-mta-build-tool/internal/exec" "github.com/SAP/cloud-mta-build-tool/internal/logs" diff --git a/internal/artifacts/project_test.go b/internal/artifacts/project_test.go index f8447eae..fb6e385a 100644 --- a/internal/artifacts/project_test.go +++ b/internal/artifacts/project_test.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - dir "github.com/SAP/cloud-mta-build-tool/internal/archive" + "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta-build-tool/internal/commands" "github.com/SAP/cloud-mta-build-tool/internal/exec" "github.com/SAP/cloud-mta/mta" diff --git a/internal/artifacts/sbom.go b/internal/artifacts/sbom.go index f70f1d4a..17884bb1 100644 --- a/internal/artifacts/sbom.go +++ b/internal/artifacts/sbom.go @@ -1,6 +1,9 @@ package artifacts import ( + "bytes" + "encoding/xml" + "io" "io/ioutil" "os" "path/filepath" @@ -29,6 +32,32 @@ const ( cyclonedx_cli = "cyclonedx" ) +type Bom struct { + XMLName xml.Name `xml:"bom"` + Metadata Metadata `xml:"metadata"` +} + +type Metadata struct { + XMLName xml.Name `xml:"metadata"` + Component Component `xml:"component"` +} + +type Component struct { + XMLName xml.Name `xml:"component"` + BomRef string `xml:"bom-ref,attr"` +} + +type Dependency struct { + XMLName xml.Name `xml:"dependency"` + Ref string `xml:"ref,attr"` + SubDepends []SubDep `xml:"dependency"` +} + +type SubDep struct { + XMLName xml.Name `xml:"dependency"` + Ref string `xml:"ref,attr"` +} + // ExecuteProjectSBomGenerate - Execute MTA project SBOM generation func ExecuteProjectSBomGenerate(source string, sbomFilePath string, wdGetter func() (string, error)) error { // (1) get loc object and mta object @@ -134,12 +163,28 @@ func generateSBomFile(loc *dir.Loc, mtaObj *mta.MTA, } // (3) merge sbom files under sbom tmp dir - sbomTmpName, err := mergeSBomFiles(loc, sbomTmpDir, sbomFileNames, sbomName, sbomType, sbomSuffix) + sbomTmpName, err := mergeSBomFiles(loc, mtaObj, sbomTmpDir, sbomFileNames, sbomName, sbomType, sbomSuffix) + if err != nil { + return err + } + + // (4) get module bom-ref info + moduleBomRefs, err := getModuleBomRefs(sbomTmpDir, sbomFileNames) + + for _, bomRef := range moduleBomRefs { + logs.Logger.Infof("moduleBomRef:%s", bomRef) + } if err != nil { return err } - // (4) generate sbom target dir, mv merged sbom file to target dir + // (4) instert xml attribute or xml node to bom->metadata + err = updateSBomMetadataNode(mtaObj, sbomTmpDir, sbomTmpName, moduleBomRefs) + if err != nil { + return err + } + + // (5) generate sbom target dir, mv merged sbom file to target dir err = moveSBomToTarget(sbomPath, sbomName, sbomTmpDir, sbomTmpName) if err != nil { return err @@ -148,6 +193,194 @@ func generateSBomFile(loc *dir.Loc, mtaObj *mta.MTA, return nil } +func getModuleBomRefs(sbomTmpDir string, sbomFileNames []string) ([]string, error) { + var moduleBomRefs []string + + for _, fileName := range sbomFileNames { + sbomfilepath := filepath.Join(sbomTmpDir, fileName) + xmlFile, err := os.Open(sbomfilepath) + if err != nil { + return nil, err + } + defer xmlFile.Close() + + byteValue, _ := ioutil.ReadAll(xmlFile) + + var bom Bom + xml.Unmarshal(byteValue, &bom) + + moduleBomRefs = append(moduleBomRefs, bom.Metadata.Component.BomRef) + } + + return moduleBomRefs, nil +} + +func removeXmlns(attrs []xml.Attr) []xml.Attr { + var result []xml.Attr + for _, attr := range attrs { + if attr.Name.Local != "xmlns" { + result = append(result, attr) + } + } + return result +} + +func addBomrefAttribute(attributes []xml.Attr, purl string) []xml.Attr { + purlAttr := xml.Attr{ + Name: xml.Name{Local: "bom-ref"}, + Value: purl, + } + + // Add bom-ref attribute to attributes list + attributes = append(attributes, purlAttr) + + return attributes +} + +func addXmlnsSchemaAttribute(attributes []xml.Attr, xmlnsSchema string) []xml.Attr { + purlAttr := xml.Attr{ + Name: xml.Name{Local: "xmlns"}, + Value: xmlnsSchema, + } + + // Add bom-ref attribute to attributes list + attributes = append(attributes, purlAttr) + + return attributes +} + +func updateSBomMetadataNode(mtaObj *mta.MTA, sbomTmpDir, sbomTmpName string, moduleBomRefs []string) error { + sbomfilepath := filepath.Join(sbomTmpDir, sbomTmpName) + file, err := os.Open(sbomfilepath) + if err != nil { + return err + } + defer file.Close() + + var purl = "pkg:mta/" + mtaObj.ID + "@" + mtaObj.Version + var xmlnsSchema = "http://cyclonedx.org/schema/bom/1.4" + + decoder := xml.NewDecoder(file) + decoder.Strict = false + + var out bytes.Buffer + encoder := xml.NewEncoder(&out) + + isInBom := false + isInBomMetadata := false + + for { + tok, err := decoder.RawToken() + // tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return err + } + + switch typedTok := tok.(type) { + case xml.ProcInst: + out.Write([]byte("")) + case xml.StartElement: + // if xml node contains 'xmlns' attribute, remove the attribute + typedTok.Attr = removeXmlns(typedTok.Attr) + // if current node is + if typedTok.Name.Local == "bom" { + // 1. set isInBom = true + isInBom = true + // 2. add xmlns schema attribute to bom xml node + typedTok.Attr = addXmlnsSchemaAttribute(typedTok.Attr, xmlnsSchema) + } + // if current node is bom->metadata + if typedTok.Name.Local == "metadata" && isInBom { + isInBomMetadata = true + // 1. write bom->metadata xml node + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + // 2. add bom->meatadata->timestamp xml node + encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "timestamp"}}) + encoder.EncodeToken(xml.CharData(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) + encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "timestamp"}}) + break + } + + // if current node is bom-metadata->component + if typedTok.Name.Local == "component" && isInBom && isInBomMetadata { + // 1. add bom-ref attribute to bom->metadata->component xml node + typedTok.Attr = addBomrefAttribute(typedTok.Attr, purl) + // 2. write bom->metadata->component xml node + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + // 3. add purl xml node to bom->metadata->component xml node + encoder.EncodeToken(xml.StartElement{Name: xml.Name{Local: "purl"}}) + encoder.EncodeToken(xml.CharData(purl)) + encoder.EncodeToken(xml.EndElement{Name: xml.Name{Local: "purl"}}) + break + } + + if typedTok.Name.Local == "dependencies" && isInBom { + // 1. write bom->dependencies xml node + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + // 2. Todo: insert new dependency xml element + dependency := Dependency{} + dependency.Ref = purl + for _, bomRefs := range moduleBomRefs { + subDependency := SubDep{} + subDependency.Ref = bomRefs + dependency.SubDepends = append(dependency.SubDepends, subDependency) + } + encoder.Encode(dependency) + break + } + + // common xml node + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + case xml.CharData: + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + case xml.EndElement: + if typedTok.Name.Local == "bom" { + isInBom = false + } + if typedTok.Name.Local == "metadata" && isInBom { + isInBomMetadata = false + } + + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + default: + err := encoder.EncodeToken(typedTok) + if err != nil { + return err + } + } + } + encoder.Flush() + content := out.Bytes() + content = bytes.Replace(content, []byte("\ufeff"), []byte(""), -1) + err = ioutil.WriteFile(sbomfilepath, content, 0644) + + if err != nil { + return err + } + return nil +} + func executeSBomGenerate(loc *dir.Loc, mtaObj *mta.MTA, source string, sbomFilePath string) error { // start generate sbom file log logs.Logger.Info(genSBomFileStartMsg) @@ -229,7 +462,7 @@ func listSBomFilesInTmpDir(sbomTmpDir, sbomSuffix string) ([]string, error) { } // mergeSBomFiles - merge sbom files of modules under sbom tmp dir -func mergeSBomFiles(loc *dir.Loc, sbomTmpDir string, sbomFileNames []string, sbomName, sbomType, sbomSuffix string) (string, error) { +func mergeSBomFiles(loc *dir.Loc, mtaObj *mta.MTA, sbomTmpDir string, sbomFileNames []string, sbomName, sbomType, sbomSuffix string) (string, error) { curtime := time.Now().Format("20230328150313") var sbomTmpName string @@ -242,7 +475,7 @@ func mergeSBomFiles(loc *dir.Loc, sbomTmpDir string, sbomFileNames []string, sbo } // get sbom file generate command - sbomMergeCmds, err := commands.GetSBomsMergeCommand(loc, cyclonedx_cli, sbomTmpDir, sbomFileNames, sbomTmpName, sbomType, sbomSuffix) + sbomMergeCmds, err := commands.GetSBomsMergeCommand(loc, cyclonedx_cli, mtaObj, sbomTmpDir, sbomFileNames, sbomTmpName, sbomType, sbomSuffix) if err != nil { return "", err } diff --git a/internal/commands/commands.go b/internal/commands/commands.go index 36691c7c..b0feb9d5 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -9,17 +9,20 @@ import ( "github.com/SAP/cloud-mta/mta" - "github.com/SAP/cloud-mta-build-tool/internal/archive" + dir "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta-build-tool/internal/logs" ) const ( - builderParam = "builder" - commandsParam = "commands" - customBuilder = "custom" - golangBuilder = "golang" - optionsSuffix = "-opts" - goModuleType = "go" + builderParam = "builder" + commandsParam = "commands" + customBuilder = "custom" + golangBuilder = "golang" + optionsSuffix = "-opts" + goModuleType = "go" + cyclonedx_npm = "@cyclonedx/cyclonedx-npm" + cyclonedx_npm_version = "1.11.0" + cyclonedx_npm_schema_version = "1.4" ) // CommandList - list of command to execute @@ -341,16 +344,16 @@ func GetModuleSBomGenCommands(loc *dir.Loc, module *mta.Module, case "npm", "npm-ci", "grunt", "evo": cmd = "npm install" cmds = append(cmds, cmd) - cmd = "npm install cyclonedx-bom@0.0.9 --no-save" + cmd = "npm install " + cyclonedx_npm + "@" + cyclonedx_npm_version + " --no-save" cmds = append(cmds, cmd) - cmd = "npx cyclonedx-bom -o " + sbomFileName + sbomFileSuffix + cmd = "npx cyclonedx-npm --output-format " + strings.ToUpper(sbomFileType) + " --spec-version " + cyclonedx_npm_schema_version + " --output-file " + sbomFileName + sbomFileSuffix cmds = append(cmds, cmd) case "golang": - cmd = "cyclonedx-gomod mod -licenses -output " + sbomFileName + sbomFileSuffix + cmd = "cyclonedx-gomod mod -output-version 1.4 -licenses -output " + sbomFileName + sbomFileSuffix cmds = append(cmds, cmd) case "maven", "fetcher", "maven_deprecated": cmd = "mvn org.cyclonedx:cyclonedx-maven-plugin:2.7.5:makeAggregateBom " + - "-DschemaVersion=1.2 -DincludeBomSerialNumber=true -DincludeCompileScope=true " + + "-DschemaVersion=1.4 -DincludeBomSerialNumber=true -DincludeCompileScope=true " + "-DincludeRuntimeScope=true -DincludeSystemScope=true -DincludeTestScope=false -DincludeLicenseText=false " + "-DoutputFormat=" + sbomFileType + " -DoutputName=" + sbomFileName + ".bom" cmds = append(cmds, cmd) @@ -368,7 +371,7 @@ func GetModuleSBomGenCommands(loc *dir.Loc, module *mta.Module, // GetSBomsMergeCommand - generate merge sbom file command under sbom tmp dir // if empty sbomFileNames, return empty commandList, nil error -func GetSBomsMergeCommand(loc *dir.Loc, cyclonedx_cli string, sbomTmpDir string, sbomFileNames []string, +func GetSBomsMergeCommand(loc *dir.Loc, cyclonedx_cli string, mtaObj *mta.MTA, sbomTmpDir string, sbomFileNames []string, sbomName, sbomType, sbomSuffix string) ([][]string, error) { var cmd string var cmds []string @@ -386,7 +389,7 @@ func GetSBomsMergeCommand(loc *dir.Loc, cyclonedx_cli string, sbomTmpDir string, // ./cyclonedx merge --input-files test_1.bom.xml test_2.bom.xml test_3.bom.xml --output-file merged.bom.xml cmd = cyclonedx_cli + " merge --input-files " + inputFiles + " --output-file " + sbomName + - " --input-format " + sbomType + " --output-format " + sbomType + " --input-format " + sbomType + " --output-format " + sbomType + " --name " + mtaObj.ID + " --version " + mtaObj.Version cmds = append(cmds, cmd) commandList, err := CmdConverter(sbomTmpDir, cmds)