diff --git a/.conanignore b/.conanignore index a259af4..d87e843 100644 --- a/.conanignore +++ b/.conanignore @@ -1,5 +1,6 @@ .idea/* .github/* +.venv/* venv/* tests/* .gitignore @@ -7,6 +8,7 @@ LICENSE *README.md* *.DS_Store* *.pytest_cache* +.*cache tmp/* tmp2/* .vscode/* diff --git a/.github/workflows/test_conan_extensions.yml b/.github/workflows/test_conan_extensions.yml index 4a587a3..a420f3a 100644 --- a/.github/workflows/test_conan_extensions.yml +++ b/.github/workflows/test_conan_extensions.yml @@ -18,7 +18,7 @@ jobs: - name: Install dependencies run: | pip install -U pip - pip install pytest + pip install pytest "cyclonedx-python-lib>=3.1.5,<5.0.0" - name: Install Conan latest run: | pip install conan diff --git a/.gitignore b/.gitignore index cebd89c..41374d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/tests/tmp* + .vscode/ # Byte-compiled / optimized / DLL files diff --git a/README.md b/README.md index 918bda4..3ee3076 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ Commands useful in Conan Center Index or its forks Commands useful for managing [JFrog Build Infos](https://www.buildinfo.org/) and properties in Artifactory. +#### [SBOM](extensions/commands/sbom) + +Commands to create Software Bill of Materials. + ### [Deployers](extensions/deployers) These are the current custom deployers. Recall they are experimental and can experience breaking changes, use them as a base to create your own under your control for stability. diff --git a/extensions/commands/sbom/README.md b/extensions/commands/sbom/README.md new file mode 100644 index 0000000..0866710 --- /dev/null +++ b/extensions/commands/sbom/README.md @@ -0,0 +1,29 @@ +## [Create SBOM](cmd_create_sbom.py) + +Creates an SBOM in CycloneDX format. +**This command is in an experimental stage, feedback is welcome.** + +**Parameters** +* `format` _Required_: `1.4_json`, `1.3_json`, `1.2_json`, `1.4_xml`, `1.3_xml`, `1.2_xml`, `1.1_xml`, or `1.0_xml` +* supports all other arguments used by `conan graph`, see `conan sbom:cyclonedx --help` + +**Dependencies** +The command requires the package `cyclonedx-python-lib`. +You can install it via + +```shellSession +$ pip install 'cyclonedx-python-lib>=3.1.5,<5.0.0' +``` + +Usage: + +```shellSession +$ cd conan-center-index/recipes/gmp/all +$ conan sbom:cyclonedx --format 1.4_json . +{"components": [{"bom-ref": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c", "description": "GNU M4 is an implementation of the traditional Unix macro processor", "externalReferences": [{"type": "website", "url": "https://www.gnu.org/software/m4/"}], "licenses": [{"license": {"id": "GPL-3.0-only"}}], "name": "m4", "purl": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c", "type": "library", "version": "1.4.19"}], "dependencies": [{"dependsOn": ["pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c"], "ref": "pkg:conan/gmp"}, {"ref": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c"}], "metadata": {"component": {"bom-ref": "pkg:conan/gmp", "description": "GMP is a free library for arbitrary precision arithmetic, operating on signed integers, rational numbers, and floating-point numbers.", "externalReferences": [{"type": "website", "url": "https://gmplib.org"}], "licenses": [{"license": {"id": "GPL-2.0"}}, {"license": {"id": "LGPL-3.0"}}], "name": "gmp", "purl": "pkg:conan/gmp", "type": "library"}, "timestamp": "2023-08-08T12:55:48.275439+00:00", "tools": [{"externalReferences": [{"type": "build-system", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions"}, {"type": "distribution", "url": "https://pypi.org/project/cyclonedx-python-lib/"}, {"type": "documentation", "url": "https://cyclonedx.github.io/cyclonedx-python-lib/"}, {"type": "issue-tracker", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues"}, {"type": "license", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE"}, {"type": "release-notes", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md"}, {"type": "vcs", "url": "https://github.com/CycloneDX/cyclonedx-python-lib"}, {"type": "website", "url": "https://cyclonedx.org"}], "name": "cyclonedx-python-lib", "vendor": "CycloneDX", "version": "4.0.1"}]}, "serialNumber": "urn:uuid:b5b7a98b-e06a-4627-a520-2db2a4427daa", "version": 1, "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.4"} +``` + +```shellSession +$ conan sbom:cyclonedx --format 1.4_json --requires gmp/6.2.1 +{"components": [{"bom-ref": "pkg:conan/gmp@6.2.1?rref=c0aec48648a7dff99f293870b95cad36", "description": "GMP is a free library for arbitrary precision arithmetic, operating on signed integers, rational numbers, and floating-point numbers.", "externalReferences": [{"type": "website", "url": "https://gmplib.org"}], "licenses": [{"license": {"id": "GPL-2.0"}}, {"license": {"id": "LGPL-3.0"}}], "name": "gmp", "purl": "pkg:conan/gmp@6.2.1?rref=c0aec48648a7dff99f293870b95cad36", "type": "library", "version": "6.2.1"}, {"bom-ref": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c", "description": "GNU M4 is an implementation of the traditional Unix macro processor", "externalReferences": [{"type": "website", "url": "https://www.gnu.org/software/m4/"}], "licenses": [{"license": {"id": "GPL-3.0-only"}}], "name": "m4", "purl": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c", "type": "library", "version": "1.4.19"}], "dependencies": [{"dependsOn": ["pkg:conan/gmp@6.2.1?rref=c0aec48648a7dff99f293870b95cad36"], "ref": "pkg:conan/UNKNOWN"}, {"dependsOn": ["pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c"], "ref": "pkg:conan/gmp@6.2.1?rref=c0aec48648a7dff99f293870b95cad36"}, {"ref": "pkg:conan/m4@1.4.19?rref=c1c4b1ee919e34630bb9b50046253d3c"}], "metadata": {"component": {"bom-ref": "pkg:conan/UNKNOWN", "name": "UNKNOWN", "purl": "pkg:conan/UNKNOWN", "type": "library"}, "timestamp": "2023-08-09T08:49:53.324361+00:00", "tools": [{"externalReferences": [{"type": "build-system", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions"}, {"type": "distribution", "url": "https://pypi.org/project/cyclonedx-python-lib/"}, {"type": "documentation", "url": "https://cyclonedx.github.io/cyclonedx-python-lib/"}, {"type": "issue-tracker", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues"}, {"type": "license", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE"}, {"type": "release-notes", "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md"}, {"type": "vcs", "url": "https://github.com/CycloneDX/cyclonedx-python-lib"}, {"type": "website", "url": "https://cyclonedx.org"}], "name": "cyclonedx-python-lib", "vendor": "CycloneDX", "version": "4.0.1"}]}, "serialNumber": "urn:uuid:b1189a91-e92c-43ee-8ed5-a51e48d5aab9", "version": 1, "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.4"} +``` diff --git a/extensions/commands/sbom/cmd_cyclonedx.py b/extensions/commands/sbom/cmd_cyclonedx.py new file mode 100644 index 0000000..1c85c69 --- /dev/null +++ b/extensions/commands/sbom/cmd_cyclonedx.py @@ -0,0 +1,186 @@ +from importlib import import_module +from functools import partial +import os.path +import sys +from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Union + +from conan.api.conan_api import ConanAPI +from conan.api.output import cli_out_write +from conan.cli.args import common_graph_args, validate_common_graph_args +from conan.cli.command import conan_command +from conan.errors import ConanException + +if TYPE_CHECKING: + from cyclonedx.model.bom import Bom + + +def format_cyclonedx(formatter_module: str, formatter_class: str, bom: 'Bom') -> None: + module = import_module(formatter_module) + klass = getattr(module, formatter_class) + serialized = klass(bom).output_as_string() + cli_out_write(serialized) + + +formatter = { + "1.4_json": partial(format_cyclonedx, "cyclonedx.output.json", "JsonV1Dot4"), + "1.3_json": partial(format_cyclonedx, "cyclonedx.output.json", "JsonV1Dot3"), + "1.2_json": partial(format_cyclonedx, "cyclonedx.output.json", "JsonV1Dot2"), + "1.4_xml": partial(format_cyclonedx, "cyclonedx.output.xml", "XmlV1Dot4"), + "1.3_xml": partial(format_cyclonedx, "cyclonedx.output.xml", "XmlV1Dot3"), + "1.2_xml": partial(format_cyclonedx, "cyclonedx.output.xml", "XmlV1Dot2"), + "1.1_xml": partial(format_cyclonedx, "cyclonedx.output.xml", "XmlV1Dot1"), + "1.0_xml": partial(format_cyclonedx, "cyclonedx.output.xml", "XmlV1Dot0") +} + + +def format_text(_: 'Bom') -> None: + supported = ', '.join([v for v in formatter.keys() if v is not 'text']) + raise ConanException(f"Format 'text' not supported. Supported values are: {supported}") + + +formatter["text"] = format_text + + +@conan_command(group="SBOM", formatters=formatter) +def cyclonedx(conan_api: ConanAPI, parser, *args) -> 'Bom': + """Create a CycloneDX Software Bill of Materials (SBOM)""" + + try: + from cyclonedx.factory.license import LicenseFactory + from cyclonedx.model import (ExternalReference, ExternalReferenceType, + LicenseChoice, Tool, XsUri) + from cyclonedx.model.bom import Bom + from cyclonedx.model.component import Component, ComponentType + from packageurl import PackageURL + except ModuleNotFoundError: + # Assert on RUNTIME of the actual conan-command, that all requirements exist. + # Since conan loads all extensions when started, this check could prevent conan from running, + # if loading dependencies is performed outside the actual conan-command in global/module scope. + print('The sbom extension needs an additional package, please run:', + # keep in synk with the instructions in `README.md` + "pip install 'cyclonedx-python-lib>=3.1.5,<5.0.0'", + sep='\n', file=sys.stderr) + sys.exit(1) + + if TYPE_CHECKING: + from conans.client.graph.graph import Node + + def cyclonedx_major_version_is_4() -> bool: + try: + LicenseChoice(license=LicenseFactory().make_from_string("MIT")) + return True + except TypeError: # argument in version 3 is named license_ + return False + + def package_type_to_component_type(pt: str) -> ComponentType: + return ComponentType.APPLICATION if pt == "application" else ComponentType.LIBRARY + + def licenses(ls: Optional[Union[Tuple[str, ...], Set[str], List[str], str]]) -> Optional[Iterable[LicenseChoice]]: + """ + see https://cyclonedx.org/docs/1.4/json/#components_items_licenses + """ + if ls is None: + return None + if not isinstance(ls, (tuple, set, list)): + ls = [ls] + if cyclonedx_major_version_is_4(): + return [LicenseChoice(license=LicenseFactory().make_from_string(i)) for i in ls] # noqa + else: + return [LicenseChoice(license_=LicenseFactory().make_from_string(i)) for i in ls] + + def package_url(node: 'Node') -> Optional[PackageURL]: + """ + Creates a PURL following https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#conan + """ + return PackageURL( + type="conan", + name=node.name, + version=node.conanfile.version, + qualifiers={ + "prev": node.prev, + "rref": node.ref.revision if node.ref else None, + "user": node.conanfile.user, + "channel": node.conanfile.channel, + "repository_url": node.remote.url if node.remote else None + } + ) if node.name else None + + def create_component(node: 'Node') -> Component: + purl = package_url(node) + if cyclonedx_major_version_is_4(): + component = Component( + type=package_type_to_component_type(node.conanfile.package_type), # noqa + name=node.name or f'UNKNOWN.{id(node)}', + version=node.conanfile.version, + licenses=licenses(node.conanfile.license), + bom_ref=purl.to_string() if purl else None, + purl=purl, + description=node.conanfile.description + ) + else: + component = Component( + component_type=package_type_to_component_type(node.conanfile.package_type), + name=node.name or f'UNKNOWN.{id(node)}', + version=node.conanfile.version, + licenses=licenses(node.conanfile.license), + bom_ref=purl.to_string() if purl else None, + purl=purl, + description=node.conanfile.description + ) + if node.conanfile.homepage and cyclonedx_major_version_is_4(): # bug in cyclonedx 3 enforces hashes + component.external_references.add(ExternalReference( + type=ExternalReferenceType.WEBSITE, # noqa + url=XsUri(node.conanfile.homepage), + )) # noqa + return component + + def me_as_tool() -> Tool: + tool = Tool(name="conan extension recipe:create-sbom") + if cyclonedx_major_version_is_4(): + tool.external_references.add(ExternalReference( + type=ExternalReferenceType.WEBSITE, # noqa + url=XsUri("https://github.com/conan-io/conan-extensions"))) # noqa + else: + tool.external_references.add(ExternalReference( + reference_type=ExternalReferenceType.WEBSITE, + url=XsUri("https://github.com/conan-io/conan-extensions"))) + return tool + + # region COPY FROM conan: cli/commands/graph.py + common_graph_args(parser) + # FIXME: Process the ``--build-require`` argument + parser.add_argument("--build-require", action='store_true', default=False, + help='Whether the provided path is a build-require') + args = parser.parse_args(*args) + validate_common_graph_args(args) + cwd = os.getcwd() + path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] + overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, + conanfile_path=path, + cwd=cwd, + partial=args.lockfile_partial, + overrides=overrides) + profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) + if path: + deps_graph = conan_api.graph.load_graph_consumer(path, args.name, args.version, + args.user, args.channel, + profile_host, profile_build, lockfile, + remotes, args.update) + else: + deps_graph = conan_api.graph.load_graph_requires(args.requires, args.tool_requires, + profile_host, profile_build, lockfile, + remotes, args.update) + # endregion COPY + + components = {node: create_component(node) for node in deps_graph.nodes} + bom = Bom() + bom.metadata.component = components[deps_graph.root] + bom.metadata.tools.add(me_as_tool()) + for node in deps_graph.nodes[1:]: # node 0 is the root + bom.components.add(components[node]) + if cyclonedx_major_version_is_4(): + for dep in deps_graph.nodes: + bom.register_dependency(components[dep], [components[dep_dep.dst] for dep_dep in dep.dependencies]) # noqa + return bom diff --git a/tests/test_sbom_cyclonedx.py b/tests/test_sbom_cyclonedx.py new file mode 100644 index 0000000..74af4d5 --- /dev/null +++ b/tests/test_sbom_cyclonedx.py @@ -0,0 +1,136 @@ +import json +import xml.dom.minidom +import os +import pytest +import tempfile +import textwrap + +from tools import save, run + +REQ_LIB = "gmp" +REQ_VER = "6.2.1" +REQ_DEP = "m4" + + +def cyclonedx_major_version_is_4() -> bool: + try: + from cyclonedx.factory.license import LicenseFactory + from cyclonedx.model import LicenseChoice + LicenseChoice(license=LicenseFactory().make_from_string("MIT")) + return True + except TypeError: # argument in version 3 is named license_ + return False + + +@pytest.fixture(autouse=True) +def conan_test(): + old_env = dict(os.environ) + env_vars = {"CONAN_HOME": tempfile.mkdtemp(suffix='conans')} + os.environ.update(env_vars) + current = tempfile.mkdtemp(suffix="conans") + cwd = os.getcwd() + os.chdir(current) + try: + yield + finally: + os.chdir(cwd) + os.environ.clear() + os.environ.update(old_env) + + +def _test_generated_sbom_json(sbom, test_metadata_name, spec_version): + assert sbom["bomFormat"] == "CycloneDX" + assert sbom["specVersion"] == spec_version + + assert "metadata" in sbom + assert "component" in sbom["metadata"] + if test_metadata_name: + assert sbom["metadata"]["component"]["name"] == "TestPackage" + + assert "components" in sbom + assert len([c for c in sbom["components"] if c["name"] == REQ_LIB and c["version"] == REQ_VER]) == 1 + assert len([c for c in sbom["components"] if c["name"] == REQ_DEP]) == 1 + + +def _test_generated_sbom_xml(sbom, test_metadata_name, spec_version): + def with_ns(key: str) -> str: + ns = "ns0:" if cyclonedx_major_version_is_4() else "" + return ns + key + + schema = sbom.getAttribute("xmlns:ns0" if cyclonedx_major_version_is_4() else "xmlns") + assert "cyclonedx" in schema + assert schema.split("/")[-1] == spec_version + + if spec_version not in ['1.1', '1.0']: + metadata = sbom.getElementsByTagName(with_ns("metadata")) + assert metadata + component = metadata[0].getElementsByTagName(with_ns("component")) + assert component + if test_metadata_name: + assert component[0].getElementsByTagName(with_ns("name"))[0].firstChild.nodeValue == "TestPackage" + + components = sbom.getElementsByTagName(with_ns("components")) + assert components + components = components[0].getElementsByTagName(with_ns("component")) + assert components + assert 1 == len([ + c for c in components if + c.getElementsByTagName(with_ns("name"))[0].firstChild.nodeValue == REQ_LIB + and + c.getElementsByTagName(with_ns("version"))[0].firstChild.nodeValue == REQ_VER + ]) + assert 1 == len([ + c for c in components if + c.getElementsByTagName(with_ns("name"))[0].firstChild.nodeValue == REQ_DEP + ]) + + +def create_conanfile_txt(): + return textwrap.dedent(f""" + [requires] + {REQ_LIB}/{REQ_VER} + """) + + +def create_conanfile_py(): + return textwrap.dedent(f""" + from conan import ConanFile + from conan.tools.cmake import cmake_layout + + class Pkg(ConanFile): + generators = "CMakeToolchain", "CMakeDeps" + name = "TestPackage" + + def requirements(self): + self.requires("{REQ_LIB}/{REQ_VER}") + + def layout(self): + cmake_layout(self) + + """) + + +params = [] +for version in ['1.4', '1.3', '1.2', '1.1', '1.0']: + for ft in ['xml', 'json']: + if ft == 'json' and version in ['1.1', '1.0']: + continue + params.append((create_conanfile_py(), "conanfile.py", ".", True, f"{version}_{ft}")) + params.append((create_conanfile_txt(), "conanfile.txt", ".", False, f"{version}_{ft}")) + params.append((str(), "doesnotmatter.txt", f"--requires {REQ_LIB}/{REQ_VER}", False, f"{version}_{ft}")) + + +@pytest.mark.parametrize("conanfile_content,conanfile_name,sbom_command,test_metadata_name,sbom_format", params) +def test_create_sbom(conanfile_content, conanfile_name, sbom_command, test_metadata_name, sbom_format): + repo = os.path.join(os.path.dirname(__file__), "..") + run(f"conan config install {repo}") + run("conan profile detect") + save(conanfile_name, conanfile_content) + + out = run(f"conan sbom:cyclonedx --format {sbom_format} {sbom_command}", stderr=None) + if sbom_format.split('_')[1] == "json": + sbom = json.loads(out) + _test_generated_sbom_json(sbom, test_metadata_name, sbom_format.split('_')[0]) + else: + dom = xml.dom.minidom.parseString(out).firstChild + _test_generated_sbom_xml(dom, test_metadata_name, sbom_format.split('_')[0]) diff --git a/tests/tools.py b/tests/tools.py index 6d3220d..01e8127 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -1,15 +1,15 @@ import subprocess -def run(cmd, error=False): +def run(cmd, error=False, *, stdout=subprocess.PIPE, stderr=subprocess.PIPE): process = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=stdout, + stderr=stderr, shell=True) out, err = process.communicate() - out = out.decode("utf-8") - err = err.decode("utf-8") + out = out.decode("utf-8") if stdout else "" + err = err.decode("utf-8") if stderr else "" ret = process.returncode output = err + out