From 055942644f43126441bb19133458a6a594b52e8c Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Mon, 7 Oct 2024 12:48:34 +0300 Subject: [PATCH 1/2] Flake update Signed-off-by: Henri Rosten --- .envrc | 1 + doc/vulnxscan.md | 1 - flake.lock | 84 +++++------------- flake.nix | 8 -- nix/devshell.nix | 4 +- nix/packages.nix | 216 +++++------------------------------------------ 6 files changed, 48 insertions(+), 266 deletions(-) diff --git a/.envrc b/.envrc index f07c6c9..f4ca5cc 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,4 @@ +#! /usr/bin/env bash # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 diff --git a/doc/vulnxscan.md b/doc/vulnxscan.md index 9552f20..7a79349 100644 --- a/doc/vulnxscan.md +++ b/doc/vulnxscan.md @@ -376,5 +376,4 @@ For now, consider `vulnxscan` as a demonstration. Some improvement ideas are lis - Nix ecosystem is not supported in OSV: the way `osv.py` makes use of OSV data for Nix targets -- as explained in section [Nix and OSV vulnerability database](#nix-and-osv-vulnerability-database) -- makes the reported OSV vulnerabilities include false positives. ### Other Future Work -- [vulnxscan](../src/vulnxscan/vulnxscan_cli.py) uses vulnix from a [forked repository](https://github.com/henrirosten/vulnix), to include vulnix support for [scanning runtime-only dependencies](https://github.com/nix-community/vulnix/compare/master...henrirosten:vulnix:master). - [vulnxscan](../src//vulnxscan/vulnxscan_cli.py) could include more scanners in addition to [vulnix](https://github.com/nix-community/vulnix), [grype](https://github.com/anchore/grype), and [osv.py](../src/vulnxscan/osv.py). Suggestions for other open-source scanners, especially those that can digest CycloneDX or SPDX SBOMs are welcome. Consider e.g. [bombon](https://github.com/nikstur/bombon) and [cve-bin-tool](https://github.com/intel/cve-bin-tool). Adding cve-bin-tool to vulnxscan was [demonstrated](https://github.com/tiiuae/sbomnix/pull/75) earlier, but not merged due to reasons explained in the [PR](https://github.com/tiiuae/sbomnix/pull/75#issuecomment-1670958503). diff --git a/flake.lock b/flake.lock index cd6397c..6b8dd47 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1688025799, - "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=", + "lastModified": 1717312683, + "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=", "owner": "nix-community", "repo": "flake-compat", - "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c", + "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea", "type": "github" }, "original": { @@ -21,11 +21,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1698882062, - "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=", + "lastModified": 1727826117, + "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877", + "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1", "type": "github" }, "original": { @@ -36,11 +36,11 @@ }, "flake-root": { "locked": { - "lastModified": 1692742795, - "narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=", + "lastModified": 1723604017, + "narHash": "sha256-rBtQ8gg+Dn4Sx/s+pvjdq3CB2wQNzx9XGFq/JVGCB6k=", "owner": "srid", "repo": "flake-root", - "rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937", + "rev": "b759a56851e10cb13f6b8e5698af7b59c44be26e", "type": "github" }, "original": { @@ -49,29 +49,13 @@ "type": "github" } }, - "nix-visualize": { - "flake": false, - "locked": { - "lastModified": 1687577587, - "narHash": "sha256-Z1r8XHszoUnQinl63yXvQG6Czp5HnYNG37AY+EEiT4w=", - "owner": "craigmbooth", - "repo": "nix-visualize", - "rev": "cafaba50cd63ba9c759c56af71fd0d22fd60a548", - "type": "github" - }, - "original": { - "owner": "craigmbooth", - "repo": "nix-visualize", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1711163522, - "narHash": "sha256-YN/Ciidm+A0fmJPWlHBGvVkcarYWSC+s3NTPk/P+q3c=", + "lastModified": 1728018373, + "narHash": "sha256-NOiTvBbRLIOe5F6RbHaAh6++BNjsb149fGZd1T4+KBg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "44d0940ea560dee511026a53f0e2e2cde489b4d4", + "rev": "bc947f541ae55e999ffdb4013441347d83b00feb", "type": "github" }, "original": { @@ -83,20 +67,14 @@ }, "nixpkgs-lib": { "locked": { - "dir": "lib", - "lastModified": 1698611440, - "narHash": "sha256-jPjHjrerhYDy3q9+s5EAsuhyhuknNfowY6yt6pjn9pc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "0cbe9f69c234a7700596e943bfae7ef27a31b735", - "type": "github" + "lastModified": 1727825735, + "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz" }, "original": { - "dir": "lib", - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz" } }, "root": { @@ -104,10 +82,8 @@ "flake-compat": "flake-compat", "flake-parts": "flake-parts", "flake-root": "flake-root", - "nix-visualize": "nix-visualize", "nixpkgs": "nixpkgs", - "treefmt-nix": "treefmt-nix", - "vulnix": "vulnix" + "treefmt-nix": "treefmt-nix" } }, "treefmt-nix": { @@ -117,11 +93,11 @@ ] }, "locked": { - "lastModified": 1699786194, - "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", + "lastModified": 1727984844, + "narHash": "sha256-xpRqITAoD8rHlXQafYZOLvUXCF6cnZkPfoq67ThN0Hc=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1", + "rev": "4446c7a6fc0775df028c5a3f6727945ba8400e64", "type": "github" }, "original": { @@ -129,22 +105,6 @@ "repo": "treefmt-nix", "type": "github" } - }, - "vulnix": { - "flake": false, - "locked": { - "lastModified": 1676379453, - "narHash": "sha256-KXvmnaMjv//zd4aSwu4qmbon1Iyzdod6CPms7LIxeVU=", - "owner": "henrirosten", - "repo": "vulnix", - "rev": "ad28b2924027a44a9b81493a0f9de1b0e8641005", - "type": "github" - }, - "original": { - "owner": "henrirosten", - "repo": "vulnix", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 23b29fc..e6ec3a8 100644 --- a/flake.nix +++ b/flake.nix @@ -17,14 +17,6 @@ url = "github:nix-community/flake-compat"; flake = false; }; - nix-visualize = { - url = "github:craigmbooth/nix-visualize"; - flake = false; - }; - vulnix = { - url = "github:henrirosten/vulnix"; - flake = false; - }; }; outputs = inputs @ {flake-parts, ...}: diff --git a/nix/devshell.nix b/nix/devshell.nix index 8957cff..1b803be 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -22,13 +22,13 @@ grype gzip nix + nix-visualize pylint reuse + vulnix ]) ++ (with self'.packages; [ - nix-visualize python # that python with all sbomnix [dev-]dependencies - vulnix ]); # Add the repo root to PYTHONPATH, so invoking entrypoints (and them being diff --git a/nix/packages.nix b/nix/packages.nix index 5cddc15..19eba0c 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 -{inputs, ...}: { +{ perSystem = { pkgs, lib, @@ -12,175 +12,6 @@ packages = rec { default = sbomnix; - # https://github.com/thombashi/df-diskcache - dfdiskcache = pp.buildPythonPackage rec { - version = "0.0.2"; - pname = "df-diskcache"; - format = "setuptools"; - - src = pkgs.fetchFromGitHub { - owner = "thombashi"; - repo = "df-diskcache"; - rev = "v${version}"; - hash = "sha256-s+sqEPXw6tbEz9mnG+qeUSF6BmDssYhaDYOmraFaRbw="; - }; - - propagatedBuildInputs = - [ - simplesqlite - ] - ++ [ - pp.pandas - pp.typing-extensions - ]; - - pythonImportsCheck = ["dfdiskcache"]; - doCheck = false; - }; - - # requests-ratelimiter currently does not support pyrate-limiter v3, - # see: https://github.com/JWCook/requests-ratelimiter/issues/78 - pyrate-limiter = pp.buildPythonPackage rec { - version = "2.10.0"; - pname = "pyrate-limiter"; - format = "pyproject"; - - src = pkgs.fetchFromGitHub { - owner = "vutran1710"; - repo = "PyrateLimiter"; - rev = "v${version}"; - hash = "sha256-CPusPeyTS+QyWiMHsU0ii9ZxPuizsqv0wQy3uicrDw0="; - }; - - propagatedBuildInputs = [ - pp.poetry-core - ]; - }; - - # requests-ratelimiter currently does not support pyrate-limiter v3, - # see: https://github.com/JWCook/requests-ratelimiter/issues/78 - requests-ratelimiter = pp.buildPythonPackage rec { - version = "0.4.0"; - pname = "requests-ratelimiter"; - format = "pyproject"; - - src = pkgs.fetchFromGitHub { - owner = "JWCook"; - repo = pname; - rev = "v${version}"; - hash = "sha256-F9bfcwijyyKzlFKBJAC/5ETc4/hZpPhm2Flckku2z6M="; - }; - - propagatedBuildInputs = [pyrate-limiter pp.requests]; - }; - - # reuse is imported by sbomdb.py. For this to work with a python.withPackages, - # reuse needs to be a buildPythonPackage, not buildPythonApplication. - # Sent to nixpkgs in https://github.com/NixOS/nixpkgs/pull/267527 - # - # Also note their library docstring: - # > reuse is a tool for compliance with the REUSE recommendations. - # > Although the API is documented, it is **NOT** guaranteed stable - # > between minor or even patch releases. - # > The semantic versioning of this program pertains exclusively to the - # > reuse CLI command. If you want to use reuse as a Python library, you - # > should pin reuse to an exact version. - # … so it might be a good idea to pin this anyways. - reuse = pp.buildPythonPackage rec { - pname = "reuse"; - version = "2.1.0"; - format = "pyproject"; - - src = pkgs.fetchFromGitHub { - owner = "fsfe"; - repo = "reuse-tool"; - rev = "refs/tags/v${version}"; - hash = "sha256-MEQiuBxe/ctHlAnmLhQY4QH62uAcHb7CGfZz+iZCRSk="; - }; - - nativeBuildInputs = with pp; [ - poetry-core - ]; - - propagatedBuildInputs = with pp; [ - binaryornot - boolean-py - debian - jinja2 - license-expression - ]; - - nativeCheckInputs = with pp; [pytestCheckHook]; - - disabledTestPaths = [ - # pytest wants to execute the actual source files for some reason, which fails with ImportPathMismatchError() - "src/reuse" - ]; - }; - - # Required due to dfdiskcache - simplesqlite = pp.buildPythonPackage rec { - version = "1.5.2"; - pname = "SimpleSQLite"; - format = "setuptools"; - - src = pkgs.fetchFromGitHub { - owner = "thombashi"; - repo = "SimpleSQLite"; - rev = "v${version}"; - hash = "sha256-Yr17T0/EwVaOjG+mzdxopivj0fuvQdZdX1bFj8vq0MM="; - }; - - propagatedBuildInputs = - [ - sqliteschema - ] - ++ [ - pp.dataproperty - pp.mbstrdecoder - pp.pathvalidate - pp.tabledata - pp.typepy - ]; - - pythonImportsCheck = ["simplesqlite"]; - doCheck = false; - }; - - # Required due to dfdiskcache - sqliteschema = pp.buildPythonPackage rec { - version = "1.4.0"; - pname = "sqliteschema"; - format = "setuptools"; - - src = pkgs.fetchFromGitHub { - owner = "thombashi"; - repo = "sqliteschema"; - rev = "v${version}"; - hash = "sha256-IzHdYBnh6udVsanWTPSsX4p4PG934YCdzs9Ow/NW86E="; - }; - - propagatedBuildInputs = [ - pp.mbstrdecoder - pp.tabledata - pp.typepy - ]; - - pythonImportsCheck = ["sqliteschema"]; - doCheck = false; - }; - - # We use vulnix from 'https://github.com/henrirosten/vulnix' to get - # vulnix support for runtime-only scan ('-C' command-line option) - # which is currently not available in released version of vulnix. - # Pending https://github.com/nix-community/vulnix/pull/80 - vulnix = (import inputs.vulnix) { - inherit (inputs) nixpkgs; # required but not used as we provide pkgs - inherit pkgs lib; - }; - - nix-visualize = (import inputs.nix-visualize) {inherit pkgs;}; - sbomnix = pp.buildPythonPackage rec { pname = "sbomnix"; version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../VERSION); @@ -188,26 +19,23 @@ src = lib.cleanSource ../.; - propagatedBuildInputs = - [ - dfdiskcache - pyrate-limiter - requests-ratelimiter - reuse - ] - ++ (with pp; [ - beautifulsoup4 - colorlog - filelock - graphviz - numpy - packageurl-python - packaging - pandas - requests - requests-cache - tabulate - ]); + propagatedBuildInputs = with pp; [ + beautifulsoup4 + colorlog + dfdiskcache + filelock + graphviz + numpy + packageurl-python + packaging + pandas + pyrate-limiter + reuse + requests + requests-cache + requests-ratelimiter + tabulate + ]; pythonImportsCheck = ["sbomnix"]; @@ -223,16 +51,18 @@ ]; }; # a python with all python packages imported by sbomnix itself - python = pkgs.python3.withPackages (ps: - (with ps; [ + python = pkgs.python3.withPackages ( + ps: (with ps; [ beautifulsoup4 colorlog + dfdiskcache filelock graphviz numpy packageurl-python packaging pandas + reuse requests requests-cache requests-ratelimiter @@ -244,7 +74,7 @@ jsonschema pytest ]) - ++ [dfdiskcache reuse]); + ); }; }; } From e4be4e14595686d62df6778955fcb7789211e13b Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Mon, 7 Oct 2024 13:45:23 +0300 Subject: [PATCH 2/2] Update github action dependencies Signed-off-by: Henri Rosten --- .github/workflows/check-commit-message.yaml | 2 +- .github/workflows/release_sbomnix.yml | 6 +++--- .github/workflows/test_sbomnix.yml | 10 ++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-commit-message.yaml b/.github/workflows/check-commit-message.yaml index 5c8577e..f84c54b 100644 --- a/.github/workflows/check-commit-message.yaml +++ b/.github/workflows/check-commit-message.yaml @@ -15,7 +15,7 @@ jobs: commit-msg: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.2.0 - name: Check Commit Message env: GITHUB_CONTEXT: ${{ toJson(github) }} diff --git a/.github/workflows/release_sbomnix.yml b/.github/workflows/release_sbomnix.yml index b0460fb..652a8cf 100644 --- a/.github/workflows/release_sbomnix.yml +++ b/.github/workflows/release_sbomnix.yml @@ -11,14 +11,14 @@ on: - 'v*' jobs: - build: + build: name: Upload Release Asset runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v22 + - uses: actions/checkout@v4.2.0 + - uses: cachix/install-nix-action@v30 with: nix_path: nixpkgs=channel:nixpkgs-unstable - name: Build release asset diff --git a/.github/workflows/test_sbomnix.yml b/.github/workflows/test_sbomnix.yml index 91f630a..b3412a4 100644 --- a/.github/workflows/test_sbomnix.yml +++ b/.github/workflows/test_sbomnix.yml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -name: Test sbomnix +name: Test sbomnix on: push: @@ -16,11 +16,13 @@ jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v22 + - uses: actions/checkout@v4.2.0 + - uses: cachix/install-nix-action@v30 with: nix_path: nixpkgs=channel:nixpkgs-unstable - - name: Make sure nix-build works + - name: Ensure nix-build works run: nix-build '' -A hello + - name: Print nix version + run: nix --version - name: Run sbomnix CI tests run: nix develop --command make test-ci