From d399537e9eb4cc18dc7b0ab2a9b356ab85e02964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Nicola?= Date: Thu, 25 Jan 2024 10:19:41 +0100 Subject: [PATCH] Add: nasl-cli notus update subcommand (#1558) * Add: nasl-cli notus update subcommand Load up Notus Advisories into redis. It performs the signature check, the hashsum check and the upload. Signature check is optional. It must be enabled with the command line option but also the environment variable to the gnupg keyring must be set. Usage: `GPGHOME=/path/to/.gnupg nasl-cli notus update --path --signature-check` * remove swap files and unnecessary commented code * Change: Rename NvtDispatcher by CacheDispatcher Since it is no not only for Nvts but also for Notus advisories. * Change: Load both notus and vts caches with `nasl-cli feed update` instead of having two subcommands. * Change: run feed-verifier with --nvt-only option * Fix: CI. Update zlib version to 1.3.1 --- rust/Cargo.lock | 2 + rust/cross.Dockerfile | 4 +- rust/cross_aarch64.Dockerfile | 4 +- rust/feed-verifier/src/main.rs | 2 +- rust/feed/src/update/mod.rs | 8 +- rust/models/src/advisories.rs | 197 ++++++++++++++++++++ rust/models/src/lib.rs | 2 + rust/nasl-builtin-knowledge-base/src/lib.rs | 1 + rust/nasl-cli/Cargo.toml | 1 + rust/nasl-cli/README.md | 14 +- rust/nasl-cli/src/feed/mod.rs | 107 +++++++++-- rust/nasl-cli/src/interpret/mod.rs | 6 +- rust/nasl-cli/src/main.rs | 6 +- rust/nasl-cli/src/notusupdate/mod.rs | 5 + rust/nasl-cli/src/notusupdate/update.rs | 88 +++++++++ rust/nasl-interpreter/src/include.rs | 2 +- rust/notus/src/loader/hashsum.rs | 87 ++++++++- rust/notus/src/loader/mod.rs | 15 +- rust/redis-storage/src/connector.rs | 55 ++++-- rust/redis-storage/src/lib.rs | 3 +- rust/storage/Cargo.toml | 1 + rust/storage/src/lib.rs | 21 +++ rust/storage/src/nvt.rs | 7 +- 23 files changed, 583 insertions(+), 55 deletions(-) create mode 100644 rust/models/src/advisories.rs create mode 100644 rust/nasl-cli/src/notusupdate/mod.rs create mode 100644 rust/nasl-cli/src/notusupdate/update.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 429d365a2..955ef29e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1682,6 +1682,7 @@ dependencies = [ "models", "nasl-interpreter", "nasl-syntax", + "notus", "redis-storage", "scanconfig", "serde", @@ -2771,6 +2772,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" name = "storage" version = "0.1.0" dependencies = [ + "models", "serde", "time", "tracing", diff --git a/rust/cross.Dockerfile b/rust/cross.Dockerfile index c5af8d582..4ec764b70 100644 --- a/rust/cross.Dockerfile +++ b/rust/cross.Dockerfile @@ -17,10 +17,10 @@ RUN ./configure --host=x86_64-unknown-linux-gnu --with-pcap=linux RUN cat config.log RUN make install -RUN curl --output /tmp/zlib.tar.gz https://www.zlib.net/zlib-1.3.tar.gz +RUN curl --output /tmp/zlib.tar.gz https://www.zlib.net/zlib-1.3.1.tar.gz WORKDIR /tmp RUN tar xvf zlib.tar.gz -WORKDIR /tmp/zlib-1.3 +WORKDIR /tmp/zlib-1.3.1 RUN ./configure RUN make install RUN ldconfig diff --git a/rust/cross_aarch64.Dockerfile b/rust/cross_aarch64.Dockerfile index e51bc14fc..60daa6fee 100644 --- a/rust/cross_aarch64.Dockerfile +++ b/rust/cross_aarch64.Dockerfile @@ -22,10 +22,10 @@ RUN ./configure --host=aarch64-unknown-linux-gnu --with-pcap=linux RUN cat config.log RUN make install -RUN curl --output /tmp/zlib.tar.gz https://www.zlib.net/zlib-1.3.tar.gz +RUN curl --output /tmp/zlib.tar.gz https://www.zlib.net/zlib-1.3.1.tar.gz WORKDIR /tmp RUN tar xvzf zlib.tar.gz -WORKDIR /tmp/zlib-1.3 +WORKDIR /tmp/zlib-1.3.1 RUN ./configure RUN make install RUN ldconfig diff --git a/rust/feed-verifier/src/main.rs b/rust/feed-verifier/src/main.rs index d8c59fb1f..1afd6126b 100644 --- a/rust/feed-verifier/src/main.rs +++ b/rust/feed-verifier/src/main.rs @@ -166,7 +166,7 @@ fn main() { }; let (ncd, nasl_cli) = run_get( &mut kb, - &format!("{} feed update", nasl_cli.to_str().unwrap_or_default()), + &format!("{} feed update --vts-only", nasl_cli.to_str().unwrap_or_default()), ) .expect("results"); let mut errors = 0; diff --git a/rust/feed/src/update/mod.rs b/rust/feed/src/update/mod.rs index 0681be708..3aaad826e 100644 --- a/rust/feed/src/update/mod.rs +++ b/rust/feed/src/update/mod.rs @@ -14,7 +14,7 @@ use nasl_interpreter::{ }; use storage::{nvt::NVTField, Dispatcher, NoOpRetriever}; -use crate::verify::{self, SignatureChecker, HashSumFileItem}; +use crate::verify::{self, HashSumFileItem, SignatureChecker}; pub use self::error::ErrorKind; @@ -69,7 +69,6 @@ pub fn feed_version>( Ok(feed_version) } - impl<'a, R, S, L, V, K> SignatureChecker for Update where S: Sync + Send + Dispatcher, @@ -77,8 +76,9 @@ where L: Sync + Send + Loader + AsBufReader, V: Iterator, verify::Error>>, R: Read + 'a, -{} - +{ +} + impl<'a, S, L, V, K, R> Update where S: Sync + Send + Dispatcher, diff --git a/rust/models/src/advisories.rs b/rust/models/src/advisories.rs new file mode 100644 index 000000000..37f38aace --- /dev/null +++ b/rust/models/src/advisories.rs @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +use std::collections::HashMap; + +/// Represents an advisory json file for notus product. +#[cfg_attr(feature = "serde_support", derive(serde::Deserialize))] +#[derive(Debug, Clone)] +pub struct ProductsAdivisories { + /// Version of the advisory file + pub version: String, + /// SPDX license identifier + #[cfg_attr(feature = "serde_support", serde(rename = "spdx-license-identifier"))] + pub license_identifier: String, + /// Copyright + pub copyright: String, + /// Vulnerability Family + pub family: String, + /// List of Advisories + #[cfg_attr(feature = "serde_support", serde(default))] + pub advisories: Vec, +} + +/// Represents an advisory json file for notus product. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde_support", + derive(serde::Serialize, serde::Deserialize) +)] +pub struct Advisories { + /// The advisory's title. + pub title: String, + /// The advisory's ID. + pub oid: String, + /// Creation Date + pub creation_date: u64, + /// Last modification date + pub last_modification: u64, + /// Advisory ID + pub advisory_id: String, + /// Advisory xref + pub advisory_xref: String, + /// List of cves + #[cfg_attr(feature = "serde_support", serde(default))] + pub cves: Vec, + /// Summary + pub summary: String, + /// Insight + #[cfg_attr(feature = "serde_support", serde(default))] + pub insight: String, + /// Affected + pub affected: String, + /// Listo of xrefs + #[cfg_attr(feature = "serde_support", serde(default))] + pub xrefs: Vec, + /// Quality of detection + pub qod_type: String, + /// Severity + pub severity: Severity, +} + +/// A single vulnerability from an advisory file to be stored +#[cfg_attr( + feature = "serde_support", + derive(serde::Serialize, serde::Deserialize) +)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct Vulnerability { + /// VT Parameters + pub vt_params: Vec, + /// Creation Date + pub creation_date: u64, + /// Last modification date + pub last_modification: u64, + /// Summary + pub summary: String, + /// Impact + pub impact: String, + /// Affected + pub affected: String, + /// Insight + pub insight: String, + /// Solution + pub solution: String, + /// Solution Type + pub solution_type: String, + /// Vuldetect + pub vuldeterct: String, + /// Quality of detection + pub qod_type: String, + /// Severity vector + pub severity_vector: String, + /// File name + pub filename: String, + /// All references: xrefs, cves, xrefs, advisory xrefs and advisory id. + pub refs: HashMap>, + /// Vulnerability Family + pub family: String, + /// Title + pub name: String, + /// Category + pub category: String, +} + +/// Severity +#[cfg_attr( + feature = "serde_support", + derive(serde::Serialize, serde::Deserialize) +)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct Severity { + /// Origin of the severity + pub origin: String, + /// severity date + pub date: u64, + /// Cvss version v2 + #[cfg_attr( + feature = "serde_support", + serde(skip_serializing_if = "Option::is_none") + )] + pub cvss_v2: Option, + /// cvss vector v3 + #[cfg_attr( + feature = "serde_support", + serde(skip_serializing_if = "Option::is_none") + )] + pub cvss_v3: Option, +} + +pub struct ProductsAdivisoriesIterator<'a> { + products_advisories: &'a ProductsAdivisories, + index: usize, +} + +impl<'a> Iterator for ProductsAdivisoriesIterator<'a> { + type Item = &'a Advisories; + + fn next(&mut self) -> Option<&'a Advisories> { + if self.index < self.products_advisories.advisories.len() { + let result = Some(&self.products_advisories.advisories[self.index]); + self.index += 1; + result + } else { + None + } + } +} + +impl ProductsAdivisories { + pub fn iter(&self) -> ProductsAdivisoriesIterator { + ProductsAdivisoriesIterator { + products_advisories: self, + index: 0, + } + } +} + +pub struct VulnerabilityData<'a> { + pub adv: &'a Advisories, + pub product_data: &'a ProductsAdivisories, + pub filename: &'a String, +} + +impl<'a> From<&VulnerabilityData<'a>> for Vulnerability { + fn from(data: &VulnerabilityData<'a>) -> Self { + let sv = match &data.adv.severity.cvss_v2 { + Some(cvss) => cvss, + None => match &data.adv.severity.cvss_v3 { + Some(cvss) => cvss, + None => "", + }, + }; + + let refs = HashMap::new(); + Self { + vt_params: Vec::new(), + creation_date: data.adv.creation_date, + last_modification: data.adv.last_modification, + summary: data.adv.summary.to_owned(), + impact: "".to_string(), + affected: data.adv.affected.to_owned(), + insight: data.adv.insight.to_owned(), + solution: "Please install the updated package(s).".to_string(), + solution_type: "VendorFix".to_string(), + vuldeterct: "Checks if a vulnerable package version is present on the target host." + .to_string(), + qod_type: data.adv.qod_type.to_owned(), + severity_vector: sv.to_string(), + filename: data.filename.to_string(), + refs, + family: data.product_data.family.to_owned(), + name: data.adv.title.to_owned(), + category: "3".to_string(), + } + } +} diff --git a/rust/models/src/lib.rs b/rust/models/src/lib.rs index 1db6720f5..5473a41f1 100644 --- a/rust/models/src/lib.rs +++ b/rust/models/src/lib.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: GPL-2.0-or-later +mod advisories; mod credential; mod host_info; mod parameter; @@ -15,6 +16,7 @@ mod status; mod target; mod vt; +pub use advisories::*; pub use credential::*; pub use host_info::*; pub use parameter::*; diff --git a/rust/nasl-builtin-knowledge-base/src/lib.rs b/rust/nasl-builtin-knowledge-base/src/lib.rs index bea352003..a6b6395d5 100644 --- a/rust/nasl-builtin-knowledge-base/src/lib.rs +++ b/rust/nasl-builtin-knowledge-base/src/lib.rs @@ -54,6 +54,7 @@ fn get_kb_item(register: &Register, c: &Context) -> Result None, + Field::NOTUS(_) => None, Field::KB(kb) => kb.value.into(), }) }) diff --git a/rust/nasl-cli/Cargo.toml b/rust/nasl-cli/Cargo.toml index d1919f0e5..4eb8d8048 100644 --- a/rust/nasl-cli/Cargo.toml +++ b/rust/nasl-cli/Cargo.toml @@ -27,6 +27,7 @@ tracing-subscriber = { version = "0.3.17" } serde_json = "1.0.96" toml = "0.8.6" serde = "1.0.190" +notus = { version = "0.1.0", path = "../notus" } [features] diff --git a/rust/nasl-cli/README.md b/rust/nasl-cli/README.md index 1033c1d76..9b340e75c 100644 --- a/rust/nasl-cli/README.md +++ b/rust/nasl-cli/README.md @@ -38,7 +38,6 @@ Hello, world! Usage: `nasl-cli execute [OPTIONS] [-t HOST]