diff --git a/Cargo.lock b/Cargo.lock index 5849073c0..d1a8cfda4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2121,20 +2121,19 @@ dependencies = [ [[package]] name = "csaf-walker" -version = "0.6.0-alpha.7" -source = "git+https://github.com/ctron/csaf-walker?rev=dff3991f7a31601a9521f423ab2e49aeffc025ae#dff3991f7a31601a9521f423ab2e49aeffc025ae" +version = "0.6.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55ae0f7a5422567c2fd37c8af2ef71d6770ce3f9b89f3371003ff267955ca3c" dependencies = [ "anyhow", "async-trait", "bytes", "chrono", "cpe", - "csaf", "csv", "digest 0.10.7", "filetime", "futures", - "html-escape", "humantime", "log", "percent-encoding", @@ -5763,8 +5762,9 @@ dependencies = [ [[package]] name = "sbom-walker" -version = "0.6.0-alpha.7" -source = "git+https://github.com/ctron/csaf-walker?rev=dff3991f7a31601a9521f423ab2e49aeffc025ae#dff3991f7a31601a9521f423ab2e49aeffc025ae" +version = "0.6.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "689dc1dc60f3dad4c59921393a36276177b9b4f184916f31782a040b08e3b36b" dependencies = [ "anyhow", "async-trait", @@ -8385,8 +8385,9 @@ dependencies = [ [[package]] name = "walker-common" -version = "0.6.0-alpha.7" -source = "git+https://github.com/ctron/csaf-walker?rev=dff3991f7a31601a9521f423ab2e49aeffc025ae#dff3991f7a31601a9521f423ab2e49aeffc025ae" +version = "0.6.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68792c84b081097284ba1295ba811fd3064979b29813e1ec543538619137c29b" dependencies = [ "anyhow", "async-trait", @@ -8417,8 +8418,9 @@ dependencies = [ [[package]] name = "walker-extras" -version = "0.6.0-alpha.7" -source = "git+https://github.com/ctron/csaf-walker?rev=dff3991f7a31601a9521f423ab2e49aeffc025ae#dff3991f7a31601a9521f423ab2e49aeffc025ae" +version = "0.6.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712ee4caf230dca176ac0fc52424d322a800fedda067ae44298521bac8ead9ff" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d77d5eda0..e650290aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -121,8 +121,3 @@ cyclonedx-bom = { git = "https://github.com/carlosthe19916/cyclonedx-rust-cargo" #cve = { git = "https://github.com/ctron/cve", rev = "77341768d893098f1f8b1397364fecfb16b4d776" } # FIXME: awaiting release #hide = { git = "https://github.com/ctron/hide", rev = "109059cce66fd85dbe40f59ad40b7dab47b70fe1" } # FIXME: awaiting release - -csaf-walker = { git = "https://github.com/ctron/csaf-walker", rev ="dff3991f7a31601a9521f423ab2e49aeffc025ae" } # FIXME: awaiting release -walker-common = { git = "https://github.com/ctron/csaf-walker", rev ="dff3991f7a31601a9521f423ab2e49aeffc025ae" } # FIXME: awaiting release -walker-extras = { git = "https://github.com/ctron/csaf-walker", rev ="dff3991f7a31601a9521f423ab2e49aeffc025ae" } # FIXME: awaiting release -sbom-walker = { git = "https://github.com/ctron/csaf-walker", rev ="dff3991f7a31601a9521f423ab2e49aeffc025ae" } # FIXME: awaiting release diff --git a/bombastic/index/Cargo.toml b/bombastic/index/Cargo.toml index 604885f8a..162d42538 100644 --- a/bombastic/index/Cargo.toml +++ b/bombastic/index/Cargo.toml @@ -21,4 +21,4 @@ tokio = { version = "1", features = ["full"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.68" env_logger = "0.10" -sbom-walker = { version = "=0.6.0-alpha.7", default-features = false, features = ["spdx-rs", "cyclonedx-bom", "crypto-openssl"] } +sbom-walker = { version = "=0.6.0-alpha.6", default-features = false, features = ["spdx-rs", "cyclonedx-bom", "crypto-openssl"] } diff --git a/bombastic/walker/Cargo.toml b/bombastic/walker/Cargo.toml index 19e79447c..35b8350fa 100644 --- a/bombastic/walker/Cargo.toml +++ b/bombastic/walker/Cargo.toml @@ -11,15 +11,15 @@ bzip2 = "0.4" clap = "4.3.3" humantime = "2.1.0" log = "0.4" -sbom-walker = { version = "=0.6.0-alpha.7", features = ["crypto-openssl", "spdx-rs", "cyclonedx-bom"], default-features = false } +sbom-walker = { version = "=0.6.0-alpha.6", features = ["crypto-openssl", "spdx-rs", "cyclonedx-bom"], default-features = false } serde_json = "1" spdx-expression = "0.5" time = "0.3.21" tokio = "1.28.2" tracing = "0.1" url = "2.4.0" -walker-common = "=0.6.0-alpha.7" -walker-extras = "=0.6.0-alpha.7" +walker-common = "=0.6.0-alpha.6" +walker-extras = "=0.6.0-alpha.6" trustification-auth = { path = "../../auth" } trustification-infrastructure = { path = "../../infrastructure" } diff --git a/bombastic/walker/src/scanner.rs b/bombastic/walker/src/scanner.rs index fb18ae3ee..dca128feb 100644 --- a/bombastic/walker/src/scanner.rs +++ b/bombastic/walker/src/scanner.rs @@ -12,7 +12,7 @@ use tracing::{instrument, log}; use url::Url; use walker_common::{ fetcher::{Fetcher, FetcherOptions}, - sender::{self, provider::TokenProvider, HttpSenderOptions}, + sender::{self, provider::TokenProvider}, since::Since, validate::ValidationOptions, }; @@ -55,12 +55,13 @@ impl Scanner { pub async fn run_once(&self) -> anyhow::Result<()> { let since = Since::new(None::, self.options.since_file.clone(), Default::default())?; let source: DispatchSource = match Url::parse(&self.options.source) { - Ok(url) => { - HttpSource::new( - url, - Fetcher::new(FetcherOptions::default()).await?, - HttpOptions::new().since(*since).keys(self.options.keys.clone()), - ) + Ok(url) => HttpSource { + url, + fetcher: Fetcher::new(FetcherOptions::default()).await?, + options: HttpOptions { + keys: self.options.keys.clone(), + since: *since, + }, } .into(), Err(_) => FileSource::new(&self.options.source, None)?.into(), @@ -68,22 +69,28 @@ impl Scanner { let sender = sender::HttpSender::new( self.options.provider.clone(), - HttpSenderOptions::default() - .additional_root_certificates(self.options.additional_root_certificates.clone()), + sender::Options { + additional_root_certificates: self.options.additional_root_certificates.clone(), + ..sender::Options::default() + }, ) .await?; - let storage = walker_extras::visitors::SendVisitor::new(self.options.target.clone(), sender) - .retries(self.options.retries) - .retry_delay(self.options.retry_delay.unwrap_or_default()); + let storage = walker_extras::visitors::SendVisitor { + url: self.options.target.clone(), + sender, + retries: self.options.retries, + retry_delay: self.options.retry_delay, + }; let process = ProcessVisitor { enabled: self.options.fix_licenses, next: storage, }; - let validation = ValidationVisitor::new(process) - .with_options(ValidationOptions::new().validation_date(self.options.validation_date)); + let validation = ValidationVisitor::new(process).with_options(ValidationOptions { + validation_date: self.options.validation_date, + }); let walker = Walker::new(source.clone()); walker.walk(RetrievingVisitor::new(source.clone(), validation)).await?; diff --git a/vexination/walker/Cargo.toml b/vexination/walker/Cargo.toml index f8420df13..cc1016d90 100644 --- a/vexination/walker/Cargo.toml +++ b/vexination/walker/Cargo.toml @@ -14,8 +14,8 @@ trustification-auth = { path = "../../auth" } clap = { version = "4", features = ["derive"] } anyhow = "1" csaf = "0.5.0" -csaf-walker = { version = "=0.6.0-alpha.7", default-features = false, features = ["crypto-openssl", "csaf"] } -walker-common = { version = "=0.6.0-alpha.7", features = [] } +csaf-walker = { version = "=0.6.0-alpha.6", default-features = false, features = ["crypto-openssl"] } +walker-common = { version = "=0.6.0-alpha.6", features = [] } url = { version = "2.3.1", features = ["serde"] } time = "0.3.21" humantime = "2.1.0" diff --git a/vexination/walker/src/lib.rs b/vexination/walker/src/lib.rs index 9df178e86..c06460ec7 100644 --- a/vexination/walker/src/lib.rs +++ b/vexination/walker/src/lib.rs @@ -56,14 +56,6 @@ pub struct Run { /// Only upload if a document's name has any of these prefixes. #[arg(long = "require-prefix")] pub required_prefixes: Vec, - - /// Path of the HTML output file - #[arg(long, default_value = "report.html")] - pub output: PathBuf, - - /// Make links relative to this URL. - #[arg(short = 'B', long)] - pub base_url: Option, } impl Run { @@ -88,7 +80,7 @@ impl Run { log::debug!("Policy date: {validation_date:?}"); - let options = ValidationOptions::new().validation_date(validation_date); + let options = ValidationOptions { validation_date }; server::run( self.workers, @@ -96,8 +88,6 @@ impl Run { self.sink, provider, options, - self.output, - self.base_url, self.ignore_distributions, self.since_file, self.additional_root_certificates, diff --git a/vexination/walker/src/server.rs b/vexination/walker/src/server.rs index 641816b27..c6d196672 100644 --- a/vexination/walker/src/server.rs +++ b/vexination/walker/src/server.rs @@ -1,25 +1,15 @@ -use std::collections::BTreeMap; -use std::path::PathBuf; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::SystemTime; - -use csaf_walker::discover::AsDiscovered; -use csaf_walker::model::metadata::Distribution; -use csaf_walker::report::{render_to_html, DocumentKey, Duplicates, ReportRenderOption, ReportResult}; -use csaf_walker::visitors::duplicates::DetectDuplicatesVisitor; -use csaf_walker::visitors::filter::{FilterConfig, FilteringVisitor}; use csaf_walker::{ - retrieve::RetrievingVisitor, + discover::DiscoveredAdvisory, + retrieve::{RetrievedAdvisory, RetrievingVisitor}, source::{FileSource, HttpSource}, validation::{ValidatedAdvisory, ValidationError, ValidationVisitor}, - verification::{ - check::{init_verifying_visitor, CheckError}, - VerificationError, VerifiedAdvisory, VerifyingVisitor, - }, + visitors::filter::{FilterConfig, FilteringVisitor}, walker::Walker, }; use reqwest::{header, StatusCode}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::SystemTime; use trustification_auth::client::{TokenInjector, TokenProvider}; use url::Url; use walker_common::{fetcher::Fetcher, since::Since, utils::url::Urlify, validate::ValidationOptions}; @@ -31,8 +21,6 @@ pub async fn run( sink: Url, provider: Arc, options: ValidationOptions, - output: PathBuf, - base_url: Option, ignore_distributions: Vec, since_file: Option, additional_root_certificates: Vec, @@ -46,166 +34,103 @@ pub async fn run( client = client.add_root_certificate(reqwest::tls::Certificate::from_pem(&pem)?); } - let total = Arc::new(AtomicUsize::default()); - let duplicates: Arc> = Default::default(); - let errors: Arc>> = Default::default(); - let warnings: Arc>>> = Default::default(); - - { - let client = Arc::new(client.build()?); - let total = total.clone(); - let duplicates = duplicates.clone(); - let errors = errors.clone(); - let warnings = warnings.clone(); - - let visitor = move |advisory: Result< - VerifiedAdvisory, - VerificationError, - >| { - (*total).fetch_add(1, Ordering::Release); - - let errors = errors.clone(); - let warnings = warnings.clone(); - let client = client.clone(); - let sink = sink.clone(); - let provider = provider.clone(); - async move { - let adv = match advisory { - Ok(adv) => { - let sink = sink.clone(); - let name = adv - .url - .path_segments() - .and_then(|s| s.last()) - .unwrap_or_else(|| adv.url.path()); - - match serde_json::to_string(&adv.csaf.clone()) { - Ok(b) => { - const MAX_RETRIES: usize = 10; - for retry in 0..MAX_RETRIES { - match client - .post(sink.clone()) - .header(header::CONTENT_TYPE, "application/json") - .body(b.clone()) - .inject_token(&provider) - .await? - .send() - .await - { - Ok(r) if r.status() == StatusCode::CREATED => { - log::info!("VEX ({}) stored successfully", &adv.csaf.document.tracking.id); - } - Ok(r) => { - log::warn!( - "(Skipped) {name}: Server's Error when storing VEX: {}, and wait to try again", - r.status() - ); - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - if retry == MAX_RETRIES - 1 { - log::warn!("(Skipped) {name}: Error storing VEX: {}", r.status()); - } - } - Err(e) => { - log::warn!("(Skipped) {name}: Client's Error when storing VEX: {e:?}"); - let _ = retry == MAX_RETRIES - 1; - } - }; - } + let client = Arc::new(client.build()?); + + let validation = ValidationVisitor::new(|advisory: Result| { + let sink = sink.clone(); + let provider = provider.clone(); + let client = client.clone(); + async move { + match advisory { + Ok(ValidatedAdvisory { + retrieved: + RetrievedAdvisory { + data, + discovered: DiscoveredAdvisory { url, .. }, + .. + }, + }) => { + let name = url.path_segments().and_then(|s| s.last()).unwrap_or_else(|| url.path()); + match serde_json::from_slice::(&data) { + Ok(doc) => match client + .post(sink) + .header(header::CONTENT_TYPE, "application/json") + .body(data.clone()) + .inject_token(&provider) + .await? + .send() + .await + { + Ok(r) if r.status() == StatusCode::CREATED => { + log::info!( + "VEX ({}) of size {} stored successfully", + doc.document.tracking.id, + &data[..].len() + ); } - Err(parse_err) => { - log::warn!("(Skipped) {name}: Serialization Error when storing VEX: {parse_err:?}"); + Ok(r) => { + log::warn!("(Skipped) {name}: Error storing VEX: {}", r.status()); } - }; - adv - } - Err(err) => { - let name = match err.as_discovered().relative_base_and_url() { - Some((base, relative)) => DocumentKey { - distribution_url: base.clone(), - url: relative, - }, - None => DocumentKey { - distribution_url: err.url().clone(), - url: Default::default(), - }, - }; - - errors.lock().unwrap().insert(name, err.to_string()); - return Ok::<_, anyhow::Error>(()); + Err(e) => { + log::warn!("(Skipped) {name}: Error storing VEX: {e:?}"); + } + }, + Err(e) => { + log::warn!("(Ignored) {name}: Error parsing advisory to retrieve ID: {e:?}"); + } } - }; - - if !adv.failures.is_empty() { - let name = DocumentKey::for_document(&adv); - warnings - .lock() - .unwrap() - .entry(name) - .or_default() - .extend(adv.failures.into_values().flatten()); } - - Ok::<_, anyhow::Error>(()) + Err(e) => { + log::warn!("Ignoring advisory {}: {:?}", e.url(), e); + } } - }; - let visitor = VerifyingVisitor::with_checks(visitor, init_verifying_visitor()); - let visitor = ValidationVisitor::new(visitor).with_options(options); - - let config = FilterConfig::new() - .ignored_distributions(None) - .ignored_prefixes(vec![]) - .only_prefixes(only_prefixes); - - if let Ok(url) = Url::parse(&source) { - let since = Since::new(None::, since_file, Default::default())?; - - log::info!("Walking VEX docs: source='{source}' workers={workers}"); - let source = HttpSource::new(url, fetcher, csaf_walker::source::HttpOptions::new().since(since.since)); - - let visitor = { RetrievingVisitor::new(source.clone(), visitor) }; - let visitor = DetectDuplicatesVisitor { visitor, duplicates }; - let visitor = FilteringVisitor { visitor, config }; - Walker::new(source.clone()) - .with_distribution_filter(Box::new(move |distribution: &Distribution| { - !ignore_distributions.contains(&distribution.directory_url) - })) - .walk_parallel(workers, visitor) - .await?; - - since.store()?; - } else { - log::info!("Walking VEX docs: path='{source}' workers={workers}"); - let source = FileSource::new(source, None)?; - let visitor = { RetrievingVisitor::new(source.clone(), visitor) }; - let visitor = DetectDuplicatesVisitor { visitor, duplicates }; - let visitor = FilteringVisitor { visitor, config }; - - Walker::new(source.clone()) - .with_distribution_filter(Box::new(move |distribution: &Distribution| { - !ignore_distributions.contains(&distribution.directory_url) - })) - .walk(visitor) - .await?; + Ok::<_, anyhow::Error>(()) } + }) + .with_options(options); + + let config = FilterConfig { + // we use the walker level filtering for this + ignored_distributions: Default::default(), + ignored_prefixes: vec![], + only_prefixes, + }; + + if let Ok(url) = Url::parse(&source) { + let since = Since::new(None::, since_file, Default::default())?; + log::info!("Walking VEX docs: source='{source}' workers={workers}"); + let source = HttpSource { + url, + fetcher, + options: csaf_walker::source::HttpOptions { since: *since }, + }; + Walker::new(source.clone()) + .with_distribution_filter(Box::new(move |distribution| { + !ignore_distributions.contains(&distribution.directory_url) + })) + .walk_parallel( + workers, + FilteringVisitor { + config, + visitor: RetrievingVisitor::new(source.clone(), validation), + }, + ) + .await?; + + since.store()?; + } else { + log::info!("Walking VEX docs: path='{source}' workers={workers}"); + let source = FileSource::new(source, None)?; + Walker::new(source.clone()) + .with_distribution_filter(Box::new(move |distribution| { + !ignore_distributions.contains(&distribution.directory_url) + })) + .walk(FilteringVisitor { + config, + visitor: RetrievingVisitor::new(source.clone(), validation), + }) + .await?; } - let total = (*total).load(Ordering::Acquire); - render( - output, - base_url, - ReportResult { - total, - duplicates: &duplicates.lock().unwrap(), - errors: &errors.lock().unwrap(), - warnings: &warnings.lock().unwrap(), - }, - )?; - Ok(()) -} - -fn render(output: PathBuf, base_url: Option, report: ReportResult) -> anyhow::Result<()> { - let mut out = std::fs::File::create(&output)?; - render_to_html(&mut out, &report, ReportRenderOption { output, base_url })?; - Ok(()) }