From 40feae303459874fca5f9b1e46f3a5af5b27a594 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Fri, 16 Feb 2024 14:47:40 +0100 Subject: [PATCH 1/9] Change: make async the methods for preference handling. Also, improve the creating the RedisHelper, since at this point the nvti cache and a main kb for the scan should have been created. Usage example. (Needs a running redis instance with an up-to-date nvticache) ``` ```use std::sync::{Arc, Mutex}; use models::{AliveTestMethods, Credential, PortRange, Port, Scan, Parameter}; use openvas::{pref_handler::PreferenceHandler, openvas_redis::RedisHelper}; use redis_storage::{NameSpaceSelector, RedisCtx}; async fn main() -> Result<(), std::fmt::Error> { // Create an scan config let mut scan = Scan::default(); scan.scan_id = Some("123-456".to_string()); scan.target.alive_test_methods = vec![AliveTestMethods::Icmp, AliveTestMethods::TcpSyn]; scan.target.credentials = vec![Credential { service: models::Service::SSH, port: Some(22), credential_type: models::CredentialType::UP { username: "user".to_string(), password: "pass".to_string(), privilege_credential: None, }, }]; scan.vts.push(models::VT { oid: "1.3.6.1.4.1.25623.1.0.112771".to_string(), parameters: vec![Parameter { id: 1, value: "llala".to_string() }] }); scan.target.excluded_hosts = vec!["127.0.0.1".to_string()]; scan.target.hosts = vec!["127.0.0.2".to_string()]; scan.target.ports = vec![Port { protocol: Some(models::Protocol::TCP), range: vec![ PortRange { start: 22, end: Some(25), }, PortRange { start: 80, end: None, }, ], }]; let redis_url = "unix:///run/redis-openvas/redis.sock"; // In this example, a fix db is taken,but the next free can be taken, using the name space selector NameSpaceSelector::Free let mut rctx = RedisCtx::open(redis_url, &[NameSpaceSelector::Fix(6)]).unwrap(); rctx.delete_namespace().unwrap(); // Create a redis connection to the nvticache, and a connection to the mainkb(). let cache = RedisCtx::open(redis_url, &[NameSpaceSelector::Key("nvticache")]).unwrap(); // New redis helper with access to the main kb for storing the preferences, and access to the nvticache // for getting info to build preferences. let rc = RedisHelper::::init(Arc::new(Mutex::new(cache)),Arc::new(Mutex::new(rctx))).unwrap(); let mut p = PreferenceHandler::new(scan, rc); Ok(p.prepare_preferences_for_openvas().await.expect("aaa")) } --- rust/Cargo.lock | 1 + rust/openvasctl/Cargo.toml | 1 + rust/openvasctl/src/openvas_redis.rs | 14 ++--- rust/openvasctl/src/pref_handler.rs | 90 ++++++++++++++-------------- 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2731bef11..77ef92261 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1972,6 +1972,7 @@ dependencies = [ "redis", "redis-storage", "storage", + "tokio", "tracing", ] diff --git a/rust/openvasctl/Cargo.toml b/rust/openvasctl/Cargo.toml index a9b2f927d..6472069b2 100644 --- a/rust/openvasctl/Cargo.toml +++ b/rust/openvasctl/Cargo.toml @@ -11,4 +11,5 @@ models = { path = "../models" } redis = "0.22.0" redis-storage = { version = "0.1.0", path = "../redis-storage" } storage = { version = "0.1.0", path = "../storage" } +tokio = { version = "1.36.0", features = ["full"] } tracing = "0.1.40" diff --git a/rust/openvasctl/src/openvas_redis.rs b/rust/openvasctl/src/openvas_redis.rs index 1009c335e..3fc890fe8 100644 --- a/rust/openvasctl/src/openvas_redis.rs +++ b/rust/openvasctl/src/openvas_redis.rs @@ -4,7 +4,7 @@ use redis_storage::{ dberror::{DbError, RedisStorageResult}, - NameSpaceSelector, RedisCtx, RedisGetNvt, RedisWrapper, + RedisCtx, RedisGetNvt, RedisWrapper, }; use std::{ collections::HashMap, @@ -28,16 +28,12 @@ where /// Initialize a RedisHelper struct with the connection to access the NVT cache /// and a empty task knowledge base to store the scan configuration to be sent to openvas. pub fn init( - redis_url: &str, - selector: &[NameSpaceSelector], + nvti_cache: Arc>, + kb_cache: Arc>, ) -> RedisStorageResult> { - let mut rctx = RedisCtx::open(redis_url, selector)?; - rctx.delete_namespace()?; - let cache = RedisCtx::open(redis_url, selector)?; - Ok(RedisHelper:: { - cache: Arc::new(Mutex::new(cache)), - task_kb: Arc::new(Mutex::new(rctx)), + cache: nvti_cache, + task_kb: kb_cache, }) } } diff --git a/rust/openvasctl/src/pref_handler.rs b/rust/openvasctl/src/pref_handler.rs index 1418e45d3..ea4ce5550 100644 --- a/rust/openvasctl/src/pref_handler.rs +++ b/rust/openvasctl/src/pref_handler.rs @@ -45,26 +45,26 @@ where } } - pub fn prepare_preferences_for_openvas(&mut self) -> RedisStorageResult<()> { - self.prepare_scan_id_for_openvas()?; - self.prepare_target_for_openvas()?; - self.prepare_ports_for_openvas()?; - self.prepare_credentials_for_openvas()?; - self.prepare_plugins_for_openvas()?; - self.prepare_main_kbindex_for_openvas()?; - self.prepare_host_options_for_openvas()?; - self.prepare_scan_params_for_openvas()?; - self.prepare_reverse_lookup_opt_for_openvas()?; - self.prepare_alive_test_option_for_openvas()?; + pub async fn prepare_preferences_for_openvas(&mut self) -> RedisStorageResult<()> { + self.prepare_scan_id_for_openvas().await?; + self.prepare_target_for_openvas().await?; + self.prepare_ports_for_openvas().await?; + self.prepare_credentials_for_openvas().await?; + self.prepare_plugins_for_openvas().await?; + self.prepare_main_kbindex_for_openvas().await?; + self.prepare_host_options_for_openvas().await?; + self.prepare_scan_params_for_openvas().await?; + self.prepare_reverse_lookup_opt_for_openvas().await?; + self.prepare_alive_test_option_for_openvas().await?; // VT preferences are stored after all preferences have been processed, // since alive tests preferences have to be able to overwrite default // preferences of ping_host.nasl for the classic method. - self.prepare_nvt_preferences()?; - self.prepare_boreas_alive_test() + self.prepare_nvt_preferences().await?; + self.prepare_boreas_alive_test().await } - fn prepare_main_kbindex_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_main_kbindex_for_openvas(&mut self) -> RedisStorageResult<()> { self.redis_connector.push_kb_item( format!( "internal/{}/scanprefs", @@ -76,7 +76,7 @@ where Ok(()) } - fn prepare_scan_id_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_scan_id_for_openvas(&mut self) -> RedisStorageResult<()> { self.redis_connector.push_kb_item( format!( "internal/{}", @@ -93,7 +93,7 @@ where Ok(()) } - fn process_vts(&self, vts: &Vec) -> (Vec, HashMap) { + async fn process_vts(&self, vts: &Vec) -> (Vec, HashMap) { let mut vts_list: Vec = vec![]; let mut pref_list: HashMap = HashMap::new(); @@ -103,11 +103,12 @@ where vts_list.push(vt.oid.clone()); // prepare vt preferences - for pref in vt.parameters.iter() { + for pref in vt.parameters.clone() { let (prefid, class, name, value): (String, String, String, String) = nvt .preferences - .get(pref.id as usize) - .expect("Valid pref id") + .iter() + .find(|p| p.id.unwrap() as u16 == pref.id) + .unwrap() .into(); let value_aux: String = if class == *"checkbox" { @@ -126,18 +127,17 @@ where continue; } } - (vts_list, pref_list) } - fn prepare_plugins_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_plugins_for_openvas(&mut self) -> RedisStorageResult<()> { let nvts = &self.scan_config.vts; if nvts.is_empty() { return Ok(()); } - let (nvts, prefs) = self.process_vts(nvts); + let (nvts, prefs) = self.process_vts(nvts).await; // update list of preferences self.nvt_params.extend(prefs); @@ -152,14 +152,16 @@ where ) } - fn prepare_nvt_preferences(&mut self) -> RedisStorageResult<()> { + async fn prepare_nvt_preferences(&mut self) -> RedisStorageResult<()> { let mut items: Vec = vec![]; + for (k, v) in self.nvt_params.iter() { + items.push(format!("{}|||{}", k, v)) + } + + if items.is_empty() { + return Ok(()); + } - let _ = self - .nvt_params - .clone() - .into_iter() - .map(|(k, v)| items.push(format!("{}|||{}", k, v))); self.redis_connector.push_kb_item( format!( "internal/{}/scanprefs", @@ -170,7 +172,7 @@ where ) } - fn prepare_alive_test_option_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_alive_test_option_for_openvas(&mut self) -> RedisStorageResult<()> { let mut prefs: HashMap = HashMap::new(); let mut alive_test = ALIVE_TEST_SCAN_CONFIG_DEFAULT; let mut value: &str = "no"; @@ -251,7 +253,7 @@ where Ok(()) } - fn prepare_boreas_alive_test(&mut self) -> RedisStorageResult<()> { + async fn prepare_boreas_alive_test(&mut self) -> RedisStorageResult<()> { // Check "test_alive_hosts_only" configuration from openvas.conf // If set no, boreas is disabled and alive_host.nasl is used instead. if let Ok(config) = cmd::read_openvas_config() { @@ -306,7 +308,7 @@ where Ok(()) } - fn prepare_reverse_lookup_opt_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_reverse_lookup_opt_for_openvas(&mut self) -> RedisStorageResult<()> { let mut lookup_opts: Vec = vec![]; if let Some(reverse_lookup_only) = self.scan_config.target.reverse_lookup_only { @@ -335,7 +337,7 @@ where ) } - fn prepare_target_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_target_for_openvas(&mut self) -> RedisStorageResult<()> { let target = self.scan_config.target.hosts.join(","); self.redis_connector.push_kb_item( format!( @@ -347,7 +349,7 @@ where ) } - fn prepare_ports_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_ports_for_openvas(&mut self) -> RedisStorageResult<()> { let ports = self.scan_config.target.ports.clone(); if let Some(ports) = ports_to_openvas_port_list(ports) { self.redis_connector.push_kb_item( @@ -363,7 +365,7 @@ where Ok(()) } - fn prepare_host_options_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_host_options_for_openvas(&mut self) -> RedisStorageResult<()> { let excluded_hosts = self.scan_config.target.excluded_hosts.join(","); if excluded_hosts.is_empty() { return Ok(()); @@ -379,7 +381,7 @@ where ) } - fn prepare_scan_params_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_scan_params_for_openvas(&mut self) -> RedisStorageResult<()> { let options = self .scan_config .scanner_preferences @@ -401,7 +403,7 @@ where options, ) } - fn prepare_credentials_for_openvas(&mut self) -> RedisStorageResult<()> { + async fn prepare_credentials_for_openvas(&mut self) -> RedisStorageResult<()> { let credentials = self.scan_config.target.credentials.clone(); let mut credential_preferences: Vec = vec![]; @@ -571,8 +573,8 @@ mod tests { use super::PreferenceHandler; use crate::openvas_redis::{FakeRedis, KbAccess}; - #[test] - fn test_prefs() { + #[tokio::test] + async fn test_prefs() { let mut scan = Scan::default(); scan.scan_id = Some("123-456".to_string()); scan.target.alive_test_methods = vec![AliveTestMethods::Icmp, AliveTestMethods::TcpSyn]; @@ -606,28 +608,28 @@ mod tests { let mut prefh = PreferenceHandler::new(scan, rc); assert_eq!(prefh.redis_connector.kb_id().unwrap(), 3); - assert!(prefh.prepare_scan_id_for_openvas().is_ok()); + assert!(prefh.prepare_scan_id_for_openvas().await.is_ok()); assert!(prefh .redis_connector .item_exists("internal/scanid", "123-456")); assert!(prefh.redis_connector.item_exists("internal/123-456", "new")); - assert!(prefh.prepare_main_kbindex_for_openvas().is_ok()); + assert!(prefh.prepare_main_kbindex_for_openvas().await.is_ok()); assert!(prefh .redis_connector .item_exists("internal/123-456/scanprefs", "ov_maindbid|||3")); - assert!(prefh.prepare_boreas_alive_test().is_ok()); + assert!(prefh.prepare_boreas_alive_test().await.is_ok()); assert!(prefh .redis_connector .item_exists("internal/123-456/scanprefs", "ALIVE_TEST|||3")); - assert!(prefh.prepare_host_options_for_openvas().is_ok()); + assert!(prefh.prepare_host_options_for_openvas().await.is_ok()); assert!(prefh .redis_connector .item_exists("internal/123-456/scanprefs", "excluded_hosts|||127.0.0.1")); - assert!(prefh.prepare_credentials_for_openvas().is_ok()); + assert!(prefh.prepare_credentials_for_openvas().await.is_ok()); assert!(prefh.redis_connector.item_exists( "internal/123-456/scanprefs", "1.3.6.1.4.1.25623.1.0.103591:3:password:SSH password (unsafe!):|||pass" @@ -637,7 +639,7 @@ mod tests { "1.3.6.1.4.1.25623.1.0.103591:1:entry:SSH login name:|||user" )); - assert!(prefh.prepare_ports_for_openvas().is_ok()); + assert!(prefh.prepare_ports_for_openvas().await.is_ok()); assert!(prefh .redis_connector .item_exists("internal/123-456/scanprefs", "PORTS|||tcp:22-25,80,")); From 0af3923dfa59f912a1914842bc3efe83603b113d Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Tue, 20 Feb 2024 13:19:15 +0100 Subject: [PATCH 2/9] Minor chages according with suggetions --- rust/openvasctl/src/openvas_redis.rs | 8 ++++---- rust/openvasctl/src/pref_handler.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/openvasctl/src/openvas_redis.rs b/rust/openvasctl/src/openvas_redis.rs index 3fc890fe8..1054a88ea 100644 --- a/rust/openvasctl/src/openvas_redis.rs +++ b/rust/openvasctl/src/openvas_redis.rs @@ -27,14 +27,14 @@ where { /// Initialize a RedisHelper struct with the connection to access the NVT cache /// and a empty task knowledge base to store the scan configuration to be sent to openvas. - pub fn init( + pub fn new( nvti_cache: Arc>, kb_cache: Arc>, - ) -> RedisStorageResult> { - Ok(RedisHelper:: { + ) -> RedisHelper { + RedisHelper:: { cache: nvti_cache, task_kb: kb_cache, - }) + } } } diff --git a/rust/openvasctl/src/pref_handler.rs b/rust/openvasctl/src/pref_handler.rs index ea4ce5550..6573dddee 100644 --- a/rust/openvasctl/src/pref_handler.rs +++ b/rust/openvasctl/src/pref_handler.rs @@ -103,7 +103,7 @@ where vts_list.push(vt.oid.clone()); // prepare vt preferences - for pref in vt.parameters.clone() { + for pref in &vt.parameters { let (prefid, class, name, value): (String, String, String, String) = nvt .preferences .iter() From e6738853f5f5a02013db2be2aac2bdaf54c3531f Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Fri, 16 Feb 2024 10:50:09 +0100 Subject: [PATCH 3/9] Add: result collector Collects results from redis and return a structure with a list of results and the amount of dead host and total amount of alive hosts --- rust/Cargo.lock | 1 + rust/openvasctl/Cargo.toml | 1 + rust/openvasctl/src/lib.rs | 1 + rust/openvasctl/src/openvas_redis.rs | 10 + rust/openvasctl/src/result_collector.rs | 242 ++++++++++++++++++++++++ rust/storage/src/item.rs | 25 ++- 6 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 rust/openvasctl/src/result_collector.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 77ef92261..a9d8d5dc6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1969,6 +1969,7 @@ version = "0.1.0" dependencies = [ "configparser", "models", + "osp", "redis", "redis-storage", "storage", diff --git a/rust/openvasctl/Cargo.toml b/rust/openvasctl/Cargo.toml index 6472069b2..2e5e37b42 100644 --- a/rust/openvasctl/Cargo.toml +++ b/rust/openvasctl/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] configparser = "3.0.4" models = { path = "../models" } +osp = { version = "0.1.0", path = "../osp" } redis = "0.22.0" redis-storage = { version = "0.1.0", path = "../redis-storage" } storage = { version = "0.1.0", path = "../storage" } diff --git a/rust/openvasctl/src/lib.rs b/rust/openvasctl/src/lib.rs index 72fb18bb5..7baa15021 100644 --- a/rust/openvasctl/src/lib.rs +++ b/rust/openvasctl/src/lib.rs @@ -7,3 +7,4 @@ pub mod ctl; pub mod error; pub mod openvas_redis; pub mod pref_handler; +pub mod result_collector; diff --git a/rust/openvasctl/src/openvas_redis.rs b/rust/openvasctl/src/openvas_redis.rs index 1054a88ea..0edb4843b 100644 --- a/rust/openvasctl/src/openvas_redis.rs +++ b/rust/openvasctl/src/openvas_redis.rs @@ -45,6 +45,9 @@ pub trait KbAccess { value: T, ) -> RedisStorageResult<()>; fn kb_id(&self) -> RedisStorageResult; + fn results(&mut self) -> RedisStorageResult> { + Ok(Vec::new()) + } } impl KbAccess for RedisHelper { @@ -67,6 +70,13 @@ impl KbAccess for RedisHelper { .map_err(|e| DbError::SystemError(format!("{e:?}")))?; Ok(cache.db) } + + fn results(&mut self) -> RedisStorageResult> { + let mut cache = Arc::as_ref(&self.cache) + .lock() + .map_err(|e| DbError::SystemError(format!("{e:?}")))?; + cache.lrange("internal/results", 0, -1) + } } pub trait VtHelper { diff --git a/rust/openvasctl/src/result_collector.rs b/rust/openvasctl/src/result_collector.rs new file mode 100644 index 000000000..182a66ae0 --- /dev/null +++ b/rust/openvasctl/src/result_collector.rs @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: 2024 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +/// This file contains structs and methods for retrieve scan information from redis +/// and store it into the given storage to be collected later for the clients. +use std::{ + str::FromStr, + sync::{Arc, Mutex}, +}; + +use crate::openvas_redis::{KbAccess, VtHelper}; +use osp::{ScanResult, StringF32}; +use redis_storage::dberror::RedisStorageResult; + +/// Structure to hold the results retrieve from redis main kb +#[derive(Default, Debug)] +pub struct Results { + /// The list of results retrieve + results: Vec, + /// The number of new dead hosts found during this retrieve. New dead hosts can be found + /// during the scan + new_dead: i64, + /// Total amount of alive hosts found. This is sent once for scan, as it is the + /// the alive host found by Boreas at the start of the scan. + count_total: i64, +} + +pub struct ResultHelper { + redis_connector: H, + results: Arc>, +} + +impl ResultHelper +where + H: KbAccess + VtHelper, +{ + pub fn init(redis_connector: H) -> Self { + Self { + redis_connector, + results: Arc::new(Mutex::new(Results::default())), + } + } + + fn process_results(&self, results: Vec) -> RedisStorageResult { + let mut new_dead = 0; + let mut count_total = 0; + let mut scan_results: Vec = Vec::new(); + for result in results.iter() { + //result_type|||host ip|||hostname|||port|||OID|||value[|||uri] + let res_fields: Vec<&str> = result.split("|||").collect(); + + let result_type = res_fields[0].trim().to_owned(); + let host_ip = res_fields[1].trim().to_owned(); + let host_name = res_fields[2].trim().to_owned(); + let port = res_fields[3].trim().to_owned(); + let oid = res_fields[4].trim().to_owned(); + let value = res_fields[5].trim().to_owned(); + let uri = { + if res_fields.len() > 6 { + Some(res_fields[6].trim().to_owned()) + } else { + None + } + }; + + let roid = oid.trim(); + + let current_host = if !host_ip.is_empty() { + host_ip + } else { + String::new() + }; + + let host_is_dead = value.contains("Host dead") || result_type == "DEADHOST"; + + let host_deny = value.contains("Host access denied"); + let start_end_msg = result_type == "HOST_START" || result_type == "HOST_END"; + let host_count = result_type == "HOST_COUNT"; + let error_msg = result_type == "ERRMSG"; + + // TODO: do we need the URI? + let _uri = if let Some(uri) = uri { + uri + } else { + "".to_string() + }; + + let mut rname = String::new(); + if !host_is_dead && !host_deny && !start_end_msg && !host_count { + if roid.is_empty() && !error_msg { + tracing::warn!("Missing VT oid for a result"); + }; + + let vt_aux = self.redis_connector.get_vt(roid)?; + match vt_aux { + None => tracing::warn!("Invalid oid"), + Some(vt) => { + rname = vt.name; + } + }; + } + + if error_msg { + scan_results.push(ScanResult { + result_type: osp::ResultType::Error, + host: current_host, + hostname: host_name, + port, + test_id: roid.to_string(), + description: value, + severity: StringF32::from(0.0), + name: rname, + }); + } else if start_end_msg || result_type == "LOG" { + scan_results.push(ScanResult { + result_type: osp::ResultType::Log, + host: current_host, + hostname: host_name, + port, + test_id: roid.to_string(), + description: value, + severity: StringF32::from(0.0), + name: rname, + }); + } else if result_type == "ALARM" { + scan_results.push(ScanResult { + result_type: osp::ResultType::Alarm, + host: current_host, + hostname: host_name, + port, + test_id: roid.to_string(), + description: value, + severity: StringF32::from(0.0), + name: rname, + }); + } else if result_type == "DEADHOST" { + println!("AAAAAAAAAAAAAAAAAAAAAAA"); + new_dead += i64::from_str(&value).expect("Valid number of dead hosts"); + } else if host_count { + count_total = i64::from_str(&value).expect("Valid number of dead hosts"); + } + } + + Ok(Results { + results: scan_results, + new_dead, + count_total, + }) + } + + pub async fn results(&mut self) -> RedisStorageResult<()> { + if let Ok(results) = self.redis_connector.results() { + if let Ok(mut res) = Arc::as_ref(&self.results).lock() { + if let Ok(res_updates) = self.process_results(results) { + res.count_total = res_updates.count_total; + res.new_dead = res_updates.new_dead; + res.results.extend(res_updates.results); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use crate::openvas_redis::FakeRedis; + use models::Result; + use std::collections::HashMap; + + use super::ResultHelper; + #[test] + fn test_results() { + let results = vec![ + "LOG|||127.0.0.1||| localhost ||||||||| HOST_START".to_string(), + "ERRMSG|||127.0.0.1||| localhost ||||||1.2.3.4.5.6||| NVT timeout".to_string(), + "ALARM|||127.0.0.1||| example.com |||22/tcp|||12.11.10.9.8.7||| Something wrong|||/var/lib/lib1.jar".to_string(), + "DEADHOST||| ||| ||| ||| |||3".to_string(), + "HOST_COUNT||| ||| ||| ||| |||12".to_string(), + "DEADHOST||| ||| ||| ||| |||1".to_string(), + + ]; + + let rc = FakeRedis { + data: HashMap::new(), + }; + + let resh = ResultHelper::init(rc); + + let res_updates = resh.process_results(results).unwrap(); + + let single_r = Result { + id: 0, + r_type: models::ResultType::Log, + ip_address: Some("127.0.0.1".to_string()), + hostname: Some("localhost".to_string()), + oid: Some("".to_string()), + port: None, + protocol: None, + message: Some("HOST_START".to_string()), + detail: None, + }; + + let b = res_updates.results.get(0).unwrap(); + assert_eq!(models::Result::from(b), single_r); + + let single_r = Result { + id: 0, + r_type: models::ResultType::Error, + ip_address: Some("127.0.0.1".to_string()), + hostname: Some("localhost".to_string()), + oid: Some("1.2.3.4.5.6".to_string()), + port: None, + protocol: None, + message: Some("NVT timeout".to_string()), + detail: None, + }; + + let b = res_updates.results.get(1).unwrap(); + assert_eq!(models::Result::from(b), single_r); + + let single_r = Result { + id: 0, + r_type: models::ResultType::Alarm, + ip_address: Some("127.0.0.1".to_string()), + hostname: Some("example.com".to_string()), + oid: Some("12.11.10.9.8.7".to_string()), + port: Some(i16::from(22i16)), + protocol: Some(models::Protocol::TCP), + message: Some("Something wrong".to_string()), + detail: None, + }; + + let b = res_updates.results.get(2).unwrap(); + assert_eq!(models::Result::from(b), single_r); + + assert_eq!(res_updates.new_dead, 4); + assert_eq!(res_updates.count_total, 12); + } +} diff --git a/rust/storage/src/item.rs b/rust/storage/src/item.rs index 28c23fd5f..6b6997beb 100644 --- a/rust/storage/src/item.rs +++ b/rust/storage/src/item.rs @@ -196,7 +196,8 @@ make_str_lookup_enum! { remote_banner_unreliable => RemoteBannerUnreliable, remote_probe => RemoteProbe, remote_vul => RemoteVul, - package_unreliable => PackageUnreliable + package_unreliable => PackageUnreliable, + default => Default } } @@ -406,6 +407,28 @@ impl From<&NvtPreference> for (String, String, String, String) { } } +impl From for i64 { + fn from(v: QodType) -> Self { + match v { + QodType::Exploit => 100, + QodType::RemoteVul => 99, + QodType::RemoteApp => 98, + QodType::Package => 97, + QodType::Registry => 97, + QodType::RemoteActive => 95, + QodType::RemoteBanner => 80, + QodType::ExecutableVersion => 80, + QodType::RemoteAnalysis => 70, + QodType::RemoteProbe => 50, + QodType::PackageUnreliable => 30, + QodType::RemoteBannerUnreliable => 30, + QodType::ExecutableVersionUnreliable => 30, + QodType::GeneralNote => 1, + QodType::Default => 70, + } + } +} + /// TagValue is a type containing value types of script_tag pub type TagValue = types::Primitive; From 47e04304dfc0696ee2e7f5b17738c9114507127d Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Wed, 21 Feb 2024 13:35:18 +0100 Subject: [PATCH 4/9] Add: pop method via a pipeline to get results and status --- rust/redis-storage/src/connector.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rust/redis-storage/src/connector.rs b/rust/redis-storage/src/connector.rs index 0709a24f3..0d47254b7 100644 --- a/rust/redis-storage/src/connector.rs +++ b/rust/redis-storage/src/connector.rs @@ -197,6 +197,7 @@ pub trait RedisWrapper { fn lindex(&mut self, key: &str, index: isize) -> RedisStorageResult; fn lrange(&mut self, key: &str, start: isize, end: isize) -> RedisStorageResult>; fn keys(&mut self, pattern: &str) -> RedisStorageResult>; + fn pop(&mut self, pattern: &str) -> RedisStorageResult>; } impl RedisWrapper for RedisCtx { @@ -252,6 +253,14 @@ impl RedisWrapper for RedisCtx { .keys(pattern)?; Ok(ret) } + + fn pop(&mut self, key: &str) -> RedisStorageResult> { + let ret: (Vec,) = redis::pipe() + .cmd("LRANGE").arg(key).arg("0").arg("-1") + .cmd("DEL").arg("internal/results").ignore() + .query(&mut self.kb.as_mut().unwrap()).unwrap(); + Ok(ret.0) + } } pub trait RedisAddAdvisory: RedisWrapper { @@ -772,6 +781,9 @@ mod tests { fn keys(&mut self, _: &str) -> crate::dberror::RedisStorageResult> { Ok(Vec::new()) } + fn pop(&mut self, _: &str) -> crate::dberror::RedisStorageResult> { + Ok(Vec::new()) + } fn lrange( &mut self, From 606101526e8e7de352d50676636c02660249ac83 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Wed, 21 Feb 2024 15:26:15 +0100 Subject: [PATCH 5/9] Add: host status --- rust/openvasctl/src/openvas_redis.rs | 14 ++++++- rust/openvasctl/src/result_collector.rs | 52 +++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/rust/openvasctl/src/openvas_redis.rs b/rust/openvasctl/src/openvas_redis.rs index 0edb4843b..beb637479 100644 --- a/rust/openvasctl/src/openvas_redis.rs +++ b/rust/openvasctl/src/openvas_redis.rs @@ -48,6 +48,9 @@ pub trait KbAccess { fn results(&mut self) -> RedisStorageResult> { Ok(Vec::new()) } + fn status(&mut self) -> RedisStorageResult> { + Ok(Vec::new()) + } } impl KbAccess for RedisHelper { @@ -72,10 +75,17 @@ impl KbAccess for RedisHelper { } fn results(&mut self) -> RedisStorageResult> { - let mut cache = Arc::as_ref(&self.cache) + let mut kb = Arc::as_ref(&self.task_kb) + .lock() + .map_err(|e| DbError::SystemError(format!("{e:?}")))?; + kb.pop("internal/results") + } + + fn status(&mut self) -> RedisStorageResult> { + let mut kb = Arc::as_ref(&self.task_kb) .lock() .map_err(|e| DbError::SystemError(format!("{e:?}")))?; - cache.lrange("internal/results", 0, -1) + kb.pop("internal/status") } } diff --git a/rust/openvasctl/src/result_collector.rs b/rust/openvasctl/src/result_collector.rs index 182a66ae0..8d89ffb89 100644 --- a/rust/openvasctl/src/result_collector.rs +++ b/rust/openvasctl/src/result_collector.rs @@ -5,6 +5,7 @@ /// This file contains structs and methods for retrieve scan information from redis /// and store it into the given storage to be collected later for the clients. use std::{ + collections::HashMap, str::FromStr, sync::{Arc, Mutex}, }; @@ -27,8 +28,9 @@ pub struct Results { } pub struct ResultHelper { - redis_connector: H, - results: Arc>, + pub redis_connector: H, + pub results: Arc>, + pub status: Arc>>, } impl ResultHelper @@ -39,6 +41,7 @@ where Self { redis_connector, results: Arc::new(Mutex::new(Results::default())), + status: Arc::new(Mutex::new(HashMap::new())), } } @@ -135,7 +138,6 @@ where name: rname, }); } else if result_type == "DEADHOST" { - println!("AAAAAAAAAAAAAAAAAAAAAAA"); new_dead += i64::from_str(&value).expect("Valid number of dead hosts"); } else if host_count { count_total = i64::from_str(&value).expect("Valid number of dead hosts"); @@ -161,6 +163,49 @@ where } Ok(()) } + + fn process_status(&self, status: Vec) -> RedisStorageResult> { + enum ScanProgress { + DeadHost = -1, + } + + let mut all_hosts: HashMap = HashMap::new(); + + for res in status { + let mut fields = res.splitn(3, '/'); + let current_host = fields.next().expect("Valid status value"); + let launched = fields.next().expect("Valid status value"); + let total = fields.next().expect("Valid status value"); + + let host_progress: i64 = match i64::from_str(total) { + // No plugins + Ok(0) => { + continue; + } + // Host Dead + Ok(-1) => ScanProgress::DeadHost as i64, + Ok(n) => (i64::from_str(launched).expect("Integer") / n) * 100, + _ => { + continue; + } + }; + + all_hosts.insert(current_host.to_string(), host_progress); + tracing::debug!("Host {} has progress: {}", current_host, host_progress); + } + + Ok(HashMap::new()) + } + pub async fn status(&mut self) -> RedisStorageResult<()> { + if let Ok(status) = self.redis_connector.status() { + if let Ok(mut stat) = Arc::as_ref(&self.status).lock() { + if let Ok(res_updates) = self.process_status(status) { + stat.extend(res_updates); + } + } + } + Ok(()) + } } #[cfg(test)] @@ -180,7 +225,6 @@ mod tests { "DEADHOST||| ||| ||| ||| |||3".to_string(), "HOST_COUNT||| ||| ||| ||| |||12".to_string(), "DEADHOST||| ||| ||| ||| |||1".to_string(), - ]; let rc = FakeRedis { From c8c03371f0f49f27e1d9ef940bce7652270ac018 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Wed, 21 Feb 2024 16:22:56 +0100 Subject: [PATCH 6/9] fix format --- rust/redis-storage/src/connector.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rust/redis-storage/src/connector.rs b/rust/redis-storage/src/connector.rs index 0d47254b7..82a9ee2a7 100644 --- a/rust/redis-storage/src/connector.rs +++ b/rust/redis-storage/src/connector.rs @@ -253,12 +253,18 @@ impl RedisWrapper for RedisCtx { .keys(pattern)?; Ok(ret) } - + fn pop(&mut self, key: &str) -> RedisStorageResult> { let ret: (Vec,) = redis::pipe() - .cmd("LRANGE").arg(key).arg("0").arg("-1") - .cmd("DEL").arg("internal/results").ignore() - .query(&mut self.kb.as_mut().unwrap()).unwrap(); + .cmd("LRANGE") + .arg(key) + .arg("0") + .arg("-1") + .cmd("DEL") + .arg("internal/results") + .ignore() + .query(&mut self.kb.as_mut().unwrap()) + .unwrap(); Ok(ret.0) } } From eae623697cff0346860253d75f34ff06b7806439 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 22 Feb 2024 08:58:58 +0100 Subject: [PATCH 7/9] revert the pop result to maintain the order, since items are lpushed --- rust/redis-storage/src/connector.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/redis-storage/src/connector.rs b/rust/redis-storage/src/connector.rs index 82a9ee2a7..ca88fa11e 100644 --- a/rust/redis-storage/src/connector.rs +++ b/rust/redis-storage/src/connector.rs @@ -261,11 +261,15 @@ impl RedisWrapper for RedisCtx { .arg("0") .arg("-1") .cmd("DEL") - .arg("internal/results") + .arg(key) .ignore() .query(&mut self.kb.as_mut().unwrap()) .unwrap(); - Ok(ret.0) + // Since items are lpushed, the returned vector must be reversed to keep the order. + let mut status = ret.0; + status.reverse(); + + Ok(status) } } From 5be07b543dd8f43ee3487bbe1288817aba8f3808 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 22 Feb 2024 08:59:32 +0100 Subject: [PATCH 8/9] add test for process_status --- rust/openvasctl/src/result_collector.rs | 40 ++++++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/rust/openvasctl/src/result_collector.rs b/rust/openvasctl/src/result_collector.rs index 8d89ffb89..53127a7bf 100644 --- a/rust/openvasctl/src/result_collector.rs +++ b/rust/openvasctl/src/result_collector.rs @@ -30,7 +30,7 @@ pub struct Results { pub struct ResultHelper { pub redis_connector: H, pub results: Arc>, - pub status: Arc>>, + pub status: Arc>>, } impl ResultHelper @@ -164,27 +164,26 @@ where Ok(()) } - fn process_status(&self, status: Vec) -> RedisStorageResult> { + fn process_status(&self, status: Vec) -> RedisStorageResult> { enum ScanProgress { DeadHost = -1, } - let mut all_hosts: HashMap = HashMap::new(); - + let mut all_hosts: HashMap = HashMap::new(); for res in status { let mut fields = res.splitn(3, '/'); let current_host = fields.next().expect("Valid status value"); let launched = fields.next().expect("Valid status value"); let total = fields.next().expect("Valid status value"); - let host_progress: i64 = match i64::from_str(total) { + let host_progress: i32 = match i32::from_str(total) { // No plugins Ok(0) => { continue; } // Host Dead - Ok(-1) => ScanProgress::DeadHost as i64, - Ok(n) => (i64::from_str(launched).expect("Integer") / n) * 100, + Ok(-1) => ScanProgress::DeadHost as i32, + Ok(n) => ((f32::from_str(launched).expect("Integer") / n as f32) * 100.0) as i32, _ => { continue; } @@ -194,7 +193,7 @@ where tracing::debug!("Host {} has progress: {}", current_host, host_progress); } - Ok(HashMap::new()) + Ok(all_hosts) } pub async fn status(&mut self) -> RedisStorageResult<()> { if let Ok(status) = self.redis_connector.status() { @@ -283,4 +282,29 @@ mod tests { assert_eq!(res_updates.new_dead, 4); assert_eq!(res_updates.count_total, 12); } + + #[test] + fn test_status() { + let status = vec![ + "127.0.0.2/0/-1".to_string(), + "127.0.0.1/188/1000".to_string(), + "127.0.0.3/750/1000".to_string(), + "127.0.0.2/15/1000".to_string(), + "127.0.0.1/0/1000".to_string(), + ]; + + let rc = FakeRedis { + data: HashMap::new(), + }; + + let resh = ResultHelper::init(rc); + + let mut r = HashMap::new(); + r.insert("127.0.0.1".to_string(), 0); + r.insert("127.0.0.2".to_string(), 1); + r.insert("127.0.0.3".to_string(), 75); + + let res_updates = resh.process_status(status).unwrap(); + assert_eq!(res_updates, r) + } } From 85fecfc9eb25bdbcc6ffae39717c26b9d8d5e7a1 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Mon, 26 Feb 2024 09:45:25 +0100 Subject: [PATCH 9/9] consider excluded hosts message type sent by openvas --- rust/openvasctl/src/result_collector.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/rust/openvasctl/src/result_collector.rs b/rust/openvasctl/src/result_collector.rs index 53127a7bf..79e39a5ae 100644 --- a/rust/openvasctl/src/result_collector.rs +++ b/rust/openvasctl/src/result_collector.rs @@ -25,6 +25,8 @@ pub struct Results { /// Total amount of alive hosts found. This is sent once for scan, as it is the /// the alive host found by Boreas at the start of the scan. count_total: i64, + /// Total amount of excluded hosts. + count_excluded: i64, } pub struct ResultHelper { @@ -48,6 +50,8 @@ where fn process_results(&self, results: Vec) -> RedisStorageResult { let mut new_dead = 0; let mut count_total = 0; + let mut count_excluded = 0; + let mut scan_results: Vec = Vec::new(); for result in results.iter() { //result_type|||host ip|||hostname|||port|||OID|||value[|||uri] @@ -76,11 +80,11 @@ where }; let host_is_dead = value.contains("Host dead") || result_type == "DEADHOST"; - let host_deny = value.contains("Host access denied"); let start_end_msg = result_type == "HOST_START" || result_type == "HOST_END"; let host_count = result_type == "HOST_COUNT"; let error_msg = result_type == "ERRMSG"; + let excluded_hosts = result_type == "HOSTS_EXCLUDED"; // TODO: do we need the URI? let _uri = if let Some(uri) = uri { @@ -90,7 +94,7 @@ where }; let mut rname = String::new(); - if !host_is_dead && !host_deny && !start_end_msg && !host_count { + if !host_is_dead && !host_deny && !start_end_msg && !host_count && !excluded_hosts { if roid.is_empty() && !error_msg { tracing::warn!("Missing VT oid for a result"); }; @@ -138,9 +142,11 @@ where name: rname, }); } else if result_type == "DEADHOST" { - new_dead += i64::from_str(&value).expect("Valid number of dead hosts"); + new_dead += i64::from_str(&value).expect("Valid amount of dead hosts"); } else if host_count { - count_total = i64::from_str(&value).expect("Valid number of dead hosts"); + count_total = i64::from_str(&value).expect("Valid amount of dead hosts"); + } else if excluded_hosts { + count_excluded = i64::from_str(&value).expect("Valid amount of excluded hosts"); } } @@ -148,6 +154,7 @@ where results: scan_results, new_dead, count_total, + count_excluded, }) } @@ -157,6 +164,7 @@ where if let Ok(res_updates) = self.process_results(results) { res.count_total = res_updates.count_total; res.new_dead = res_updates.new_dead; + res.count_excluded = res_updates.count_excluded; res.results.extend(res_updates.results); } }