refactor: clean no use code

.devcontainer/.env

@@ -1,4 +1,4 @@
 POSTGRES_DB=watchvuln
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=123456
-DATABASE_URL=postgres://postgres:123456@db:5432/watchvuln
+POSTGRES_USER=watchvuln
+POSTGRES_PASSWORD=watchvuln
+DATABASE_URL=postgres://watchvuln:watchvuln@db:5432/watchvuln

src/grab/avd.rs

@@ -11,11 +11,12 @@ use crate::{
     utils::http_client::Help,
 };
 
-use super::{Grab, Provider};
+use super::Grab;
 
 const PAGE_REGEXP: &str = r"第 \d+ 页 / (\d+) 页 ";
 const CVEID_REGEXP: &str = r"^CVE-\d+-\d+$";
 
+#[derive(Default)]
 pub struct AVDCrawler {
     pub name: String,
     pub display_name: String,
@@ -37,25 +38,12 @@ impl Grab for AVDCrawler {
         }
         Ok(res)
     }
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
 
     fn get_name(&self) -> String {
         self.name.to_owned()
     }
 }
 
-impl Default for AVDCrawler {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 impl AVDCrawler {
     pub fn new() -> AVDCrawler {
         let headers = header::HeaderMap::new();
@@ -174,7 +162,8 @@ impl AVDCrawler {
     }
 
     fn get_references(&self, document: &Html) -> Result<Vec<String>> {
-        let reference_selector = Selector::parse("td[nowrap='nowrap'] a").unwrap();
+        let reference_selector = Selector::parse("td[nowrap='nowrap'] a")
+            .map_err(|err| eyre!("avd get references selector parse error {}", err))?;
         let references = document
             .select(&reference_selector)
             .filter_map(|el| el.attr("href"))
@@ -184,12 +173,12 @@ impl AVDCrawler {
     }
 
     fn get_solutions(&self, document: &Html) -> Result<String> {
-        let solutions_selector =
-            Selector::parse(".text-detail").map_err(|err| eyre!("parse html error {}", err))?;
+        let solutions_selector = Selector::parse(".text-detail")
+            .map_err(|err| eyre!("avd get solutions selector parse error {}", err))?;
         let solutions = document
             .select(&solutions_selector)
             .nth(1)
-            .unwrap()
+            .ok_or_else(|| Error::Message("avd solutions value not found".to_string()))?
             .text()
             .map(|el| el.trim())
             .collect::<Vec<_>>()
@@ -198,8 +187,8 @@ impl AVDCrawler {
     }
 
     fn get_description(&self, document: &Html) -> Result<String> {
-        let description_selector =
-            Selector::parse(".text-detail div").map_err(|err| eyre!("parse html error {}", err))?;
+        let description_selector = Selector::parse(".text-detail div")
+            .map_err(|err| eyre!("avd get description selector parse error {}", err))?;
         let description = document
             .select(&description_selector)
             .map(|e| e.text().collect::<String>())
@@ -210,23 +199,24 @@ impl AVDCrawler {
 
     fn get_title(&self, document: &Html) -> Result<String> {
         let title_selector = Selector::parse("h5[class='header__title'] .header__title__text")
-            .map_err(|err| eyre!("parse html error {}", err))?;
+            .map_err(|err| eyre!("avd get title selector parse error {}", err))?;
         let title = document
             .select(&title_selector)
             .nth(0)
-            .ok_or_else(|| eyre!("title value not found"))?
+            .ok_or_else(|| eyre!("avd title value not found"))?
             .inner_html()
             .trim()
             .to_string();
         Ok(title)
     }
 
     fn get_severity(&self, document: &Html) -> Result<Severity> {
-        let level_selector = Selector::parse("h5[class='header__title'] .badge").unwrap();
+        let level_selector = Selector::parse("h5[class='header__title'] .badge")
+            .map_err(|err| eyre!("avd get severity selector parse error {}", err))?;
         let level = document
             .select(&level_selector)
             .nth(0)
-            .ok_or_else(|| eyre!("level value not found"))?
+            .ok_or_else(|| eyre!("avd level value not found"))?
             .inner_html()
             .trim()
             .to_string();
@@ -246,7 +236,7 @@ impl AVDCrawler {
         let metric_value = document
             .select(&value_selector)
             .nth(index)
-            .ok_or_else(|| eyre!("metric value not found"))?
+            .ok_or_else(|| eyre!("avd metric value not found"))?
             .inner_html()
             .trim()
             .to_string();
@@ -256,7 +246,6 @@ impl AVDCrawler {
     fn get_cve_id(&self, document: &Html) -> Result<String> {
         let mut cve_id = self.get_mertric_value(document, 0)?;
         if !Regex::new(CVEID_REGEXP)?.is_match(&cve_id) {
-            // warn!("cve id not found in {}", href);
             cve_id = "".to_string();
         }
         Ok(cve_id)

src/grab/kev.rs

@@ -5,14 +5,15 @@ use chrono::{DateTime, FixedOffset};
 use reqwest::header::{self};
 use serde::{Deserialize, Serialize};
 
-use super::{Grab, Provider, VulnInfo};
+use super::{Grab, VulnInfo};
 use crate::error::Result;
 use crate::{grab::Severity, utils::http_client::Help};
 
 const KEV_URL: &str =
     "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
 const KEV_PAGE_SIZE: usize = 10;
 
+#[derive(Default)]
 pub struct KevCrawler {
     pub name: String,
     pub display_name: String,
@@ -62,25 +63,12 @@ impl Grab for KevCrawler {
         }
         Ok(res)
     }
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
 
     fn get_name(&self) -> String {
         self.name.to_owned()
     }
 }
 
-impl Default for KevCrawler {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 impl KevCrawler {
     pub fn new() -> KevCrawler {
         let headers = header::HeaderMap::new();

src/grab/mod.rs

@@ -87,17 +87,9 @@ impl fmt::Display for Severity {
     }
 }
 
-#[derive(Debug)]
-pub struct Provider {
-    pub name: String,
-    pub display_name: String,
-    pub link: String,
-}
-
 #[async_trait]
 pub trait Grab: Send + Sync {
     async fn get_update(&self, page_limit: i32) -> Result<Vec<VulnInfo>>;
-    fn get_provider(&self) -> Provider;
     fn get_name(&self) -> String;
 }
 

src/grab/oscs.rs

@@ -10,14 +10,15 @@ use crate::{
     utils::{http_client::Help, timestamp_to_date},
 };
 
-use super::{Grab, Provider};
+use super::Grab;
 
 const OSCS_PAGE_SIZE: i32 = 10;
 const OSCS_PAGE_DEFAULT: i32 = 1;
 const OSCS_PER_PAGE_DEFAULT: i32 = 10;
 const OSCS_LIST_URL: &str = "https://www.oscs1024.com/oscs/v1/intelligence/list";
 const OSCS_DETAIL_URL: &str = "https://www.oscs1024.com/oscs/v1/vdb/info";
 
+#[derive(Default)]
 pub struct OscCrawler {
     pub name: String,
     pub display_name: String,
@@ -39,25 +40,12 @@ impl Grab for OscCrawler {
         }
         Ok(res)
     }
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
 
     fn get_name(&self) -> String {
         self.name.to_owned()
     }
 }
 
-impl Default for OscCrawler {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 impl OscCrawler {
     pub fn new() -> Self {
         let headers = header::HeaderMap::new();

src/grab/seebug.rs

@@ -8,10 +8,11 @@ use crate::error::{Error, Result};
 use crate::grab::{Severity, VulnInfo};
 use crate::utils::http_client::Help;
 
-use super::{Grab, Provider};
+use super::Grab;
 
 const SEEBUG_LIST_URL: &str = "https://www.seebug.org/vuldb/vulnerabilities";
 
+#[derive(Default)]
 pub struct SeeBugCrawler {
     pub name: String,
     pub display_name: String,
@@ -33,25 +34,12 @@ impl Grab for SeeBugCrawler {
         }
         Ok(res)
     }
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
 
     fn get_name(&self) -> String {
         self.name.to_owned()
     }
 }
 
-impl Default for SeeBugCrawler {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 impl SeeBugCrawler {
     pub fn new() -> SeeBugCrawler {
         let headers = header::HeaderMap::new();
@@ -85,7 +73,7 @@ impl SeeBugCrawler {
         let url = format!("{}?page={}", SEEBUG_LIST_URL, page);
         let document = self.get_document(&url).await?;
         let selector = Selector::parse(".sebug-table tbody tr")
-            .map_err(|err| eyre!("parse html error {}", err))?;
+            .map_err(|err| eyre!("seebug parse html error {}", err))?;
         let tr_elements = document.select(&selector).collect::<Vec<_>>();
         if tr_elements.is_empty() {
             return Err(Error::Message("failed to get seebug page".into()));

src/grab/threatbook.rs

@@ -9,7 +9,7 @@ use crate::{
     utils::{check_over_two_week, http_client::Help},
 };
 
-use super::{Grab, Provider, VulnInfo};
+use super::{Grab, VulnInfo};
 
 const HOME_PAGE_URL: &str = "https://x.threatbook.com/v5/node/vul_module/homePage";
 const LINK: &str = "https://x.threatbook.com/v5/vulIntelligence";
@@ -75,14 +75,6 @@ impl Grab for ThreadBookCrawler {
         Ok(res)
     }
 
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
-
     fn get_name(&self) -> String {
         self.name.to_owned()
     }

src/grab/ti.rs

@@ -2,12 +2,13 @@ use async_trait::async_trait;
 use reqwest::header::{self};
 use serde::{Deserialize, Serialize};
 
-use super::{Grab, Provider, Severity, VulnInfo};
+use super::{Grab, Severity, VulnInfo};
 use crate::error::Result;
 use crate::utils::http_client::Help;
 
 const ONE_URL: &str = "https://ti.qianxin.com/alpha-api/v2/vuln/one-day";
 
+#[derive(Default)]
 pub struct TiCrawler {
     pub name: String,
     pub display_name: String,
@@ -20,25 +21,12 @@ impl Grab for TiCrawler {
     async fn get_update(&self, _page_limit: i32) -> Result<Vec<VulnInfo>> {
         self.get_vuln_infos().await
     }
-    fn get_provider(&self) -> Provider {
-        Provider {
-            name: self.name.to_owned(),
-            display_name: self.display_name.to_owned(),
-            link: self.link.to_owned(),
-        }
-    }
 
     fn get_name(&self) -> String {
         self.name.to_owned()
     }
 }
 
-impl Default for TiCrawler {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
 impl TiCrawler {
     pub fn new() -> TiCrawler {
         let mut headers = header::HeaderMap::new();