diff --git a/README.md b/README.md index e7d3344..29d75d1 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ # lookuper -**lookuper** performs lookups against VirusTotal/ThreatExpert/Google Safe Browsing data for data types such as MD5, SHA256, URL, IP, Domains, Strings e.g. mutexes +**lookuper** performs lookups against VirusTotal/ThreatExpert/Google Safe Browsing/HaveIBeenPnwed data for data types such as MD5, SHA256, URL, IP, Domains, Strings, Mutexes and email addresses Data is cached in a local SQLite database. The VirusTotal functionality supports multiple API keys, so you can supply four public API keys and it will run continuously, iterating through the API keys. ## Features - Data caching in SQLite database -- Support for multiple API keys -- Batched requesting -- Supports hashes (MD5 & SHA256), URL's, IP's and domains from VirusTotal -- Supports hashes (MD5, strings) from ThreatExpert -- Supports domains/URL's from Google SafeBrowsing +- Support for multiple API keys (VT) +- Batched requesting (VT) +- Supports hashes (MD5 & SHA256), URL's, IP's and domains from VirusTotal (https://www.virustotal.com/en/documentation/public-api/) +- Supports hashes (MD5, strings) from ThreatExpert (http://www.threatexpert.com/) +- Supports domains/URL's from Google SafeBrowsing (https://developers.google.com/safe-browsing/v4/) +- Supports email addresses from HaveIBeenPwned (https://haveibeenpwned.com/API/v2) ## Configuration @@ -34,7 +35,7 @@ virus_total_api_keys: - BBB... - CCC... - DDD... -max_hash_age: 30 +max_data_age: 30 ``` ## Command Line @@ -48,7 +49,7 @@ USAGE: lookuper [global options] command [command options] [arguments...] VERSION: - 0.0.1 + 0.0.7 AUTHOR(S): Mark Woan @@ -64,6 +65,7 @@ COMMANDS: urlvt Check URL's via VirusTotal stringte Check strings via ThreatExpert gsb Check Url's/Domains via Google Safe Browsing + hibp Check email addresses via HaveIBeenPwned help, h Shows a list of commands or help for one command GLOBAL OPTIONS: diff --git a/README.pdf b/README.pdf index 0f077f4..853f97b 100644 Binary files a/README.pdf and b/README.pdf differ diff --git a/source/src/woanware/lookuper/database.go b/source/src/woanware/lookuper/database.go index f0444f5..e69e45c 100644 --- a/source/src/woanware/lookuper/database.go +++ b/source/src/woanware/lookuper/database.go @@ -120,9 +120,10 @@ const SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING string = const SQL_CREATE_TABLE_HIBP string = `CREATE TABLE "hibp" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'email' text NOT NULL, - 'breaches' text + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'email' text NOT NULL, + 'breaches' text, + 'update_date' bigint );` const SQL_CREATE_INDEX_WORK string = diff --git a/source/src/woanware/lookuper/hibp.go b/source/src/woanware/lookuper/hibp.go index 956ace9..e825bab 100644 --- a/source/src/woanware/lookuper/hibp.go +++ b/source/src/woanware/lookuper/hibp.go @@ -14,6 +14,7 @@ type HaveIBeenPwned struct { Id int64 `db:"id"` Email string `db:"email"` Breaches string `db:"breaches"` + UpdateDate int64 `db:"update_date"` } // ##### Public Methods ################################################################################################ @@ -23,20 +24,19 @@ func (h *HaveIBeenPwned) Process(data string) int8 { var c hibp.HibpClient - log.Printf("%v", data) - err, resp, breaches := c.BreachesForAccount(data, "", true) if err != nil { if err.Error() == "EOF" { + // EOF means that there is no data on that account return WORK_RESPONSE_OK } - log.Printf("HIBP response status1: %v (%s)", err, data) + log.Printf("Error retrieving HIBP response: %v (%s)", err, data) return WORK_RESPONSE_ERROR } if len(resp) > 0 { - log.Printf("HIBP response status2: %v (%s)", resp, data) + log.Printf("Error retrieving HIBP response: %v (%s)", resp, data) return WORK_RESPONSE_ERROR } @@ -46,18 +46,19 @@ func (h *HaveIBeenPwned) Process(data string) int8 { temp := make([]string, 0) for _, b := range *breaches { - log.Printf("N: %v", b.Name) temp = append(temp, strings.TrimSpace(b.Name)) } - log.Printf("%v", temp) - return h.setRecord(data, strings.Join(temp, ",")) } // func (h *HaveIBeenPwned) DoesDataExist(data string, staleTimestamp time.Time) (error, bool) { - return nil, false + var temp HaveIBeenPwned + err := dbMap.SelectOne(&temp, "SELECT * FROM hibp WHERE email = $1", strings.ToLower(data)) + err, exists := validateDbData(temp.UpdateDate, staleTimestamp.Unix(), err) + + return err, exists } // ##### Private Methods ############################################################################################### @@ -105,6 +106,7 @@ func (h *HaveIBeenPwned) updateObject( hibp.Email = strings.ToLower(email) hibp.Breaches = breaches + hibp.UpdateDate = time.Now().UTC().Unix() } diff --git a/source/src/woanware/lookuper/main.go b/source/src/woanware/lookuper/main.go index 718155e..233c032 100644 --- a/source/src/woanware/lookuper/main.go +++ b/source/src/woanware/lookuper/main.go @@ -158,8 +158,8 @@ func loadConfig() (*Config) { log.Fatalf("Error unmarshalling the config file: %v", err) } - if c.MaxHashAge == 0 { - c.MaxHashAge = 30 + if c.MaxDataAge == 0 { + c.MaxDataAge = 30 } if c.Retries == 0 { diff --git a/source/src/woanware/lookuper/objects.go b/source/src/woanware/lookuper/objects.go index 23e9358..ca0fb2e 100644 --- a/source/src/woanware/lookuper/objects.go +++ b/source/src/woanware/lookuper/objects.go @@ -2,7 +2,7 @@ package main // Stores the yaml config file data type Config struct { - MaxHashAge uint16 `yaml:"max_hash_age"` + MaxDataAge uint16 `yaml:"max_data_age"` Retries uint8 `yaml:"retries"` VtApiKeys []string `yaml:"virus_total_api_keys"` SafeBrowsingApiKey string `yaml:"safe_browsing_api_key"` diff --git a/source/src/woanware/lookuper/worker.go b/source/src/woanware/lookuper/worker.go index fe21418..00c22ad 100644 --- a/source/src/woanware/lookuper/worker.go +++ b/source/src/woanware/lookuper/worker.go @@ -287,7 +287,7 @@ func (w *Worker) loadBatch(batchSize int) (BatchData) { var data string var ret bool - staleTimestamp := time.Now().UTC().Add(-time.Duration(24*config.MaxHashAge) * time.Hour) + staleTimestamp := time.Now().UTC().Add(-time.Duration(24*config.MaxDataAge) * time.Hour) workData := make(map[string]int8) @@ -363,7 +363,8 @@ func (w *Worker) doesDataExistInDb(staleTimestamp time.Time, data string) (error gsb := GoogleSafeBrowsing{} return gsb.DoesDataExist(data, staleTimestamp) case dataTypeHibp: - return nil, false + hibp := HaveIBeenPwned{} + return hibp.DoesDataExist(data, staleTimestamp) } return nil, false