Final HIBP changes

Updated docs
Renamed config file value

README.md

@@ -1,17 +1,18 @@
 # lookuper
 
-**lookuper** performs lookups against VirusTotal/ThreatExpert/Google Safe Browsing data for data types such as MD5, SHA256, URL, IP, Domains, Strings e.g. mutexes
+**lookuper** performs lookups against VirusTotal/ThreatExpert/Google Safe Browsing/HaveIBeenPnwed data for data types such as MD5, SHA256, URL, IP, Domains, Strings, Mutexes and email addresses
 
 Data is cached in a local SQLite database. The VirusTotal functionality supports multiple API keys, so you can supply four public API keys and it will run continuously, iterating through the API keys.
 
 ## Features
 
 - Data caching in SQLite database
-- Support for multiple API keys
-- Batched requesting
-- Supports hashes (MD5 & SHA256), URL's, IP's and domains from VirusTotal
-- Supports hashes (MD5, strings) from ThreatExpert
-- Supports domains/URL's from Google SafeBrowsing
+- Support for multiple API keys (VT)
+- Batched requesting (VT)
+- Supports hashes (MD5 & SHA256), URL's, IP's and domains from VirusTotal (https://www.virustotal.com/en/documentation/public-api/)
+- Supports hashes (MD5, strings) from ThreatExpert (http://www.threatexpert.com/)
+- Supports domains/URL's from Google SafeBrowsing (https://developers.google.com/safe-browsing/v4/)
+- Supports email addresses from HaveIBeenPwned (https://haveibeenpwned.com/API/v2)
 
 ## Configuration
 
@@ -34,7 +35,7 @@ virus_total_api_keys:
 - BBB...
 - CCC...
 - DDD...
-max_hash_age: 30
+max_data_age: 30
 ```
 
 ## Command Line
@@ -48,7 +49,7 @@ USAGE:
    lookuper [global options] command [command options] [arguments...]
 
 VERSION:
-   0.0.1
+   0.0.7
 
 AUTHOR(S):
    Mark Woan <[email protected]>
@@ -64,6 +65,7 @@ COMMANDS:
      urlvt     Check URL's via VirusTotal
      stringte  Check strings via ThreatExpert
      gsb       Check Url's/Domains via Google Safe Browsing
+     hibp      Check email addresses via HaveIBeenPwned
      help, h   Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:

source/src/woanware/lookuper/database.go

@@ -120,9 +120,10 @@ const SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING string =
 
 const SQL_CREATE_TABLE_HIBP string =
 	`CREATE TABLE "hibp" (
-		'id'		INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
-		'email'		text NOT NULL,
-		'breaches'	text
+		'id'			INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+		'email'			text NOT NULL,
+		'breaches'		text,
+		'update_date'	bigint
 	);`
 
 const SQL_CREATE_INDEX_WORK string =

source/src/woanware/lookuper/hibp.go

@@ -14,6 +14,7 @@ type HaveIBeenPwned struct {
 	Id         	int64	`db:"id"`
 	Email      	string	`db:"email"`
 	Breaches	string	`db:"breaches"`
+	UpdateDate	int64 	`db:"update_date"`
 }
 
 // ##### Public Methods ################################################################################################
@@ -23,20 +24,19 @@ func (h *HaveIBeenPwned) Process(data string) int8 {
 
 	var c hibp.HibpClient
 
-	log.Printf("%v", data)
-
 	err, resp, breaches := c.BreachesForAccount(data, "", true)
 	if err != nil {
 		if err.Error() == "EOF" {
+			// EOF means that there is no data on that account
 			return WORK_RESPONSE_OK
 		}
 
-		log.Printf("HIBP response status1: %v (%s)", err, data)
+		log.Printf("Error retrieving HIBP response: %v (%s)", err, data)
 		return WORK_RESPONSE_ERROR
 	}
 
 	if len(resp) > 0 {
-		log.Printf("HIBP response status2: %v (%s)", resp, data)
+		log.Printf("Error retrieving HIBP response: %v (%s)", resp, data)
 		return WORK_RESPONSE_ERROR
 	}
 
@@ -46,18 +46,19 @@ func (h *HaveIBeenPwned) Process(data string) int8 {
 
 	temp := make([]string, 0)
 	for _, b := range *breaches {
-		log.Printf("N: %v", b.Name)
 		temp = append(temp, strings.TrimSpace(b.Name))
 	}
 
-	log.Printf("%v", temp)
-
 	return h.setRecord(data, strings.Join(temp, ","))
 }
 
 //
 func (h *HaveIBeenPwned) DoesDataExist(data string, staleTimestamp time.Time) (error, bool) {
-	return nil, false
+	var temp HaveIBeenPwned
+	err := dbMap.SelectOne(&temp, "SELECT * FROM hibp WHERE email = $1", strings.ToLower(data))
+	err, exists := validateDbData(temp.UpdateDate, staleTimestamp.Unix(), err)
+
+	return err, exists
 }
 
 // ##### Private Methods ###############################################################################################
@@ -105,6 +106,7 @@ func (h *HaveIBeenPwned) updateObject(
 
 	hibp.Email = strings.ToLower(email)
 	hibp.Breaches = breaches
+	hibp.UpdateDate = time.Now().UTC().Unix()
 }
 
 

source/src/woanware/lookuper/main.go

@@ -158,8 +158,8 @@ func loadConfig() (*Config) {
 		log.Fatalf("Error unmarshalling the config file: %v", err)
 	}
 
-	if c.MaxHashAge == 0 {
-		c.MaxHashAge = 30
+	if c.MaxDataAge == 0 {
+		c.MaxDataAge = 30
 	}
 
 	if c.Retries == 0 {

source/src/woanware/lookuper/objects.go

@@ -2,7 +2,7 @@ package main
 
 // Stores the yaml config file data
 type Config struct {
-	MaxHashAge 				uint16  	`yaml:"max_hash_age"`
+	MaxDataAge 				uint16  	`yaml:"max_data_age"`
 	Retries 				uint8   	`yaml:"retries"`
 	VtApiKeys				[]string	`yaml:"virus_total_api_keys"`
 	SafeBrowsingApiKey 		string 		`yaml:"safe_browsing_api_key"`

source/src/woanware/lookuper/worker.go

@@ -287,7 +287,7 @@ func (w *Worker) loadBatch(batchSize int) (BatchData) {
 	var data string
 	var ret bool
 
-	staleTimestamp := time.Now().UTC().Add(-time.Duration(24*config.MaxHashAge) * time.Hour)
+	staleTimestamp := time.Now().UTC().Add(-time.Duration(24*config.MaxDataAge) * time.Hour)
 
 	workData := make(map[string]int8)
 
@@ -363,7 +363,8 @@ func (w *Worker) doesDataExistInDb(staleTimestamp time.Time, data string) (error
 		gsb := GoogleSafeBrowsing{}
 		return gsb.DoesDataExist(data, staleTimestamp)
 	case dataTypeHibp:
-		return nil, false
+		hibp := HaveIBeenPwned{}
+		return hibp.DoesDataExist(data, staleTimestamp)
 	}
 
 	return nil, false