From 593bf5e97d2dfbefb66031bc2ceb6475c71d7a24 Mon Sep 17 00:00:00 2001 From: Mark Woan Date: Mon, 7 Nov 2016 22:19:07 +0000 Subject: [PATCH] Added new HaveIBeenPwned functionality --- source/src/woanware/lookuper/command_line.go | 23 ++++ source/src/woanware/lookuper/database.go | 114 +++++++++-------- source/src/woanware/lookuper/global.go | 5 + source/src/woanware/lookuper/hibp.go | 111 +++++++++++++++++ source/src/woanware/lookuper/job.go | 38 ++++++ source/src/woanware/lookuper/main.go | 4 +- source/src/woanware/lookuper/worker.go | 11 +- source/vendor/manifest | 6 + .../infoassure/go-haveibeenpwned/hibp.go | 117 ++++++++++++++++++ 9 files changed, 376 insertions(+), 53 deletions(-) create mode 100644 source/src/woanware/lookuper/hibp.go create mode 100644 source/vendor/src/github.com/infoassure/go-haveibeenpwned/hibp.go diff --git a/source/src/woanware/lookuper/command_line.go b/source/src/woanware/lookuper/command_line.go index a91bd86..13665e8 100644 --- a/source/src/woanware/lookuper/command_line.go +++ b/source/src/woanware/lookuper/command_line.go @@ -310,5 +310,28 @@ func setupCli(app *cli.App) { apiFlag, }, }, + { + Name: "hibp", + Usage: "Check email addresses via HaveIBeenPwned", + Action: func(c *cli.Context) error { + + err := checkInputFile(c.String("input")) + if err != nil { + return err + } + + err = checkOutputDirectory(c.String("output")) + if err != nil { + return err + } + + run(dataTypeHibp, c.String("input"), c.String("output"), []string{FAKE_API_KEY3}) + return nil + }, + Flags: []cli.Flag{ + inputFileFlag, + outputDirFlag, + }, + }, } } \ No newline at end of file diff --git a/source/src/woanware/lookuper/database.go b/source/src/woanware/lookuper/database.go index 2220f70..f0444f5 100644 --- a/source/src/woanware/lookuper/database.go +++ b/source/src/woanware/lookuper/database.go @@ -9,7 +9,7 @@ import ( const SQL_CREATE_TABLE_JOB string = `CREATE TABLE "job" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 'type' smallint NOT NULL, 'api_keys' text, 'are_api_keys_private' smallint @@ -17,39 +17,39 @@ const SQL_CREATE_TABLE_JOB string = const SQL_CREATE_TABLE_WORK string = `CREATE TABLE "work" ( - 'md5' text NOT NULL, - 'response_code' smallint, - 'data' text + 'md5' text NOT NULL, + 'response_code' smallint, + 'data' text );` const SQL_CREATE_TABLE_VT_HASH string = `CREATE TABLE 'vt_hash' ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'md5' text NOT NULL, - 'sha256' text NOT NULL, - 'positives' smallint NOT NULL, - 'total' smallint NOT NULL, - 'permalink' text, - 'scans' text, - 'scan_date' bigint NOT NULL, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'md5' text NOT NULL, + 'sha256' text NOT NULL, + 'positives' smallint NOT NULL, + 'total' smallint NOT NULL, + 'permalink' text, + 'scans' text, + 'scan_date' bigint NOT NULL, 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_VT_DOMAIN_DETECTED_URL string = `CREATE TABLE "vt_domain_detected_url" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'url' text NOT NULL, - 'url_md5' text NOT NULL, - 'positives' smallint NOT NULL, - 'total' smallint NOT NULL, - 'scan_date' bigint NOT NULL, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'url' text NOT NULL, + 'url_md5' text NOT NULL, + 'positives' smallint NOT NULL, + 'total' smallint NOT NULL, + 'scan_date' bigint NOT NULL, 'update_date' bigint NOT NULL, 'domain_md5' text );` const SQL_CREATE_TABLE_VT_DOMAIN_RESOLUTION string = `CREATE TABLE "vt_domain_resolution" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 'domain_md5' text NOT NULL, 'last_resolved' bigint NOT NULL, 'ip_address' bigint NOT NULL, @@ -58,66 +58,73 @@ const SQL_CREATE_TABLE_VT_DOMAIN_RESOLUTION string = const SQL_CREATE_TABLE_TE_HASH string = `CREATE TABLE "te_hash" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'md5' text NOT NULL, - 'name' text, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'md5' text NOT NULL, + 'name' text, 'severities' text, - 'scan_date' bigint NOT NULL, + 'scan_date' bigint NOT NULL, 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_VT_IP_DETECTED_URL string = `CREATE TABLE "vt_ip_detected_url" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'ip' bigint NOT NULL, - 'url' text NOT NULL, - 'url_md5' text NOT NULL, - 'positives' smallint NOT NULL, - 'total' smallint NOT NULL, - 'scan_date' bigint NOT NULL, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'ip' bigint NOT NULL, + 'url' text NOT NULL, + 'url_md5' text NOT NULL, + 'positives' smallint NOT NULL, + 'total' smallint NOT NULL, + 'scan_date' bigint NOT NULL, 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_VT_IP_RESOLUTION string = `CREATE TABLE "vt_ip_resolution" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'ip' bigint NOT NULL, - 'last_resolved' bigint NOT NULL, - 'host_name' text NOT NULL, - 'host_name_md5' text NOT NULL, - 'update_date' bigint NOT NULL + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'ip' bigint NOT NULL, + 'last_resolved' bigint NOT NULL, + 'host_name' text NOT NULL, + 'host_name_md5' text NOT NULL, + 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_TE_STRING string = `CREATE TABLE "te_string" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'string' text NOT NULL, - 'count' integer, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'string' text NOT NULL, + 'count' integer, 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_VT_URL string = `CREATE TABLE "vt_url" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'url' text NOT NULL, - 'url_md5' text NOT NULL, - 'positives' smallint NOT NULL, - 'total' smallint NOT NULL, - 'permalink' text, - 'scans' text, - 'scan_date' bigint NOT NULL, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'url' text NOT NULL, + 'url_md5' text NOT NULL, + 'positives' smallint NOT NULL, + 'total' smallint NOT NULL, + 'permalink' text, + 'scans' text, + 'scan_date' bigint NOT NULL, 'update_date' bigint NOT NULL );` const SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING string = `CREATE TABLE "google_safe_browsing" ( - 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - 'url' text, - 'url_md5' text, - 'data' text, + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'url' text, + 'url_md5' text, + 'data' text, 'update_date' bigint );` +const SQL_CREATE_TABLE_HIBP string = + `CREATE TABLE "hibp" ( + 'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + 'email' text NOT NULL, + 'breaches' text + );` + const SQL_CREATE_INDEX_WORK string = `CREATE INDEX 'idx_work' ON 'work' ('md5');` @@ -145,6 +152,9 @@ const SQL_CREATE_INDEX_TE_STRING string = const SQL_CREATE_INDEX_GSB string = `CREATE INDEX 'idx_gsb' ON 'google_safe_browsing' ('url_md5');` +const SQL_CREATE_INDEX_HIBP string = + `CREATE INDEX 'idx_hibp' ON 'hibp' ('email');` + var DATABASE_SQL_CREATES = []string { SQL_CREATE_TABLE_JOB, SQL_CREATE_TABLE_WORK, @@ -157,6 +167,7 @@ var DATABASE_SQL_CREATES = []string { SQL_CREATE_TABLE_TE_STRING, SQL_CREATE_TABLE_VT_URL, SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING, + SQL_CREATE_TABLE_HIBP, } var DATABASE_SQL_INDEXES = []string { @@ -169,6 +180,7 @@ var DATABASE_SQL_INDEXES = []string { SQL_CREATE_INDEX_TE_HASH, SQL_CREATE_INDEX_TE_STRING, SQL_CREATE_INDEX_GSB, + SQL_CREATE_INDEX_HIBP, } // ##### Methods ####################################################################################################### diff --git a/source/src/woanware/lookuper/global.go b/source/src/woanware/lookuper/global.go index e0a8c69..e639aaf 100644 --- a/source/src/woanware/lookuper/global.go +++ b/source/src/woanware/lookuper/global.go @@ -10,6 +10,7 @@ const ( dataTypeMd5Te = 6 dataTypeStringTe = 7 dataTypeGsb = 8 // Google SafeBrowsing + dataTypeHibp = 9 // HaveIBeenPwned ) // String values for the job data types @@ -22,6 +23,7 @@ var dataTypes = []string{ dataTypeMd5Te: "MD5 (TE)", dataTypeStringTe: "String (TE)", dataTypeGsb: "Google Safe Browsing", + dataTypeHibp: "HaveIBeenPwned", } // Response codes for use in the "work" table @@ -41,3 +43,6 @@ const FAKE_API_KEY string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777 // Used for Google SafeBrowsing const FAKE_API_KEY2 string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777788889999AAAB" + +// Used for HIBP +const FAKE_API_KEY3 string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777788889999AABC" diff --git a/source/src/woanware/lookuper/hibp.go b/source/src/woanware/lookuper/hibp.go new file mode 100644 index 0000000..956ace9 --- /dev/null +++ b/source/src/woanware/lookuper/hibp.go @@ -0,0 +1,111 @@ +package main + +import ( + hibp "github.com/infoassure/go-haveibeenpwned" + "log" + "time" + "strings" +) + +// ##### Structs ####################################################################################################### + +// Encapsulates the data from the "hibp" table +type HaveIBeenPwned struct { + Id int64 `db:"id"` + Email string `db:"email"` + Breaches string `db:"breaches"` +} + +// ##### Public Methods ################################################################################################ + +// Processes a TE request for a single string +func (h *HaveIBeenPwned) Process(data string) int8 { + + var c hibp.HibpClient + + log.Printf("%v", data) + + err, resp, breaches := c.BreachesForAccount(data, "", true) + if err != nil { + if err.Error() == "EOF" { + return WORK_RESPONSE_OK + } + + log.Printf("HIBP response status1: %v (%s)", err, data) + return WORK_RESPONSE_ERROR + } + + if len(resp) > 0 { + log.Printf("HIBP response status2: %v (%s)", resp, data) + return WORK_RESPONSE_ERROR + } + + if len(*breaches) == 0 { + return WORK_RESPONSE_OK + } + + temp := make([]string, 0) + for _, b := range *breaches { + log.Printf("N: %v", b.Name) + temp = append(temp, strings.TrimSpace(b.Name)) + } + + log.Printf("%v", temp) + + return h.setRecord(data, strings.Join(temp, ",")) +} + +// +func (h *HaveIBeenPwned) DoesDataExist(data string, staleTimestamp time.Time) (error, bool) { + return nil, false +} + +// ##### Private Methods ############################################################################################### + +// Inserts a new TE string record, if that fails due to it already existing, then retrieve details and update +func (h *HaveIBeenPwned) setRecord(email string, breaches string) int8 { + + hibp := new(HaveIBeenPwned) + h.updateObject(hibp, email, breaches) + + err := dbMap.SelectOne(hibp, "SELECT * FROM hibp WHERE email = $1", hibp.Email) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no rows in result set") == false { + log.Printf("Error inserting HIBP record: %v", err) + return WORK_RESPONSE_ERROR + } + + err := dbMap.Insert(hibp) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "duplicate key value violates") == false { + log.Printf("Error inserting HIBP record: %v", err) + return WORK_RESPONSE_ERROR + } + } + + return WORK_RESPONSE_OK + } + + h.updateObject(hibp, email, breaches) + _, err = dbMap.Update(hibp) + if err != nil { + log.Printf("Error updating HIBP record: %v", err) + return WORK_RESPONSE_ERROR + } + + return WORK_RESPONSE_OK +} + +// Generic method to copy the data into the HIBP object +func (h *HaveIBeenPwned) updateObject( + + hibp *HaveIBeenPwned, + email string, + breaches string) { + + hibp.Email = strings.ToLower(email) + hibp.Breaches = breaches +} + + + diff --git a/source/src/woanware/lookuper/job.go b/source/src/woanware/lookuper/job.go index e2666ee..b9d1d2c 100644 --- a/source/src/woanware/lookuper/job.go +++ b/source/src/woanware/lookuper/job.go @@ -73,6 +73,9 @@ func (j *Job) GenerateCsv(outputFilePath string) { case dataTypeGsb: j.OutputGsb(outputFilePath) + + case dataTypeHibp: + j.OutputHibp(outputFilePath) } } @@ -420,5 +423,40 @@ func (j *Job) OutputGsb(outputDir string) { gsb.Data}) } + csvWriter.Flush() +} + +// Creates a CSV results file for HIBP jobs +func (j *Job) OutputHibp(outputDir string) { + + w := Work{} + data := w.GetAllWork() + + file, err := os.Create(path.Join(outputDir, "lookuper-hibp.csv")); + defer file.Close() + if err != nil { + log.Fatalf("Error opening output file: %v (%s)", err, path.Join(outputDir, "lookuper-hibp.csv")) + } + + csvWriter := csv.NewWriter(file) + csvWriter.Write([]string{"Email", "Breaches"}) + + var hibp HaveIBeenPwned + + for _, d := range data { + + err = dbMap.SelectOne(&hibp, "SELECT * FROM hibp WHERE email = $1", strings.ToLower(d)) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no rows in result set") == false { + log.Printf("Error retrieving data for HIBP output: %v", err) + } + continue + } + + csvWriter.Write([]string{ + hibp.Email, + hibp.Breaches}) + } + csvWriter.Flush() } \ No newline at end of file diff --git a/source/src/woanware/lookuper/main.go b/source/src/woanware/lookuper/main.go index 6600a51..718155e 100644 --- a/source/src/woanware/lookuper/main.go +++ b/source/src/woanware/lookuper/main.go @@ -29,7 +29,7 @@ var ( // ##### Constants ############################################################ const APP_NAME string = "lookuper" -const APP_VERSION string = "0.0.6" +const APP_VERSION string = "0.0.7" const DB_FILE_NAME string = "./lookuper.db" const CONFIG_FILE_NAME string = "./lookuper.config" @@ -113,6 +113,8 @@ func initialiseDb() (*gorp.DbMap) { dbMap.AddTableWithName(TeHash{}, "te_hash").SetKeys(true, "id") dbMap.AddTableWithName(TeString{}, "te_string").SetKeys(true, "id") dbMap.AddTableWithName(VtUrl{}, "vt_url").SetKeys(true, "id") + dbMap.AddTableWithName(VtUrl{}, "vt_url").SetKeys(true, "id") + dbMap.AddTableWithName(HaveIBeenPwned{}, "hibp").SetKeys(true, "id") dbMap.AddTableWithName(GoogleSafeBrowsing{}, "google_safe_browsing").SetKeys(true, "id") dbMap.AddTableWithName(VtIpResolution{}, "vt_ip_resolution").SetKeys(true, "id") dbMap.AddTableWithName(VtIpDetectedUrl{}, "vt_ip_detected_url").SetKeys(true, "id") diff --git a/source/src/woanware/lookuper/worker.go b/source/src/woanware/lookuper/worker.go index bb32323..fe21418 100644 --- a/source/src/woanware/lookuper/worker.go +++ b/source/src/woanware/lookuper/worker.go @@ -247,7 +247,8 @@ func (w *Worker) process(apiKey string) int8 { w.dataType != dataTypeSha256Vt || w.dataType == dataTypeMd5Te || w.dataType == dataTypeStringTe || - w.dataType == dataTypeGsb { + w.dataType == dataTypeGsb || + w.dataType == dataTypeHibp { batchSize = 1 } @@ -361,6 +362,8 @@ func (w *Worker) doesDataExistInDb(staleTimestamp time.Time, data string) (error case dataTypeGsb: gsb := GoogleSafeBrowsing{} return gsb.DoesDataExist(data, staleTimestamp) + case dataTypeHibp: + return nil, false } return nil, false @@ -450,6 +453,9 @@ func (w *Worker) processBatch(apiKey string, batch BatchData) int8 { case dataTypeGsb: gsb := GoogleSafeBrowsing{} response_code = gsb.Process(batch.Items[0]) + case dataTypeHibp: + hibp := HaveIBeenPwned{} + response_code = hibp.Process(batch.Items[0]) } } else { switch w.dataType { @@ -507,6 +513,9 @@ func (w *Worker) doPause(apiKey string) { time.Sleep(500 * time.Millisecond) } else if apiKey == FAKE_API_KEY2 { // This is a fake API key set for Google SB requests. + } else if apiKey == FAKE_API_KEY3 { + // This is a fake API key set for HIBP requests. + time.Sleep(1600 * time.Millisecond) } else { // VT pause is about 15 seconds so we give a pause of 17 to be on the safe side time.Sleep(17 * time.Second) diff --git a/source/vendor/manifest b/source/vendor/manifest index 9f0a012..3a492ee 100644 --- a/source/vendor/manifest +++ b/source/vendor/manifest @@ -45,6 +45,12 @@ "revision": "c2df0f33af31470035c6adf857e60670d582374c", "branch": "master" }, + { + "importpath": "github.com/infoassure/go-haveibeenpwned", + "repository": "https://github.com/infoassure/go-haveibeenpwned", + "revision": "80b3d7c569723892e98e125284881ede209f0915", + "branch": "master" + }, { "importpath": "github.com/mattn/go-sqlite3", "repository": "https://github.com/mattn/go-sqlite3", diff --git a/source/vendor/src/github.com/infoassure/go-haveibeenpwned/hibp.go b/source/vendor/src/github.com/infoassure/go-haveibeenpwned/hibp.go new file mode 100644 index 0000000..6b796e5 --- /dev/null +++ b/source/vendor/src/github.com/infoassure/go-haveibeenpwned/hibp.go @@ -0,0 +1,117 @@ +package hibp + +import ( + "net/http" + "time" + "encoding/json" + "fmt" + "net/url" +) + +const API_URL = "https://haveibeenpwned.com/api/v2/%s" + +// Struct that represents our HIBP client +type HibpClient struct { +} + +// Parameters for the HTTP requests +type Parameters map[string]string + +type Breaches []Breach + +type Breach struct { + Name string `json:"Name"` + Title string `json:"Title"` + Domain string `json:"Domain"` + BreachDate string `json:"BreachDate"` + AddedDate time.Time `json:"AddedDate"` + PwnCount int `json:"PwnCount"` + DataClasses []string `json:"DataClasses"` + Description string `json:"Description"` + IsVerified bool `json:"IsVerified"` + IsSensitive bool `json:"IsSensitive"` + IsRetired bool `json:"IsRetired"` +} + +// ***** Private Methods *********************************************************************************************** + +func (h *HibpClient) getApiJson(actionUrl string, parameters Parameters, result interface{}) (err error, resp string) { + + values := url.Values{} + for k, v := range parameters { + values.Add(k, v) + } + + client := new(http.Client) + + req, err := http.NewRequest("GET", fmt.Sprintf(API_URL, actionUrl) + "?" + values.Encode(), nil) + if err != nil { + return err, "" + } + + req.Header.Add("Accept", "application/vnd.haveibeenpwned.v2+json") + req.Header.Add("User-Agent", "go-haveibeenpwned (HIBP golang API client) - https://github.com/infoassure/go-haveibeenpwned") + + res, err := client.Do(req) + if err != nil { + return err, "" + } + + if err != nil { + return err, "" + } + defer res.Body.Close() + + dec := json.NewDecoder(res.Body) + if err = dec.Decode(result); err != nil { + return err, "" + } + + return nil, h.getResponseString(res.StatusCode, res.Status) +} + +// Returns the API specific HTTP response descriptions +func (h *HibpClient) getResponseString(code int, desc string) string { + switch (code) { + case 400: + return "Bad request — the account does not comply with an acceptable format (i.e. it's an empty string)" + case 403: + return "Forbidden — no user agent has been specified in the request" + case 404: + return "Not found — the account could not be found and has therefore not been pwned" + case 429: + return "Too many requests — the rate limit has been exceeded" + default: + return "" + } +} + +// ***** Public Methods ***********************************************************************************************+ + +func (h *HibpClient) BreachesForAccount(email string, domain string, truncateResponse bool) (err error, resp string, breaches *Breaches) { + + var p Parameters + + if len(domain) > 0 { + if len(p) == 0 { + p = make(map[string]string) + } + p["domain"] = domain + } + + if truncateResponse == true { + if len(p) == 0 { + p = make(map[string]string) + } + p["truncateResponse"] = "true" + } + + breaches = &Breaches{} + err, resp = h.getApiJson("breachedaccount/" + email, p, breaches) + if err != nil { + return err, resp, nil + } + + return nil, resp, breaches +} +