Skip to content

Commit

Permalink
Added new HaveIBeenPwned functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
woanware committed Nov 7, 2016
1 parent bad714d commit 593bf5e
Show file tree
Hide file tree
Showing 9 changed files with 376 additions and 53 deletions.
23 changes: 23 additions & 0 deletions source/src/woanware/lookuper/command_line.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,5 +310,28 @@ func setupCli(app *cli.App) {
apiFlag,
},
},
{
Name: "hibp",
Usage: "Check email addresses via HaveIBeenPwned",
Action: func(c *cli.Context) error {

err := checkInputFile(c.String("input"))
if err != nil {
return err
}

err = checkOutputDirectory(c.String("output"))
if err != nil {
return err
}

run(dataTypeHibp, c.String("input"), c.String("output"), []string{FAKE_API_KEY3})
return nil
},
Flags: []cli.Flag{
inputFileFlag,
outputDirFlag,
},
},
}
}
114 changes: 63 additions & 51 deletions source/src/woanware/lookuper/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,47 +9,47 @@ import (

const SQL_CREATE_TABLE_JOB string =
`CREATE TABLE "job" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'type' smallint NOT NULL,
'api_keys' text,
'are_api_keys_private' smallint
);`

const SQL_CREATE_TABLE_WORK string =
`CREATE TABLE "work" (
'md5' text NOT NULL,
'response_code' smallint,
'data' text
'md5' text NOT NULL,
'response_code' smallint,
'data' text
);`

const SQL_CREATE_TABLE_VT_HASH string =
`CREATE TABLE 'vt_hash' (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'md5' text NOT NULL,
'sha256' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'permalink' text,
'scans' text,
'scan_date' bigint NOT NULL,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'md5' text NOT NULL,
'sha256' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'permalink' text,
'scans' text,
'scan_date' bigint NOT NULL,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_VT_DOMAIN_DETECTED_URL string =
`CREATE TABLE "vt_domain_detected_url" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'scan_date' bigint NOT NULL,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'scan_date' bigint NOT NULL,
'update_date' bigint NOT NULL,
'domain_md5' text
);`

const SQL_CREATE_TABLE_VT_DOMAIN_RESOLUTION string =
`CREATE TABLE "vt_domain_resolution" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'domain_md5' text NOT NULL,
'last_resolved' bigint NOT NULL,
'ip_address' bigint NOT NULL,
Expand All @@ -58,66 +58,73 @@ const SQL_CREATE_TABLE_VT_DOMAIN_RESOLUTION string =

const SQL_CREATE_TABLE_TE_HASH string =
`CREATE TABLE "te_hash" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'md5' text NOT NULL,
'name' text,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'md5' text NOT NULL,
'name' text,
'severities' text,
'scan_date' bigint NOT NULL,
'scan_date' bigint NOT NULL,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_VT_IP_DETECTED_URL string =
`CREATE TABLE "vt_ip_detected_url" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'ip' bigint NOT NULL,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'scan_date' bigint NOT NULL,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'ip' bigint NOT NULL,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'scan_date' bigint NOT NULL,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_VT_IP_RESOLUTION string =
`CREATE TABLE "vt_ip_resolution" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'ip' bigint NOT NULL,
'last_resolved' bigint NOT NULL,
'host_name' text NOT NULL,
'host_name_md5' text NOT NULL,
'update_date' bigint NOT NULL
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'ip' bigint NOT NULL,
'last_resolved' bigint NOT NULL,
'host_name' text NOT NULL,
'host_name_md5' text NOT NULL,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_TE_STRING string =
`CREATE TABLE "te_string" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'string' text NOT NULL,
'count' integer,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'string' text NOT NULL,
'count' integer,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_VT_URL string =
`CREATE TABLE "vt_url" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'permalink' text,
'scans' text,
'scan_date' bigint NOT NULL,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text NOT NULL,
'url_md5' text NOT NULL,
'positives' smallint NOT NULL,
'total' smallint NOT NULL,
'permalink' text,
'scans' text,
'scan_date' bigint NOT NULL,
'update_date' bigint NOT NULL
);`

const SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING string =
`CREATE TABLE "google_safe_browsing" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text,
'url_md5' text,
'data' text,
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'url' text,
'url_md5' text,
'data' text,
'update_date' bigint
);`

const SQL_CREATE_TABLE_HIBP string =
`CREATE TABLE "hibp" (
'id' INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
'email' text NOT NULL,
'breaches' text
);`

const SQL_CREATE_INDEX_WORK string =
`CREATE INDEX 'idx_work' ON 'work' ('md5');`

Expand Down Expand Up @@ -145,6 +152,9 @@ const SQL_CREATE_INDEX_TE_STRING string =
const SQL_CREATE_INDEX_GSB string =
`CREATE INDEX 'idx_gsb' ON 'google_safe_browsing' ('url_md5');`

const SQL_CREATE_INDEX_HIBP string =
`CREATE INDEX 'idx_hibp' ON 'hibp' ('email');`

var DATABASE_SQL_CREATES = []string {
SQL_CREATE_TABLE_JOB,
SQL_CREATE_TABLE_WORK,
Expand All @@ -157,6 +167,7 @@ var DATABASE_SQL_CREATES = []string {
SQL_CREATE_TABLE_TE_STRING,
SQL_CREATE_TABLE_VT_URL,
SQL_CREATE_TABLE_GOOGLE_SAFE_BROWSING,
SQL_CREATE_TABLE_HIBP,
}

var DATABASE_SQL_INDEXES = []string {
Expand All @@ -169,6 +180,7 @@ var DATABASE_SQL_INDEXES = []string {
SQL_CREATE_INDEX_TE_HASH,
SQL_CREATE_INDEX_TE_STRING,
SQL_CREATE_INDEX_GSB,
SQL_CREATE_INDEX_HIBP,
}

// ##### Methods #######################################################################################################
Expand Down
5 changes: 5 additions & 0 deletions source/src/woanware/lookuper/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const (
dataTypeMd5Te = 6
dataTypeStringTe = 7
dataTypeGsb = 8 // Google SafeBrowsing
dataTypeHibp = 9 // HaveIBeenPwned
)

// String values for the job data types
Expand All @@ -22,6 +23,7 @@ var dataTypes = []string{
dataTypeMd5Te: "MD5 (TE)",
dataTypeStringTe: "String (TE)",
dataTypeGsb: "Google Safe Browsing",
dataTypeHibp: "HaveIBeenPwned",
}

// Response codes for use in the "work" table
Expand All @@ -41,3 +43,6 @@ const FAKE_API_KEY string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777

// Used for Google SafeBrowsing
const FAKE_API_KEY2 string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777788889999AAAB"

// Used for HIBP
const FAKE_API_KEY3 string = "AAAABBBBCCCCEEEEFFFF0000111122223333444455556666777788889999AABC"
111 changes: 111 additions & 0 deletions source/src/woanware/lookuper/hibp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package main

import (
hibp "github.com/infoassure/go-haveibeenpwned"
"log"
"time"
"strings"
)

// ##### Structs #######################################################################################################

// Encapsulates the data from the "hibp" table
type HaveIBeenPwned struct {
Id int64 `db:"id"`
Email string `db:"email"`
Breaches string `db:"breaches"`
}

// ##### Public Methods ################################################################################################

// Processes a TE request for a single string
func (h *HaveIBeenPwned) Process(data string) int8 {

var c hibp.HibpClient

log.Printf("%v", data)

err, resp, breaches := c.BreachesForAccount(data, "", true)
if err != nil {
if err.Error() == "EOF" {
return WORK_RESPONSE_OK
}

log.Printf("HIBP response status1: %v (%s)", err, data)
return WORK_RESPONSE_ERROR
}

if len(resp) > 0 {
log.Printf("HIBP response status2: %v (%s)", resp, data)
return WORK_RESPONSE_ERROR
}

if len(*breaches) == 0 {
return WORK_RESPONSE_OK
}

temp := make([]string, 0)
for _, b := range *breaches {
log.Printf("N: %v", b.Name)
temp = append(temp, strings.TrimSpace(b.Name))
}

log.Printf("%v", temp)

return h.setRecord(data, strings.Join(temp, ","))
}

//
func (h *HaveIBeenPwned) DoesDataExist(data string, staleTimestamp time.Time) (error, bool) {
return nil, false
}

// ##### Private Methods ###############################################################################################

// Inserts a new TE string record, if that fails due to it already existing, then retrieve details and update
func (h *HaveIBeenPwned) setRecord(email string, breaches string) int8 {

hibp := new(HaveIBeenPwned)
h.updateObject(hibp, email, breaches)

err := dbMap.SelectOne(hibp, "SELECT * FROM hibp WHERE email = $1", hibp.Email)
if err != nil {
if strings.Contains(strings.ToLower(err.Error()), "no rows in result set") == false {
log.Printf("Error inserting HIBP record: %v", err)
return WORK_RESPONSE_ERROR
}

err := dbMap.Insert(hibp)
if err != nil {
if strings.Contains(strings.ToLower(err.Error()), "duplicate key value violates") == false {
log.Printf("Error inserting HIBP record: %v", err)
return WORK_RESPONSE_ERROR
}
}

return WORK_RESPONSE_OK
}

h.updateObject(hibp, email, breaches)
_, err = dbMap.Update(hibp)
if err != nil {
log.Printf("Error updating HIBP record: %v", err)
return WORK_RESPONSE_ERROR
}

return WORK_RESPONSE_OK
}

// Generic method to copy the data into the HIBP object
func (h *HaveIBeenPwned) updateObject(

hibp *HaveIBeenPwned,
email string,
breaches string) {

hibp.Email = strings.ToLower(email)
hibp.Breaches = breaches
}



Loading

0 comments on commit 593bf5e

Please sign in to comment.