Stage

README.md

@@ -176,7 +176,7 @@ aegis -config app.json -cpath "path to configuration file"
 ### Jobs involved:
 1. RescanQueueJob
 
-   Starts automatically and runs continuously. This job monitors JIRA and kicks off a RescanJob for tickets in particular statuses. There are four types of RSQ:
+   Starts automatically and runs continuously. This job monitors JIRA and kicks off a RescanJob for tickets in particular statuses. There are four types of RSQ (the type being controlled by the Payload in the JobHistory entry for the RSQ):
    1. Normal - looks tickets in a Resolved-Remediated status. These are for standard vulnerability rescans. Tickets are moved to Closed-Remediated once remediation has been confirmed by a scanner, or reopened if the scanner still detects the vulnerability.
    2. Decommission - looks for tickets in a Resolved-Decommission status. These are for confirming a device has been moved offline. These tickets are moved to Closed-Decommissioned once a scanner has confirmed they are a dead host, or reopened if the host is still alive.
    3. Passive - looks for tickets in an open, reopened, in-progress, resolved-exception status created within 20 days and due within 15 days
@@ -221,7 +221,7 @@ aegis -config app.json -cpath "path to configuration file"
     Pulls recently updated tickets from JIRA and keeps the DB information on the ticket fresh
 5. AssetSyncJob
 
-    Pulls detection information from Qualys/Nexpose API and stores it in db. Uses the vulnerability information information pulled and stored during the VulnSyncJob. Also stores device information in DB. Looks at Ignore table and attaches it to detection if appropriate. Global exceptions are also checked here. Detections created/updated here are what are processed during a ticketing run
+    Pulls detection information from Qualys/Nexpose API and stores it in db. Uses the vulnerability information pulled and stored during the VulnSyncJob. Also stores device information in DB. Looks at Ignore table and attaches it to detection if appropriate. Global exceptions are also checked here. Detections created/updated here are what are processed during a ticketing run
 
 ## Ticketing Process
 ### Jobs involved:

cmd/aegis.go

@@ -57,7 +57,7 @@ func main() {
 			var dbconn domain.DatabaseConnection
 			if dbconn, err = database.NewConnection(appConfig); err == nil {
 
-				var orgIDToOrgCode map[string]string
+				var orgIDToOrgCode map[string]domain.Organization
 				if orgIDToOrgCode, err = getOrgMap(dbconn); err == nil {
 					var logger log.Logger
 					if logger, err = log.NewLogStream(ctx, dbconn, appConfig); err == nil {
@@ -97,18 +97,18 @@ func main() {
 	}
 }
 
-func getOrgMap(dbconn domain.DatabaseConnection) (orgIDToCode map[string]string, err error) {
-	orgIDToCode = make(map[string]string)
+func getOrgMap(dbconn domain.DatabaseConnection) (orgIDToOrg map[string]domain.Organization, err error) {
+	orgIDToOrg = make(map[string]domain.Organization)
 	var orgs []domain.Organization
 	if orgs, err = dbconn.GetOrganizations(); err == nil {
 		for _, org := range orgs {
-			orgIDToCode[org.ID()] = org.Code()
+			orgIDToOrg[org.ID()] = org
 		}
 	} else {
 		err = fmt.Errorf("error while caching organizations - %s", err.Error())
 	}
 
-	return orgIDToCode, err
+	return orgIDToOrg, err
 }
 
 func populateAutoStartJobs(dbconn domain.DatabaseConnection) (err error) {

docs/faq.txt

@@ -0,0 +1,53 @@
+FAQ
+
+Q) How are jobs started?
+A) The JobHistory table is checked periodically for an entry with a StatusID of 1. If one is found, a job is created. There are several ways for a JobHistory entry to be created
+1) Several jobs (RescanQueueJob/ExceptionJob) are marked as AutoStart by default. All job configurations are controlled in the JobConfig table. The AutoStart functionality can be enabled/disabled by changing the AutoStart field in this table. If a JobConfig has its AutoStart field marked as 1, a JobHistory entry will be created with the job configuration every time Aegis starts.
+2) The JobSchedule table holds JobConfig IDs, as well as the days/times that the job should be scheduled for. Once that time/day arrives, a job history entry will be created for the job configuration
+3) A JobHistory can be created by hand using a SQL entry. This can be done with the following SQL command
+
+INSERT INTO `VM`.`JobHistory` (`JobId`, `ConfigID`, `StatusId`, `Priority`, `Payload`)
+                VALUES
+(
+                (select ID from Job where Struct = 'TicketingJob'),
+                (select ID from JobConfig where JobID = (select ID from Job where Struct = 'TicketingJob' LIMIT 1) LIMIT 1),
+                '1',
+                '0',
+                (select Payload from JobConfig where JobID = (select ID from Job where Struct = 'TicketingJob') LIMIT 1)
+);
+
+OR if you want to create a JobHistory using a specific ConfigID you can use the following
+INSERT INTO `VM`.`JobHistory` (`JobId`, `ConfigID`, `StatusId`, `Priority`, `Payload`) select JobID, ID, 1, 0, Payload from JobConfig where ID = 'YOUR JobConfig ID HERE';
+
+If you have more than one Organization, or more than one JobConfig entry for a single job, you'll have to specify which exact JobConfig you're using.
+
+Q) How are tickets created?
+A) Aegis saves your vulnerability detection information in its database during the AssetSyncJob (the Detection table). The Detection table is used by the TicketingJob to create the tickets. There is no use running an AssetSyncJob on a schedule for Nexpose, as Nexpose doesn't tell you when the detection was updated, so you can't only process recently updated detections. I recommend you run an AssetSyncJob right before you do your monthly ticketing. The way you run an AssetSyncJob is exactly the same you create a TicketingJob – you just replace the string "TicketingJob" with "AssetSyncJob". For Qualys, I recommend you create a JobSchedule to run AssetSyncs daily, as it tells you when the detection was last updated, so stale detections can be skipped
+
+Q) What if a device goes offline? How does Aegis handle its tickets?
+A) If you mark a ticket as Resolved-Decommissioned, Aegis should kickoff a Decommission scan against the assets. Once the scan finishes, Aegis will mark down all devices that were discovered to be offline (in the Ignore table) to prevent the asset from being ticketed again
+
+Q) My JIRA/Nexpose/Qualys instance is getting hit too hard. How can I make Aegis make API calls less frequently?
+A) The rate at which API requests are made against an API, as well as the amount of concurrent API requests that are allowed to be made against an API are controlled in the SourceConfig entry in the database corresponding to the API connection (all API connection information is stored in the SourceConfig table)
+If you execute the following SQL query
+select AuthInfo from SourceConfig where Source = 'JIRA';
+
+You'll see this column has two fields: Delay and Concurrency. The Delay field is a number in nanoseconds that controls how long is waited between API calls. The Concurrency field is a number that controls the amount of concurrent API requests are allowed
+
+
+Q) What is the purpose of printing the goroutine count in the JobRunner? Ex: Job Runner: Sleeping for 60 seconds - [1250 goroutines]
+A) This is a count of goroutines that the entire Aegis system uses, and is not reflective of the amount of workers. The baseline should hover at around 1200, and this log is used to detect goroutine leakage. If this baseline number ever starts to steadily increase as Aegis is allowed to run over time, there's a goroutine leak.
+
+Q) Why do certain jobs restart after finishing?
+A) Most AutoStart jobs are ALSO marked Continuous in the same JobConfig table. If the Continuous field is marked, the job will wait for an amount of seconds depending on the WaitInSeconds field in the JobConfig before executing again
+
+Q) How do I change the frequency that these Continuous jobs execute?
+A) The default time between job runs for continuous jobs are 60 seconds. If you'd like to change it, you can modify the WaitInSeconds field in the JobConfig table. You can use the following command
+UPDATE JobConfig Set WaitInSeconds = 120 where ID = (select ConfigID from JobHistory where ID = 'PUT JOB ID HERE');
+
+Q) How do I cancel a job?
+A) You can cancel a job by setting its status to -1. For example, the following SQL query could be used
+Update JobHistory where ID = 'PUT JOB ID HERE' set StatusID = -1;
+You can find a job ID by looking at any log from the job, for example, in the following log
+2020-04-21 17:18:30 | [INFO] [TicketingJob:NORTON:AAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA] - Ticket created for vulnerability
+The JobHistory ID for the job that produced this log would be AAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAA

docs/jira_ticket_schema.csv

@@ -1,39 +1,46 @@
-Field Name, Field Type
-Epic Name,Name of Epic
-Method of Discovery,Select List (single choice)
-Summary,System field
-Hostname,Text Field (single line)
-IP Address,Text Field (single line)
-MAC Address,Text Field (single line)
-Service Port,Text Field (single line)
-Description,System field
-Criticality,Select List (single choice)
-Scan/Alert Date,Date Picker
-Assets Affected,Text Field (multi-line)
-Assignee,System field
-ScanID,Text Field (single line)
-Assignment Group,Group Picker (single group)
-CVE References,Text Field (multi-line)
-Resolution Date,Date Time Picker
-Due Date,System field
-Operating System,Select List (single choice)
-Vulnerablility,Text Field (single line)
-cvss_score,Select List (single choice)
-GroupID,Text Field (single line)
-DeviceID,Text Field (single line)
-VulnerabilityID,Text Field (single line)
-Org,Text Field (single line)
-AutomationID,Text Field (single line)
-Reopen Reason,Text Field (multi-line)
-VendorRef,Text Field (multi-line)
-Linked Exception,Query issue linker custom field
-Last Rescan,Date Time Picker
-Reference,URL Field
-Latest Detection Date,Date Picker
-OS_Detailed,Text Field (single line)
-Solution,Text Field (multi-line)
-CloudID,Text Field (single line)
-Config,Select List (single choice)
-LastChecked,Date Picker
-Application,Text Field (single line)
-Application Type,Select List (single choice)
+"Method of Discovery" - dropdown (None,Nexpose,Internal Discovery,External Discovery,Penetration Test,Vendor Notice,Server Build,Qualys,SynAck,Security Scorecard,BitSight,Bugcrowd,Dome9,PCI ASV,Stacks for Hacks,Aqua,Black Duck,CloudView)
+"Status" - dropdown (Open, Reopened, In-Progress, Resolved-FalsePositive, Resolved-Exception, Resolved-Remediated, Resolved-Decommissioned, Approved-Exception, Approved-False-Positive, Closed-Decommission, Closed-Remediated, Scan-Error, Closed-NA, Closed-Error, Closed-CERF)
+"created" - datetime
+"Summary" - string
+"Hostname" - string
+"IP Address" - string
+"MAC Address" - string
+"Service Port" - string
+"Description" - textbox
+"Solution" - textbox
+"VRR Priority" - drowndown (None,Immediate,Critical,High,Medium,Low)
+"LastFound" - datetime
+"Assignment Group" - string
+"Resolution Date" - datetime
+"Reopen Reason" - textbox
+"Operating System" - dropdown (None,BIG-IP,Data ONTAP,embedded,IOS,Linux,MAC OS X,Solaris,VMware ESX/ESXi,Windows,z/OS,Unknown,None,AIX,Amazon Linux,Redhat,CentOS,Cisco,CoreOS,NetApp,Oracle,UNIX,Ubuntu,FreeBSD)
+"Vulnerability" - string
+"cvss_score" - float
+"VulnerabilityID" - string
+"OWASP" - dropdown (None,Injection,Broken Authentication,Sensitive Data Exposure,XML External Entities,Broken Access Control,Security Misconfiguration,Cross-site Scripting XSS,Insecure Deserialization,Using Components with known vulnerabilities,Insufficient Logging & Monitoring)
+"Exception Date" - datetime
+"GroupID" - string
+"DeviceID" - string
+"Patchable" - dropdown (None,Yes,No)
+"ScanID" - string
+"Org" - string
+"SystemName" - string
+"CERF" - string
+"Exception Expiration" - datetime
+"CVE References" - string
+"VendorRef" - string
+"Updated" - datetime
+"OS_Detailed" - string
+"VRR Category" - dropdown (None,Security Logging,Network Security,Key Management,Asset Classification,User Access Control,Host,Malware,Third Party Application,Database,Firewall,Forensics,General remote services,Hardware,Information gathering,OEL,Configuration,Web Application,Function,Container)
+"Config" - string
+"LastChecked" - datetime
+"CloudID" - string
+"Application Name" - string
+"Hub Project Name" - string
+"Hub Project Version" - string
+"Hub Severity" - string
+"Component Name" - string
+"Component Version" - string
+"Policy Rule" - string
+"Policy Severity" - string
+"Application Type" - dropdown (None,Web,API,Mobile-iOS,Mobile-Android,Desktop,Other)

go.mod

@@ -8,21 +8,19 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1
-	github.com/hashicorp/vault/api v1.0.4 // indirect
 	github.com/nortonlifelock/aws v1.0.0
-	github.com/nortonlifelock/azure v1.0.1-0.20200127180045-bed3edc24747
+	github.com/nortonlifelock/azure v1.0.1-0.20200923201949-b266eecf6d03
 	github.com/nortonlifelock/connection v1.0.1-0.20200116160344-4d492cf1f581
 	github.com/nortonlifelock/crypto v1.0.1-0.20200213175907-562e87a94143
-	github.com/nortonlifelock/database v1.0.1-0.20200131202803-7ada08c09229
-	github.com/nortonlifelock/domain v1.0.1-0.20200402220235-483467f871ec
-	github.com/nortonlifelock/dome9 v1.0.0
+	github.com/nortonlifelock/domain v1.0.1-0.20201112203103-c6f2cc0a6b9d
+	github.com/nortonlifelock/dome9 v1.0.1-0.20201110210803-19f1f4e6909d
 	github.com/nortonlifelock/files v1.0.1-0.20200127165427-5178f1323f54
-	github.com/nortonlifelock/jira v1.0.1-0.20200323180417-150bbc4b11c1
+	github.com/nortonlifelock/jira v1.0.1-0.20201113183101-71935beca9f6
 	github.com/nortonlifelock/jira-tool v1.0.0
-	github.com/nortonlifelock/job v1.0.1-0.20200127165455-ca80a64d226c
-	github.com/nortonlifelock/log v1.0.1-0.20200129171320-c4a4dd839ed8
-	github.com/nortonlifelock/nexpose v1.0.1-0.20200330180420-43c658c76465
-	github.com/nortonlifelock/qualys v1.0.1-0.20200402232932-891d582341f2
+	github.com/nortonlifelock/job v1.0.1-0.20200723165359-98431175f14c
+	github.com/nortonlifelock/log v1.0.1-0.20200723215351-a71254d3c335
+	github.com/nortonlifelock/nexpose v1.0.1-0.20200716202017-2fd71616e68b
+	github.com/nortonlifelock/qualys v1.0.1-0.20201112222231-962c2bde903f
 	github.com/nortonlifelock/scaffold v1.0.1-0.20200128220520-41da8a42d6d5
 	github.com/pkg/errors v0.9.1
 	github.com/rs/cors v1.7.0