Evaluation and Labeling notebooks.

Greenhost (#62)

* IPCache class added
* no counts in logging
* Full cache warning instead of exception.
* IP cache persisting
* logs message corrected with total
* Thread safe singleton ip cache
* White list fix. Challenged default zero fix.
* Banned/passed reports from banjax
* Extra logging removed
* Two ip caches: passed and pending
* Saving ip_passed cache in the file.
* start report consumer even without -e
* Banjax thread moved into AttackDetectin task
* The new version of spark-iforest
* White list ips optimized.
* White list hosts added.
* Host white listing is moved to send_challenge()
* Spark standalone 3nodes deployment procedure.
* ats_log_schema.json added. IForest removed from requirements.txt. spark_standalone.md updated.
* count() is deleted everywhere
* Spark secret. Count() removed from postprocessing.
* spark encryption configuration added
* spark ssl for ui, standalone and history config.
* Sliding window in postprocesing is optional. Set sliding_window config param to zero to disable sliding_window(defaut is also zero now)
* Markdown dependency removed due to the confict with pdoc.
* model interpretation helpers and functional shapley with anomaly model
* improve shap calculation's udf
* OriginIPs class for getting the list of original ips for whitelisting in postprocessing pipeline.
* Hot model upgrade (#65)

notebook commits

DEPLOYMENT.md

@@ -201,7 +201,7 @@ Follow an [example...](https://unix.stackexchange.com/questions/332641/how-to-in
 ```
 wget https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz
 tar xvf Python-3.6.6.tgz
-cd Python-3.6.9
+cd Python-3.6.6
 ./configure --enable-optimizations --enable-shared --prefix=/usr/local LDFLAGS="-Wl,-rpath /usr/local/lib" --with-ensurepip=install
 make -j8
 sudo make altinstall
@@ -536,7 +536,7 @@ cd ..
 ```
 git clone https://github.com/equalitie/baskerville.git
 cd baskerville
-sudo pip install -e .
+pip install -e .
 ```
 * set up a cron job for Java GC memory leak workaround
 ```

data/samples/ats_log_schema.json

@@ -0,0 +1,100 @@
+{
+   "name": "ATSLogs",
+   "properties": {
+      "@timestamp": {
+         "type": "string",
+         "format": "date",
+         "pattern": "(\\d\\d\\d\\d-([0-2])?\\d-([0-3])?\\dT?([0-2])?\\d:([0-5])?\\d:([0-5])?\\d\\.\\d?\\d?\\d?Z?)",
+         "required": true
+      },
+      "ISP": {
+         "type": "string"
+      },
+      "cache_result": {
+         "type": "string"
+      },
+      "client_ip": {
+         "type": "string",
+         "pattern": "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
+      },
+      "client_request_host": {
+         "type": "string"
+      },
+      "client_request_method": {
+         "type": "string",
+         "default": ""
+      },
+      "client_ua": {
+         "type": "string"
+      },
+      "client_url": {
+         "type": "string"
+      },
+      "client_user": {
+         "type": "string"
+      },
+      "content_type": {
+        "type": "string"
+      },
+      "device": {
+        "type": "string"
+      },
+      "dnet": {
+         "type": "string"
+      },
+      "host": {
+         "type": "string"
+      },
+      "http_request_scheme": {
+         "type": "string"
+      },
+      "http_request_version": {
+         "type": "string"
+      },
+      "http_response_code": {
+         "type": "string",
+         "pattern": "[1-5][0-9][0-9]"
+      },
+      "major": {
+         "type": "string"
+      },
+      "os": {
+         "type": "string"
+      },
+      "os_name": {
+         "type": "string"
+      },
+      "querystring": {
+         "type": "string"
+      },
+      "reply_length_bytes": {
+         "type": "string"
+      },
+      "ts_process_time": {
+         "type": "string"
+      },
+      "ts_timestamp": {
+         "type": "string"
+      },
+      "ua_name": {
+         "type": "string"
+      },
+      "geoip": {
+	  "location": {
+              "lon": {
+		  "type": "string",
+		  "format": "number"
+	      },
+	      "lat": {
+		  "type": "string",
+		  "format": "number"
+	      }
+	  },
+	  "country_name":{
+              "type": "string"
+	  }
+      }
+   },
+   "required": ["@timestamp", "client_ip", "client_request_host", "client_ua", "client_url", "content_type", "http_response_code", "querystring", "reply_length_bytes", "geoip"],
+   "additionalProperties": false
+}