From 25d3ccdcd6c4f3c9d434d89122a1e40285ceaa2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Tue, 12 Dec 2017 13:25:31 +0100
Subject: [PATCH 01/11] add example of production deployment

---
 docker-compose.demo.yml       | 24 ++++++++++++------------
 docker-compose.production.yml | 16 ++++++++++++++++
 2 files changed, 28 insertions(+), 12 deletions(-)
 create mode 100644 docker-compose.production.yml

diff --git a/docker-compose.demo.yml b/docker-compose.demo.yml
index 5bab2b44..8414f522 100644
--- a/docker-compose.demo.yml
+++ b/docker-compose.demo.yml
@@ -1,6 +1,6 @@
 version: '2'
 services:
-  kqueen:
+  api:
     image: kqueen/api:v0.11
     ports:
       - 127.0.0.1:5000:5000
@@ -17,14 +17,7 @@ services:
       BOOTSTRAP_ADMIN_NAMESPACE: demoorg
     extra_hosts:
       - "ci.mcp.mirantis.net:172.16.48.254"
-  kqueen_mail:
-    image: modularitycontainers/postfix
-    volumes:
-      - /var/spool/postfix:/var/spool/postfix
-      - /var/spool/mail:/var/spool/mail
-    environment:
-      MYHOSTNAME: 'kqueen_mail'
-  kqueen_ui:
+  ui:
     image: kqueen/ui:v0.1
     ports:
       - 127.0.0.1:5080:5080
@@ -32,9 +25,16 @@ services:
       - kqueen
     environment:
       KQUEEN_UI_CONFIG_FILE: config/prod.py
-      KQUEENUI_KQUEEN_API_URL: http://kqueen:5000/api/v1/
-      KQUEENUI_KQUEEN_AUTH_URL: http://kqueen:5000/api/v1/auth
+      KQUEENUI_KQUEEN_API_URL: http://api:5000/api/v1/
+      KQUEENUI_KQUEEN_AUTH_URL: http://api:5000/api/v1/auth
       KQUEENUI_KQUEEN_SERVICE_USER_USERNAME: admin
       KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD: default
-      KQUEENUI_MAIL_SERVER: kqueen_mail
+      KQUEENUI_MAIL_SERVER: mail
       KQUEENUI_MAIL_PORT: 10025
+  mail:
+    image: modularitycontainers/postfix
+    volumes:
+      - /var/spool/postfix:/var/spool/postfix
+      - /var/spool/mail:/var/spool/mail
+    environment:
+      MYHOSTNAME: 'mail'
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
new file mode 100644
index 00000000..daebd6d2
--- /dev/null
+++ b/docker-compose.production.yml
@@ -0,0 +1,16 @@
+version: '2'
+services:
+  etcd:
+    restart: always
+  api:
+    image: kqueen/api:v0.11
+    restart: always
+  #proxy:
+  #  images: nginx
+  #  restart: always
+  ui:
+    image: kqueen/ui:v0.11
+    restart: always
+  mail:
+    image: modularitycontainers/postfix
+    restart: always

From 8638673154d2c0548d5c0e1f950b9bba0e072af8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Mon, 18 Dec 2017 17:01:04 +0100
Subject: [PATCH 02/11] WIP: saving work

---
 docker-compose.demo.yml       | 16 +++----
 docker-compose.production.yml | 29 ++++++++++---
 nginx/Dockerfile              | 15 +++++++
 nginx/vhost.conf              | 82 +++++++++++++++++++++++++++++++++++
 4 files changed, 127 insertions(+), 15 deletions(-)
 create mode 100644 nginx/Dockerfile
 create mode 100644 nginx/vhost.conf

diff --git a/docker-compose.demo.yml b/docker-compose.demo.yml
index 8414f522..b71f84a6 100644
--- a/docker-compose.demo.yml
+++ b/docker-compose.demo.yml
@@ -22,7 +22,7 @@ services:
     ports:
       - 127.0.0.1:5080:5080
     depends_on:
-      - kqueen
+      - api
     environment:
       KQUEEN_UI_CONFIG_FILE: config/prod.py
       KQUEENUI_KQUEEN_API_URL: http://api:5000/api/v1/
@@ -31,10 +31,10 @@ services:
       KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD: default
       KQUEENUI_MAIL_SERVER: mail
       KQUEENUI_MAIL_PORT: 10025
-  mail:
-    image: modularitycontainers/postfix
-    volumes:
-      - /var/spool/postfix:/var/spool/postfix
-      - /var/spool/mail:/var/spool/mail
-    environment:
-      MYHOSTNAME: 'mail'
+#  mail:
+#    image: modularitycontainers/postfix
+#    volumes:
+#      - /var/spool/postfix:/var/spool/postfix
+#      - /var/spool/mail:/var/spool/mail
+#    environment:
+#      MYHOSTNAME: 'mail'
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index daebd6d2..e2349c37 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -5,12 +5,27 @@ services:
   api:
     image: kqueen/api:v0.11
     restart: always
-  #proxy:
-  #  images: nginx
-  #  restart: always
-  ui:
-    image: kqueen/ui:v0.11
+  proxy:
+    build: ./nginx/
     restart: always
-  mail:
-    image: modularitycontainers/postfix
+    ports:
+      - 443:443
+      - 80:80
+    volumes:
+      - ./certs/:/mnt/letsencrypt/demo.kqueen.net/:Z
+    volumes_from:
+      - ui:ro
+  ui:
+    #image: kqueen/ui:v0.1
+    image: kqueen/ui:copy-static
+    volumes:
+      - /mnt/static/
     restart: always
+#  mail:
+#    image: modularitycontainers/postfix
+#    restart: always
+#    volumes:
+#      - /var/spool/postfix
+#      - /var/spool/mail
+#    environment:
+#      MYHOSTNAME: 'mail'
diff --git a/nginx/Dockerfile b/nginx/Dockerfile
new file mode 100644
index 00000000..9e3c484b
--- /dev/null
+++ b/nginx/Dockerfile
@@ -0,0 +1,15 @@
+FROM nginx
+
+# environment
+ENV DIR_CONF /etc/nginx/conf.d/
+ENV DIR_APP /var/www/app/
+ENV VHOSTNAME demo.kqueen.net
+
+# flush nginx config
+RUN rm -v /etc/nginx/conf.d/*
+
+# copy config
+COPY vhost.conf $DIR_CONF
+
+# edit vhost.conf
+RUN sed -i "s/vhostname/$VHOSTNAME/g" $DIR_CONF/vhost.conf
diff --git a/nginx/vhost.conf b/nginx/vhost.conf
new file mode 100644
index 00000000..1ffd6a43
--- /dev/null
+++ b/nginx/vhost.conf
@@ -0,0 +1,82 @@
+# upstream app
+upstream django {
+	server ui:5080;
+}
+
+server { #default server
+	listen 80 default_server;
+	server_name _;
+
+	access_log /dev/stdout main;
+	error_log /dev/stdout info;
+
+	root /dev/null;
+}
+
+server { # http://vhostname
+	listen 0.0.0.0:80;
+	server_name vhostname;
+	return 301 https://$server_name$request_uri;
+}
+
+server { # https://vhostname
+
+	listen 0.0.0.0:443 ssl http2;
+	server_name vhostname;
+
+	access_log /dev/stdout main;
+	error_log /dev/stdout info;
+
+	ssl_certificate /mnt/letsencrypt/vhostname/fullchain.cer;
+	ssl_certificate_key /mnt/letsencrypt/vhostname/vhostname.key;
+	ssl_session_timeout 1d;
+	ssl_session_cache shared:SSL:50m;
+
+	# Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
+	#ssl_dhparam /etc/nginx/certs/marast/dhparam.pem;
+
+	# intermediate configuration. tweak to your needs.
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+	ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
+	ssl_prefer_server_ciphers on;
+
+	# HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
+	add_header Strict-Transport-Security max-age=315360000; # 10 years
+
+	# OCSP Stapling ---
+	# fetch OCSP records from URL in ssl_certificate and cache them
+	#ssl_stapling on;
+	#ssl_stapling_verify on;
+
+	## verify chain of trust of OCSP response using Root CA and Intermediate certs
+	ssl_trusted_certificate /mnt/letsencrypt/vhostname/ca.cer;
+
+	client_max_body_size 64M;
+
+	server_name _;
+
+	root /dev/null/;
+
+	location /static/ {
+		alias /mnt/static/;
+		#gzip_static on;
+		#expires 24h;
+		#add_header Cache-Control public;
+		autoindex on;
+	}
+
+	# letsencrypt validation
+	location /.well-known/ {
+		alias /mnt/letsencrypt/.well-known/;
+	}
+
+	location / {
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header Host $http_host;
+		proxy_set_header X-Forwarded-Proto https;
+		add_header Cache-Control no-cache;
+		expires -1;
+		proxy_pass http://django;
+	}
+}

From be664de99dba91723d612fc4d4196da8a628446e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Mon, 18 Dec 2017 18:52:54 +0100
Subject: [PATCH 03/11] workaround for proxy

---
 docker-compose.production.yml | 21 +++++++++++++++++++--
 nginx/vhost.conf              |  3 ++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index e2349c37..0874c65c 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -5,6 +5,15 @@ services:
   api:
     image: kqueen/api:v0.11
     restart: always
+    environment:
+      KQUEEN_CONFIG_FILE: config/prod.py
+      KQUEEN_ETCD_HOST: etcd
+      #KQUEEN_DEBUG: 'True'
+      BOOTSTRAP_ADMIN: 1
+      BOOTSTRAP_ADMIN_USERNAME: admin
+      BOOTSTRAP_ADMIN_PASSWORD: default
+      BOOTSTRAP_ADMIN_ORGANIZATION: DemoOrg
+      BOOTSTRAP_ADMIN_NAMESPACE: demoorg
   proxy:
     build: ./nginx/
     restart: always
@@ -16,8 +25,16 @@ services:
     volumes_from:
       - ui:ro
   ui:
-    #image: kqueen/ui:v0.1
-    image: kqueen/ui:copy-static
+    image: kqueen/ui:master
+    environment:
+      KQUEENUI_PREFERRED_URL_SCHEME: https
+      KQUEEN_UI_CONFIG_FILE: config/prod.py
+      KQUEENUI_KQUEEN_API_URL: http://api:5000/api/v1/
+      KQUEENUI_KQUEEN_AUTH_URL: http://api:5000/api/v1/auth
+      KQUEENUI_KQUEEN_SERVICE_USER_USERNAME: admin
+      KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD: default
+      KQUEENUI_MAIL_SERVER: mail
+      KQUEENUI_MAIL_PORT: 10025
     volumes:
       - /mnt/static/
     restart: always
diff --git a/nginx/vhost.conf b/nginx/vhost.conf
index 1ffd6a43..2f363115 100644
--- a/nginx/vhost.conf
+++ b/nginx/vhost.conf
@@ -74,8 +74,9 @@ server { # https://vhostname
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_set_header Host $http_host;
-		proxy_set_header X-Forwarded-Proto https;
+		proxy_set_header X-Forwarded-Proto $scheme;
 		add_header Cache-Control no-cache;
+		proxy_redirect http:// https://;
 		expires -1;
 		proxy_pass http://django;
 	}

From 07192770726aeb67854c4cfde6e050930ee92c5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Tue, 19 Dec 2017 10:06:21 +0100
Subject: [PATCH 04/11] add prometheus

---
 docker-compose.production.yml    | 10 +++++++++-
 {nginx => prod/nginx}/Dockerfile |  0
 {nginx => prod/nginx}/vhost.conf |  0
 prod/prometheus/prometheus.yml   |  7 +++++++
 4 files changed, 16 insertions(+), 1 deletion(-)
 rename {nginx => prod/nginx}/Dockerfile (100%)
 rename {nginx => prod/nginx}/vhost.conf (100%)
 create mode 100644 prod/prometheus/prometheus.yml

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index 0874c65c..f9022a15 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -14,8 +14,9 @@ services:
       BOOTSTRAP_ADMIN_PASSWORD: default
       BOOTSTRAP_ADMIN_ORGANIZATION: DemoOrg
       BOOTSTRAP_ADMIN_NAMESPACE: demoorg
+      KQUEEN_PROMETHEUS_WHITELIST: '172.16.238.0/24'
   proxy:
-    build: ./nginx/
+    build: ./prod/nginx/
     restart: always
     ports:
       - 443:443
@@ -46,3 +47,10 @@ services:
 #      - /var/spool/mail
 #    environment:
 #      MYHOSTNAME: 'mail'
+  prometheus:
+    image: prom/prometheus
+    restart: always
+    ports:
+      - 127.0.0.1:9090:9090
+    volumes:
+      - ./prod/prometheus/:/etc/prometheus/:Z
diff --git a/nginx/Dockerfile b/prod/nginx/Dockerfile
similarity index 100%
rename from nginx/Dockerfile
rename to prod/nginx/Dockerfile
diff --git a/nginx/vhost.conf b/prod/nginx/vhost.conf
similarity index 100%
rename from nginx/vhost.conf
rename to prod/nginx/vhost.conf
diff --git a/prod/prometheus/prometheus.yml b/prod/prometheus/prometheus.yml
new file mode 100644
index 00000000..0304c7fb
--- /dev/null
+++ b/prod/prometheus/prometheus.yml
@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 1m
+scrape_configs:
+- job_name: 'kqueen'
+  static_configs:
+  - targets:
+    - 'api:5000'

From 2185e770a83e3605957b217712fea649f7c2d7fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Tue, 19 Dec 2017 11:14:02 +0100
Subject: [PATCH 05/11] read metrics from etcd

---
 docker-compose.production.yml         |  27 +++--
 kqueen/config/prod.py                 |   4 +-
 prod/prometheus/etcd3_alert.rules.yml | 143 ++++++++++++++++++++++++++
 prod/prometheus/prometheus.yml        |  21 +++-
 4 files changed, 177 insertions(+), 18 deletions(-)
 create mode 100644 prod/prometheus/etcd3_alert.rules.yml

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index f9022a15..cc077460 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -8,13 +8,14 @@ services:
     environment:
       KQUEEN_CONFIG_FILE: config/prod.py
       KQUEEN_ETCD_HOST: etcd
-      #KQUEEN_DEBUG: 'True'
+      KQUEEN_PROMETHEUS_WHITELIST: '172.16.238.0/24'
+      # TODO: set SECRET_KEY
+      KQUEEN_SECRET_KEY: ''
       BOOTSTRAP_ADMIN: 1
       BOOTSTRAP_ADMIN_USERNAME: admin
       BOOTSTRAP_ADMIN_PASSWORD: default
       BOOTSTRAP_ADMIN_ORGANIZATION: DemoOrg
       BOOTSTRAP_ADMIN_NAMESPACE: demoorg
-      KQUEEN_PROMETHEUS_WHITELIST: '172.16.238.0/24'
   proxy:
     build: ./prod/nginx/
     restart: always
@@ -22,7 +23,7 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - ./certs/:/mnt/letsencrypt/demo.kqueen.net/:Z
+      - /mnt/storage/kqueen/certs/:/mnt/letsencrypt/demo.kqueen.net/:Z
     volumes_from:
       - ui:ro
   ui:
@@ -36,17 +37,18 @@ services:
       KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD: default
       KQUEENUI_MAIL_SERVER: mail
       KQUEENUI_MAIL_PORT: 10025
+      STATIC_DIR: /mnt/static/
     volumes:
       - /mnt/static/
     restart: always
-#  mail:
-#    image: modularitycontainers/postfix
-#    restart: always
-#    volumes:
-#      - /var/spool/postfix
-#      - /var/spool/mail
-#    environment:
-#      MYHOSTNAME: 'mail'
+  mail:
+    image: modularitycontainers/postfix
+    restart: always
+    volumes:
+      - /var/spool/postfix/
+      - /var/spool/mail/
+    environment:
+      MYHOSTNAME: 'mail'
   prometheus:
     image: prom/prometheus
     restart: always
@@ -54,3 +56,6 @@ services:
       - 127.0.0.1:9090:9090
     volumes:
       - ./prod/prometheus/:/etc/prometheus/:Z
+    links:
+      - api
+      - etcd
diff --git a/kqueen/config/prod.py b/kqueen/config/prod.py
index aeb168ed..daa551f5 100644
--- a/kqueen/config/prod.py
+++ b/kqueen/config/prod.py
@@ -7,8 +7,8 @@ class Config(BaseConfig):
 
     KQUEEN_HOST = '0.0.0.0'
 
-    # App secret
-    SECRET_KEY = 'secret'
+    # App secret - set this to random string >= 16 chars
+    #SECRET_KEY = 'secret'
 
     # Jenkins engine settings
     JENKINS_API_URL = 'https://ci.mcp.mirantis.net'
diff --git a/prod/prometheus/etcd3_alert.rules.yml b/prod/prometheus/etcd3_alert.rules.yml
new file mode 100644
index 00000000..e68f1a42
--- /dev/null
+++ b/prod/prometheus/etcd3_alert.rules.yml
@@ -0,0 +1,143 @@
+groups:
+- name: etcd3_alert.rules
+  rules:
+  - alert: InsufficientMembers
+    expr: count(up{job="etcd"} == 0) > (count(up{job="etcd"}) / 2 - 1)
+    for: 3m
+    labels:
+      severity: critical
+    annotations:
+      description: If one more etcd member goes down the cluster will be unavailable
+      summary: etcd cluster insufficient members
+  - alert: NoLeader
+    expr: etcd_server_has_leader{job="etcd"} == 0
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      description: etcd member {{ $labels.instance }} has no leader
+      summary: etcd member has no leader
+  - alert: HighNumberOfLeaderChanges
+    expr: increase(etcd_server_leader_changes_seen_total{job="etcd"}[1h]) > 3
+    labels:
+      severity: warning
+    annotations:
+      description: etcd instance {{ $labels.instance }} has seen {{ $value }} leader
+        changes within the last hour
+      summary: a high number of leader changes within the etcd cluster are happening
+  - alert: HighNumberOfFailedGRPCRequests
+    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed
+        on etcd instance {{ $labels.instance }}'
+      summary: a high number of gRPC requests are failing
+  - alert: HighNumberOfFailedGRPCRequests
+    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed
+        on etcd instance {{ $labels.instance }}'
+      summary: a high number of gRPC requests are failing
+  - alert: GRPCRequestsSlow
+    expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le))
+      > 0.15
+    for: 10m
+    labels:
+      severity: critical
+    annotations:
+      description: on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method
+        }} are slow
+      summary: slow gRPC requests
+  - alert: HighNumberOfFailedHTTPRequests
+    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
+      BY (method) > 0.01
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
+        instance {{ $labels.instance }}'
+      summary: a high number of HTTP requests are failing
+  - alert: HighNumberOfFailedHTTPRequests
+    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
+      BY (method) > 0.05
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
+        instance {{ $labels.instance }}'
+      summary: a high number of HTTP requests are failing
+  - alert: HTTPRequestsSlow
+    expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
+      > 0.15
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method
+        }} are slow
+      summary: slow HTTP requests
+  - record: instance:fd_utilization
+    expr: process_open_fds / process_max_fds
+  - alert: FdExhaustionClose
+    expr: predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust
+        its file descriptors soon'
+      summary: file descriptors soon exhausted
+  - alert: FdExhaustionClose
+    expr: predict_linear(instance:fd_utilization[10m], 3600) > 1
+    for: 10m
+    labels:
+      severity: critical
+    annotations:
+      description: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust
+        its file descriptors soon'
+      summary: file descriptors soon exhausted
+  - alert: EtcdMemberCommunicationSlow
+    expr: histogram_quantile(0.99, rate(etcd_network_member_round_trip_time_seconds_bucket[5m]))
+      > 0.15
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: etcd instance {{ $labels.instance }} member communication with
+        {{ $labels.To }} is slow
+      summary: etcd member communication is slow
+  - alert: HighNumberOfFailedProposals
+    expr: increase(etcd_server_proposals_failed_total{job="etcd"}[1h]) > 5
+    labels:
+      severity: warning
+    annotations:
+      description: etcd instance {{ $labels.instance }} has seen {{ $value }} proposal
+        failures within the last hour
+      summary: a high number of proposals within the etcd cluster are failing
+  - alert: HighFsyncDurations
+    expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))
+      > 0.5
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: etcd instance {{ $labels.instance }} fync durations are high
+      summary: high fsync durations
+  - alert: HighCommitDurations
+    expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m]))
+      > 0.25
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      description: etcd instance {{ $labels.instance }} commit durations are high
+      summary: high commit durations
diff --git a/prod/prometheus/prometheus.yml b/prod/prometheus/prometheus.yml
index 0304c7fb..4cfd5ed7 100644
--- a/prod/prometheus/prometheus.yml
+++ b/prod/prometheus/prometheus.yml
@@ -1,7 +1,18 @@
 global:
-  scrape_interval: 1m
+  scrape_interval: '30s'
+
+rule_files:
+  - '/etc/prometheus/*.rules.yml'
+
 scrape_configs:
-- job_name: 'kqueen'
-  static_configs:
-  - targets:
-    - 'api:5000'
+  - job_name: 'kqueen'
+    static_configs:
+      - targets: ['api:5000']
+
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'etcd'
+    static_configs:
+      - targets: ['etcd:4001']

From 77ae299ab6cac925dddc3401ae51f8a60b427ae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Thu, 21 Dec 2017 09:21:43 +0100
Subject: [PATCH 06/11] add flask shell to readme

---
 README.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.rst b/README.rst
index 90f28711..c2d033ec 100644
--- a/README.rst
+++ b/README.rst
@@ -39,6 +39,14 @@ Development
     etcdctl rm --recursive /kqueen
     ./devenv.py
 
+- Run flask shell
+
+::
+
+  export FLASK_APP=kqueen.server
+  export prometheus_multiproc_dir=$(mktemp -d)
+  flask shell
+
 Demo environment
 ----------------
 

From 6d42b1fa0ebdefd655f6b32f3e3447408fd613fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?=
 <tomkukral@users.noreply.github.com>
Date: Thu, 21 Dec 2017 09:54:47 +0100
Subject: [PATCH 07/11] update settings to match with latest demo env

---
 docker-compose.production.yml | 18 +++++++++++++-----
 kqueen/config/base.py         |  2 ++
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index cc077460..0e6a997c 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -2,8 +2,10 @@ version: '2'
 services:
   etcd:
     restart: always
+    volumes:
+      - /0.etcd/:/mnt/storage/kqueen/etcd/
   api:
-    image: kqueen/api:v0.11
+    image: kqueen/api:v0.14
     restart: always
     environment:
       KQUEEN_CONFIG_FILE: config/prod.py
@@ -13,9 +15,12 @@ services:
       KQUEEN_SECRET_KEY: ''
       BOOTSTRAP_ADMIN: 1
       BOOTSTRAP_ADMIN_USERNAME: admin
-      BOOTSTRAP_ADMIN_PASSWORD: default
+      # TODO: set admin password
+      BOOTSTRAP_ADMIN_PASSWORD:
       BOOTSTRAP_ADMIN_ORGANIZATION: DemoOrg
       BOOTSTRAP_ADMIN_NAMESPACE: demoorg
+    volumes:
+      - ./kqueen/config/prod.py:/code/kqueen/config/prod.py
   proxy:
     build: ./prod/nginx/
     restart: always
@@ -23,20 +28,22 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - /mnt/storage/kqueen/certs/:/mnt/letsencrypt/demo.kqueen.net/:Z
+      - /mnt/storage/kqueen/certs/:/mnt/letsencrypt/demo.kqueen.net/:ro
     volumes_from:
       - ui:ro
   ui:
-    image: kqueen/ui:master
+    image: kqueen/ui:v0.2
     environment:
       KQUEENUI_PREFERRED_URL_SCHEME: https
       KQUEEN_UI_CONFIG_FILE: config/prod.py
       KQUEENUI_KQUEEN_API_URL: http://api:5000/api/v1/
       KQUEENUI_KQUEEN_AUTH_URL: http://api:5000/api/v1/auth
       KQUEENUI_KQUEEN_SERVICE_USER_USERNAME: admin
-      KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD: default
+      # TODO: set same password as in api BOOTSTRAP_ADMIN_PASSWORD
+      KQUEENUI_KQUEEN_SERVICE_USER_PASSWORD:
       KQUEENUI_MAIL_SERVER: mail
       KQUEENUI_MAIL_PORT: 10025
+      KQUEENUI_ENABLE_PUBLIC_REGISTRATION: 1
       STATIC_DIR: /mnt/static/
     volumes:
       - /mnt/static/
@@ -56,6 +63,7 @@ services:
       - 127.0.0.1:9090:9090
     volumes:
       - ./prod/prometheus/:/etc/prometheus/:Z
+      - /mnt/storage/kqueen/prometheus/:/prometheus/
     links:
       - api
       - etcd
diff --git a/kqueen/config/base.py b/kqueen/config/base.py
index 3a79ade5..8401b62f 100644
--- a/kqueen/config/base.py
+++ b/kqueen/config/base.py
@@ -32,6 +32,8 @@ class BaseConfig:
     PROVISIONER_OK_STATE = 'OK'
     PROVISIONER_UNKNOWN_STATE = 'Not Reachable'
 
+    PROVISIONER_ENGINE_WHITELIST = []
+
     PROMETHEUS_WHITELIST = '127.0.0.0/8'
 
     @classmethod

From 1037c5b5599bf8909ebb0842d0cd369101a1332a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?= <tkukral@mirantis.com>
Date: Fri, 22 Dec 2017 17:22:34 +0100
Subject: [PATCH 08/11] fix pep8

---
 kqueen/config/prod.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kqueen/config/prod.py b/kqueen/config/prod.py
index daa551f5..21429d83 100644
--- a/kqueen/config/prod.py
+++ b/kqueen/config/prod.py
@@ -8,7 +8,7 @@ class Config(BaseConfig):
     KQUEEN_HOST = '0.0.0.0'
 
     # App secret - set this to random string >= 16 chars
-    #SECRET_KEY = 'secret'
+    # SECRET_KEY = 'secret'
 
     # Jenkins engine settings
     JENKINS_API_URL = 'https://ci.mcp.mirantis.net'

From 24f9dca5c95679ab393b034d5915e3ad96559a42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?= <tkukral@mirantis.com>
Date: Fri, 22 Dec 2017 17:30:48 +0100
Subject: [PATCH 09/11] dont install dependencie for docker-compose tests

---
 .travis.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2497be0b..f6e179a9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,6 +39,8 @@ jobs:
     - sleep 2
     - wget -O - http://localhost:5000/api/v1/health
   - stage: build
+    install:
+    - /bin/true
     script:
     - export IMAGE="kqueen/api:${TRAVIS_BRANCH/\//-}"
     - echo "Building image $IMAGE"
@@ -47,6 +49,8 @@ jobs:
     - docker push $IMAGE
     - docker logout
   - stage: test
+    install:
+    - /bin/true
     script:
     - docker-compose -f docker-compose.etcd-volume.yml up -d
     - docker-compose -f docker-compose.etcd-volume.yml exec etcd etcdctl mkdir /travis/
@@ -62,8 +66,10 @@ jobs:
         echo "$VALUE == $TRAVIS_COMMIT";
         [ "$VALUE" == "$TRAVIS_COMMIT" ] && exit 0 || exit 1
   - stage: publish
+    install:
+    - /bin/true
     script:
-    - "/bin/true"
+    - /bin/true
     deploy:
       provider: pypi
       user: tomkukral

From 88dd2572e3caa1cea5d5146c58d0f82a3d608d2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?= <tkukral@mirantis.com>
Date: Fri, 22 Dec 2017 17:38:55 +0100
Subject: [PATCH 10/11] speedup flake8 test

---
 .travis.yml | 2 +-
 setup.py    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f6e179a9..ea890554 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ jobs:
   include:
   - stage: test
     install:
-    - pip install -e ".[dev]"
+    - pip install -e ".[test]"
     script:
     - python3 -m flake8
   - stage: test
diff --git a/setup.py b/setup.py
index f4762a93..0312fa07 100644
--- a/setup.py
+++ b/setup.py
@@ -55,6 +55,7 @@
     ],
     tests_require=test_require,
     extras_require={
+        'test': test_require,
         'dev': test_require + [
             'ipython',
             'sphinx',

From 97893c43390ebf5574c32610b50179dda6725ae9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Kukr=C3=A1l?= <tkukral@mirantis.com>
Date: Fri, 22 Dec 2017 17:45:44 +0100
Subject: [PATCH 11/11] don't limit provisioner engines in default config

---
 kqueen/config/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kqueen/config/base.py b/kqueen/config/base.py
index bae98817..e424db9c 100644
--- a/kqueen/config/base.py
+++ b/kqueen/config/base.py
@@ -37,7 +37,7 @@ class BaseConfig:
     PROVISIONER_OK_STATE = 'OK'
     PROVISIONER_UNKNOWN_STATE = 'Not Reachable'
 
-    PROVISIONER_ENGINE_WHITELIST = []
+    PROVISIONER_ENGINE_WHITELIST = None
 
     PROMETHEUS_WHITELIST = '127.0.0.0/8'