Merge pull request #6 from camilb/kubernetes-1.6-support

Add Kubernetes 1.6 support.

README.md

@@ -31,13 +31,13 @@ _____________________________________________________________________
 
 Clone repository
 
-    git clone github.com/camilb/prometheus-kubernetes && cd prometehus-kubernetes/definitions
+    git clone github.com/camilb/prometheus-kubernetes && cd prometehus-kubernetes
 
 Change these values in `init.sh`.
 
-`GRAFANA_VERSION=4.1.0-beta1`
+`GRAFANA_VERSION=4.3.0`
 
-`PROMETHEUS_VERSION=v1.4.1`
+`PROMETHEUS_VERSION=v1.6.3`
 
 `DOCKER_USER=your_dockerhub_user`
 

definitions/cleanup.sh → cleanup.sh

@@ -1,6 +1,9 @@
 git checkout  k8s/ingress/01-basic-auth.secret.yaml
 git checkout  k8s/prometheus/01-prometheus.configmap.yaml
+git checkout  k8s/prometheus/02-prometheus.svc.statefulset.yaml
 git checkout  k8s/prometheus/03-alertmanager.configmap.yaml
+git checkout  k8s/prometheus/04-alertmanager.svc.deployment.yaml
+git checkout  k8s/prometheus/05-node-exporter.svc.daemonset.yaml
 git checkout  k8s/grafana/grafana.svc.deployment.yaml
+git checkout  grafana/Dockerfile
 rm auth
-rm dhparam.pem

definitions/grafana/Dockerfile → grafana/Dockerfile

@@ -1,4 +1,4 @@
-FROM grafana/grafana:4.1.1
+FROM grafana/grafana:GRAFANA_VERSION
 MAINTAINER Camil Blanaru <[email protected]>
 
 ADD grafana-config/grafana.ini /etc/grafana/grafana.ini

definitions/init.sh → init.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
-GRAFANA_DEFAULT_VERSION=4.2.0
+GRAFANA_DEFAULT_VERSION=4.3.0
 PROMETHEUS_DEFAULT_VERSION=v1.6.3
+ALERT_MANAGER_DEFAULT_VERSION=v0.7.0-rc.0
+NODE_EXPORTER_DEFAULT_VERSION=v0.14.0
 DOCKER_USER_DEFAULT=$(docker info|grep Username:|awk '{print $2}')
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -25,6 +27,17 @@ echo
 read -p "Enter Prometheus version [$PROMETHEUS_DEFAULT_VERSION]: " PROMETHEUS_VERSION
 PROMETHEUS_VERSION=${PROMETHEUS_VERSION:-$PROMETHEUS_DEFAULT_VERSION}
 
+#Ask for alertmanager version or apply default
+echo
+read -p "Enter Alert Manager version [$ALERT_MANAGER_DEFAULT_VERSION]: " ALERT_MANAGER_VERSION
+ALERT_MANAGER_VERSION=${ALERT_MANAGER_VERSION:-$ALERT_MANAGER_DEFAULT_VERSION}
+
+
+#Ask for node exporter version or apply default
+echo
+read -p "Enter Node Exporter version [$NODE_EXPORTER_DEFAULT_VERSION]: " NODE_EXPORTER_VERSION
+NODE_EXPORTER_VERSION=${NODE_EXPORTER_VERSION:-$NODE_EXPORTER_DEFAULT_VERSION}
+
 #Ask for dockerhub user or apply default of the current logged-in username
 echo
 read -p "Enter Dockerhub username [$DOCKER_USER_DEFAULT]: " DOCKER_USER
@@ -87,13 +100,13 @@ tput sgr0
 echo
 if [ ! -z $AWS_ACCESS_KEY_ID ] && [ ! -z $AWS_SECRET_ACCESS_KEY ]; then
   aws_access_key=$AWS_ACCESS_KEY_ID
-  aws_access_password=$AWS_SECRET_ACCESS_KEY
+  aws_secret_key=$AWS_SECRET_ACCESS_KEY
   echo -e "${ORANGE}AWS_ACCESS_KEY_ID found, using $aws_access_key."
   tput sgr0
   echo
 elif [ ! -z $AWS_ACCESS_KEY ] && [ ! -z $AWS_SECRET_KEY ]; then
   aws_access_key=$AWS_ACCESS_KEY
-  aws_access_password=$AWS_SECRET_KEY
+  aws_secret_key=$AWS_SECRET_KEY
   echo -e "${ORANGE}AWS_ACCESS_KEY found, using $aws_access_key."
   tput sgr0
   echo
@@ -129,15 +142,15 @@ else
           break
       fi
       prompt='*'
-      aws_access_password+="$char"
+      aws_secret_key+="$char"
   done
   echo
 fi
 
 #sed in the AWS credentials. this looks odd because aws secret access keys can have '/' as a valid character
 #so we use ',' as a delimiter for sed, since that won't appear in the secret key
 sed -i -e 's/aws_access_key/'"$aws_access_key"'/g' k8s/prometheus/01-prometheus.configmap.yaml
-sed -i -e 's,aws_access_password,'"$aws_access_password"',g' k8s/prometheus/01-prometheus.configmap.yaml
+sed -i -e 's,aws_secret_key,'"$aws_secret_key"',g' k8s/prometheus/01-prometheus.configmap.yaml
 
 #slack channel
 echo -e "${PURPLE}Insert your slack channel name where you wish to receive alerts and press [ENTER]:"
@@ -156,21 +169,34 @@ do
 done
 echo
 sed -i -e 's/slack_channel/'"$slack_channel"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+echo
+
+read -r -p "Is the RBAC plugin enabled? [y/N] " response
+if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
+then
+    kubectl create -f ./k8s/rbac
+    sed -i -e 's/default/'prometheus'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml
+else
+    echo -e "${GREEN}Skipping RBAC configuration."
+    tput sgr0
+fi
 
+#set prometheus version
+sed -i -e 's/PROMETHEUS_VERSION/'"$PROMETHEUS_VERSION"'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml
 
-#remove  "sed" generated files
-rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e
+#set grafana version
+sed -i -e 's/GRAFANA_VERSION/'"$GRAFANA_VERSION"'/g' grafana/Dockerfile
+sed -i -e 's/GRAFANA_VERSION/'"$GRAFANA_VERSION"'/g' k8s/grafana/grafana.svc.deployment.yaml
 
-echo
+#set alertmanager version
+sed -i -e 's/ALERT_MANAGER_VERSION/'"$ALERT_MANAGER_VERSION"'/g' k8s/prometheus/04-alertmanager.svc.deployment.yaml
 
-#nginx load balancer display errors if dhparam is not set
-echo -e "${RED}Generate DH parameters for nginx."
-openssl dhparam -out dhparam.pem 1024
-tput sgr0
+#set node-exporter version
+sed -i -e 's/NODE_EXPORTER_VERSION/'"$NODE_EXPORTER_VERSION"'/g' k8s/prometheus/05-node-exporter.svc.daemonset.yaml
 
-echo -e "${BLUE}Create dhparam secret."
-tput sgr0
-kubectl create secret generic dhparam --from-file=dhparam.pem -n monitoring
+
+#remove  "sed" generated files
+rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e && rm grafana/*-e
 
 echo
 
@@ -210,7 +236,7 @@ echo
 #deploy prometheus
 echo -e "${ORANGE}Deploying Prometheus"
 tput sgr0
-kubectl create -f ./k8s/prometheus
+kubectl create -R -f ./k8s/prometheus
 
 echo
 

definitions/k8s/grafana/grafana.svc.deployment.yaml → k8s/grafana/grafana.svc.deployment.yaml

@@ -26,7 +26,7 @@ spec:
         app: grafana
     spec:
       containers:
-      - image: DOCKER_USER/grafana:4.1.1
+      - image: DOCKER_USER/grafana:GRAFANA_VERSION
         name: grafana
         imagePullPolicy: Always
         ports:

definitions/k8s/ingress/02-nginx-lb.svc.rc.yaml → k8s/ingress/02-nginx-lb.svc.deployment.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: default-http-backend
-  namespace: monitoring
+  namespace: nginx-ingress
   labels:
     k8s-app: default-http-backend
 spec:
@@ -18,9 +18,13 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: default-http-backend
-  namespace: monitoring
+  namespace: nginx-ingress
 spec:
   replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
   selector:
     matchLabels:
       k8s-app: default-http-backend
@@ -29,10 +33,11 @@ spec:
       labels:
         k8s-app: default-http-backend
     spec:
+      serviceAccountName: default
       terminationGracePeriodSeconds: 60
       containers:
       - name: default-http-backend
-        image: gcr.io/google_containers/defaultbackend:1.0
+        image: gcr.io/google_containers/defaultbackend:1.3
         livenessProbe:
           httpGet:
             path: /healthz
@@ -42,6 +47,7 @@ spec:
           timeoutSeconds: 5
         ports:
         - containerPort: 8080
+          name: http
         resources:
           limits:
             cpu: 10m
@@ -54,7 +60,7 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: nginx-ingress-controller
-  namespace: monitoring
+  namespace: nginx-ingress
   labels:
     k8s-app: nginx-ingress-lb
 spec:
@@ -70,12 +76,8 @@ spec:
     spec:
       terminationGracePeriodSeconds: 60
       hostNetwork: true
-      volumes:
-      - name: dhparam
-        secret:
-          secretName: dhparam
       containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.5
         name: nginx-ingress-lb
         imagePullPolicy: Always
         livenessProbe:
@@ -101,10 +103,6 @@ spec:
         - containerPort: 443
           hostPort: 443
         - containerPort: 8080
-          hostPort: 8080
-        volumeMounts:
-        - mountPath: /etc/nginx-ssl/dhparam
-          name: dhparam
         args:
         - /nginx-ingress-controller
-        - --default-backend-service=monitoring/default-http-backend
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend

definitions/k8s/kube-state-metrics/deployment.yml → k8s/kube-state-metrics/deployment.yml

@@ -9,11 +9,11 @@ spec:
     metadata:
       labels:
         app: kube-state-metrics
-        version: "v0.3.0"
+        version: "v0.5.0"
     spec:
       containers:
       - name: kube-state-metrics
-        image: gcr.io/google_containers/kube-state-metrics:v0.3.0
+        image: gcr.io/google_containers/kube-state-metrics:v0.5.0
         ports:
         - containerPort: 8080
         imagePullPolicy: Always

definitions/k8s/prometheus/01-prometheus.configmap.yaml → k8s/prometheus/00-alerts.cm.yaml

@@ -1,108 +1,10 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: prometheus
+  name: alerts
   namespace: monitoring
 data:
-  prometheus.yml: |-
-    global:
-      evaluation_interval: 30s
-    scrape_configs:
-
-    - job_name: kubelets
-
-      scrape_interval: 20s
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
-      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      kubernetes_sd_configs:
-      - role: node
-
-    - job_name: standard-endpoints
-
-      scrape_interval: 20s
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
-      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        source_labels: [__meta_kubernetes_service_name]
-        regex: kubernetes|node-exporter|kube-state-metrics|etcd-k8s|prometheus
-      - action: replace
-        source_labels: [__meta_kubernetes_service_name]
-        target_label: job
-      - action: replace
-        source_labels: [__meta_kubernetes_service_name]
-        regex: kubernetes
-        target_label: __scheme__
-        replacement: https
-
-    - job_name: kube-components
-
-      scrape_interval: 20s
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: replace
-        source_labels: [__meta_kubernetes_service_name]
-        target_label: job
-        regex: "kube-(.*)-prometheus-discovery"
-        replacement: "kube-${1}"
-      - action: keep
-        source_labels: [__meta_kubernetes_service_name]
-        regex: "kube-(.*)-prometheus-discovery"
-      - action: keep
-        source_labels: [__meta_kubernetes_endpoint_port_name]
-        regex: "prometheus"
-
-    - job_name: 'kubernetes-pods'
-
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-        action: keep
-        regex: true
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        action: replace
-        target_label: __metrics_path__
-        regex: (.+)
-      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-        action: replace
-        regex: (.+):(?:\d+);(\d+)
-        replacement: ${1}:${2}
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - source_labels: [__meta_kubernetes_pod_namespace]
-        action: replace
-        target_label: kubernetes_namespace
-      - source_labels: [__meta_kubernetes_pod_name]
-        action: replace
-        target_label: kubernetes_pod_name
-
-    - job_name: ec2
-
-      ec2_sd_configs:
-      - region: us-east-1
-        access_key: aws_access_key
-        secret_key: aws_secret_key
-        refresh_interval: 60s
-        port: 9100
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_ec2_tag_(.+)
-    rule_files:
-      - '/etc/prometheus/alert.rules'
-  alert.rules: |-
+  kubernetes.rules: |-
        ### Container resources ###
        cluster_namespace_controller_pod_container:spec_memory_limit_bytes =
          sum by (cluster,namespace,controller,pod_name,container_name) (
@@ -303,7 +205,7 @@ data:
          }
        ALERT K8SApiserverDown
          IF up{job="kubernetes"} == 0
-         FOR 5m
+         FOR 10m
          LABELS {
            service = "k8s",
            severity = "warning"
@@ -417,3 +319,11 @@ data:
            summary = "Kubelet is close to pod limit",
            description = "Kubelet {{$labels.instance}} is running {{$value}} pods, close to the limit of 110",
          }
+       ALERT PodRestartingTooMuch
+         IF          rate(kube_pod_container_status_restarts[10m])*600 > 2
+         FOR         5m
+         LABELS      { severity="warning" }
+         ANNOTATIONS {
+           summary = "Pod {{$labels.namespace}}/{{$label.name}} restarting too much.",
+           description = "Pod {{$labels.namespace}}/{{$label.name}} restarting too much.",
+         }