init.sh improvements (#8)

* updating init.sh to improve messaging and ease configuration workflow * moved namespace creation lower to ease iterative testing * improved question flow for basic-auth with a default user * improved SMTP config question flow as well as making it optional * same for slack config * updated read statements in AWS key config to ensure backspace works * removed superfluous messaging * add liveness probe to all containers
camilb · May 25, 2017 · 9c341ad · 9c341ad
1 parent 9e3115e
commit 9c341ad
Show file tree

Hide file tree

Showing 10 changed files with 118 additions and 155 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1 @@
-.swp
+*.swp
diff --git a/init.sh b/init.sh
@@ -11,175 +11,140 @@ ORANGE='\033[0;33m'
 BLUE='\033[0;34m'
 PURPLE='\033[0;35m'
 
-echo -e "${BLUE}Creating 'monitoring' namespace."
-tput sgr0
-
-#create a separate namespace for monitoring
-kubectl create namespace monitoring
-
 #Ask for grafana version or apply default
 echo
 read -p "Enter Grafana version [$GRAFANA_DEFAULT_VERSION]: " GRAFANA_VERSION
 GRAFANA_VERSION=${GRAFANA_VERSION:-$GRAFANA_DEFAULT_VERSION}
 
 #Ask for prometheus version or apply default
-echo
 read -p "Enter Prometheus version [$PROMETHEUS_DEFAULT_VERSION]: " PROMETHEUS_VERSION
 PROMETHEUS_VERSION=${PROMETHEUS_VERSION:-$PROMETHEUS_DEFAULT_VERSION}
 
 #Ask for alertmanager version or apply default
-echo
 read -p "Enter Alert Manager version [$ALERT_MANAGER_DEFAULT_VERSION]: " ALERT_MANAGER_VERSION
 ALERT_MANAGER_VERSION=${ALERT_MANAGER_VERSION:-$ALERT_MANAGER_DEFAULT_VERSION}
 
 
 #Ask for node exporter version or apply default
-echo
 read -p "Enter Node Exporter version [$NODE_EXPORTER_DEFAULT_VERSION]: " NODE_EXPORTER_VERSION
 NODE_EXPORTER_VERSION=${NODE_EXPORTER_VERSION:-$NODE_EXPORTER_DEFAULT_VERSION}
 
 #Ask for dockerhub user or apply default of the current logged-in username
-echo
 read -p "Enter Dockerhub username [$DOCKER_USER_DEFAULT]: " DOCKER_USER
 DOCKER_USER=${DOCKER_USER:-$DOCKER_USER_DEFAULT}
 
 #Set username and password for basic-auth
 echo
-echo -e "${BLUE}Please set the username and password for basic-auth and [ENTER]:"
+echo -e "${BLUE}Please set the username and password for basic-auth to prometheus and alertmanager:"
 tput sgr0
-
-prompt="Set username:"
-while IFS= read -p "$prompt" -r -s -n 1 char
-do
-    if [[ $char == $'\0' ]]
-    then
-        break
-    fi
-    prompt=$char
-    username+="$char"
-done
-echo
-
-htpasswd -c auth $username
+read -p "Set username [monitor]: " username
+htpasswd -c auth ${username:-'monitor'}
 
 #base64 encode the basic-auth and set the secret
 BASIC_AUTH=$(cat ./auth | base64)
-
 sed -i -e 's/htpasswd/'"$BASIC_AUTH"'/g' k8s/ingress/01-basic-auth.secret.yaml
 
-echo
-
 #Replace Dockerhub username in grafana deployment.
-
 sed -i -e 's/DOCKER_USER/'"$DOCKER_USER"'/g' k8s/grafana/grafana.svc.deployment.yaml
 
-#password for SMTP account in alertmanager ConfigMap.
-echo -e "${BLUE}Insert the password for SMTP account and press [ENTER]:"
+#Do you want to set up an SMTP relay?
+echo
+echo -e "${BLUE}Do you want to set up an SMTP relay?"
 tput sgr0
+read -p "Y/N [N]: " use_smtp
+
+#if so, fill out this form...
+if [[ $use_smtp =~ ^([yY][eE][sS]|[yY])$ ]]; then
+  #smtp smarthost
+  read -p "SMTP smarthost: " smtp_smarthost
+  #smtp from address
+  read -p "SMTP from ([email protected]): " smtp_from
+  #smtp to address
+  read -p "Email address to send alerts to ([email protected]): " alert_email_address
+  #smtp username
+  read -p "SMTP auth username: " smtp_user
+  #smtp password
+  prompt="SMTP auth password: "
+  while IFS= read -p "$prompt" -r -s -n 1 char
+  do
+      if [[ $char == $'\0' ]]
+      then
+          break
+      fi
+      prompt='*'
+      smtp_password+="$char"
+  done
 
-prompt="SMTP Password:"
-while IFS= read -p "$prompt" -r -s -n 1 char
-do
-    if [[ $char == $'\0' ]]
-    then
-        break
-    fi
-    prompt='*'
-    smtp_password+="$char"
-done
-echo
+  #update configmap with SMTP relay info
+  sed -i -e 's/your_smtp_smarthost/'"$smtp_smarthost"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+  sed -i -e 's/your_smtp_from/'"$smtp_from"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+  sed -i -e 's/your_smtp_user/'"$smtp_user"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+  sed -i -e 's/your_smtp_pass/'"$smtp_password"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+  sed -i -e 's/your_alert_email_address/'"$alert_email_address"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+fi
 
-sed -i -e 's/smtp_pass/'"$smtp_password"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
-echo -e "${BLUE}SMTP password set."
-tput sgr0
+#Do you want to set up slack?
 echo
+echo -e "${BLUE}Do you want to set up slack alerts?"
+tput sgr0
+read -p "Y/N [N]: " use_slack
+
+#if so, fill out this form...
+if [[ $use_slack =~ ^([yY][eE][sS]|[yY])$ ]]; then
+
+  read -p "Slack api token: " slack_api_token
+  read -p "Slack channel: " slack_channel
+
+  #again, our sed is funky due to slashes appearing in slack api tokens
+  sed -i -e 's,your_slack_api_token,'"$slack_api_token"',g' k8s/prometheus/03-alertmanager.configmap.yaml
+  sed -i -e 's/your_slack_channel/'"$slack_channel"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
+fi
 
 #try to figure out AWS credentials for EC2 monitoring, if not...ask.
+echo
 echo -e "${BLUE}Detecting AWS access keys."
 tput sgr0
-echo
 if [ ! -z $AWS_ACCESS_KEY_ID ] && [ ! -z $AWS_SECRET_ACCESS_KEY ]; then
   aws_access_key=$AWS_ACCESS_KEY_ID
   aws_secret_key=$AWS_SECRET_ACCESS_KEY
   echo -e "${ORANGE}AWS_ACCESS_KEY_ID found, using $aws_access_key."
   tput sgr0
-  echo
 elif [ ! -z $AWS_ACCESS_KEY ] && [ ! -z $AWS_SECRET_KEY ]; then
   aws_access_key=$AWS_ACCESS_KEY
   aws_secret_key=$AWS_SECRET_KEY
   echo -e "${ORANGE}AWS_ACCESS_KEY found, using $aws_access_key."
   tput sgr0
-  echo
 else
   echo -e "${RED}Unable to determine AWS credetials from environment variables."
-  echo -e "${ORANGE}Insert your AWS Access Key ID and press [ENTER]:"
   tput sgr0
-
   #aws access key
-  prompt="AWS Access Key ID:"
-  tput sgr0
-  while IFS= read -p "$prompt" -r -s -n 1 char
-  do
-      if [[ $char == $'\0' ]]
-      then
-          break
-      fi
-      prompt='*'
-      aws_access_key+="$char"
-  done
-  echo
-
+  read -p "AWS Access Key ID: " aws_access_key
   #aws secret access key
-  echo -e "${ORANGE}Insert your AWS Secret Access Key and press [ENTER]:"
-  tput sgr0
-
-  prompt="AWS Secret Access Key:"
-  tput sgr0
-  while IFS= read -p "$prompt" -r -s -n 1 char
-  do
-      if [[ $char == $'\0' ]]
-      then
-          break
-      fi
-      prompt='*'
-      aws_secret_key+="$char"
-  done
-  echo
+  read -p "AWS Secret Access Key: " aws_secret_key
 fi
 
 #sed in the AWS credentials. this looks odd because aws secret access keys can have '/' as a valid character
 #so we use ',' as a delimiter for sed, since that won't appear in the secret key
 sed -i -e 's/aws_access_key/'"$aws_access_key"'/g' k8s/prometheus/01-prometheus.configmap.yaml
 sed -i -e 's,aws_secret_key,'"$aws_secret_key"',g' k8s/prometheus/01-prometheus.configmap.yaml
 
-#slack channel
-echo -e "${PURPLE}Insert your slack channel name where you wish to receive alerts and press [ENTER]:"
+echo
+echo -e "${BLUE}Creating ${ORANGE}'monitoring' ${BLUE}and ${ORANGE}'nginx-ingress' ${BLUE}namespaces."
 tput sgr0
+#create a separate namespace for monitoring
+kubectl create namespace monitoring
+kubectl create namespace nginx-ingress
 
-prompt="Slack channel:"
-tput sgr0
-while IFS= read -p "$prompt" -r -s -n 1 char
-do
-    if [[ $char == $'\0' ]]
-    then
-        break
-    fi
-    prompt=$char
-    slack_channel+="$char"
-done
-echo
-sed -i -e 's/slack_channel/'"$slack_channel"'/g' k8s/prometheus/03-alertmanager.configmap.yaml
 echo
-
-read -r -p "Is the RBAC plugin enabled? [y/N] " response
+read -r -p "Is the RBAC plugin enabled? [y/N]: " response
 if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
 then
     kubectl create -f ./k8s/rbac
     sed -i -e 's/default/'prometheus'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml
 else
-    echo -e "${GREEN}Skipping RBAC configuration."
-    tput sgr0
+    echo -e "${GREEN}Skipping RBAC configuration"
 fi
+tput sgr0
 
 #set prometheus version
 sed -i -e 's/PROMETHEUS_VERSION/'"$PROMETHEUS_VERSION"'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml
@@ -194,21 +159,14 @@ sed -i -e 's/ALERT_MANAGER_VERSION/'"$ALERT_MANAGER_VERSION"'/g' k8s/prometheus/
 #set node-exporter version
 sed -i -e 's/NODE_EXPORTER_VERSION/'"$NODE_EXPORTER_VERSION"'/g' k8s/prometheus/05-node-exporter.svc.daemonset.yaml
 
-
 #remove  "sed" generated files
-rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e && rm grafana/*-e
+rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e && rm grafana/*-e 2> /dev/null
 
+#build grafana image, push to dockerhub
 echo
-
-#build grafana image
-echo -e "${BLUE}Building Grafana Docker image"
+echo -e "${BLUE}Building Grafana Docker image and pushing to dockerhub"
 tput sgr0
 docker build -t $DOCKER_USER/grafana:$GRAFANA_VERSION ./grafana --no-cache
-
-echo
-
-echo -e "${BLUE}Pushing grafana docker image to DockerHub"
-tput sgr0
 docker push $DOCKER_USER/grafana:$GRAFANA_VERSION
 #upon failure, run docker login
 if [ $? -eq 1 ];then
@@ -224,47 +182,41 @@ if [ $? -eq 1 ];then
   fi
 fi
 
-echo
 
 #deploy grafana
-echo -e "${RED}Deploying Grafana"
+echo
+echo -e "${ORANGE}Deploying Grafana"
 tput sgr0
 kubectl create -f k8s/grafana
 
-echo
-
 #deploy prometheus
+echo
 echo -e "${ORANGE}Deploying Prometheus"
 tput sgr0
 kubectl create -R -f ./k8s/prometheus
 
-echo
-
 #deploy kube-state-metrics
+echo
 echo -e "${ORANGE}Deploying Kube State Metrics exporter"
 tput sgr0
 kubectl create -f ./k8s/kube-state-metrics
 
-echo
-
 #deploy ingress controller
+echo
 echo -e "${BLUE}Deploying  K8S Ingress Controller"
 tput sgr0
 kubectl create -f ./k8s/ingress
 
-echo
-
 #wait for the ingress to become available.
+echo
 echo -e "${BLUE}Waiting 10 seconds for the Ingress Controller to become available."
 tput sgr0
 sleep 10
 
-#get ingress IP and hosts
+#get ingress IP and hosts, display for user
 PROM_INGRESS=$(kubectl get ing --namespace=monitoring)
-
-echo 'Configure "/etc/hosts" or create DNS records for these hosts:' && printf "${RED}$PROM_INGRESS"
-
 echo
+echo 'Configure "/etc/hosts" or create DNS records for these hosts:' && printf "${RED}$PROM_INGRESS"
 echo
 
 #cleanup
@@ -289,33 +241,20 @@ kubectl port-forward $GRAFANA_POD --namespace=monitoring 3000:3000 > /dev/null 2
 
 echo -e "${GREEN}Importing Prometheus datasource."
 tput sgr0
-
-echo -e "${GREEN}Waiting 5 seconds to establish the proxy connection"
-tput sgr0
 sleep 5
-echo
-
-curl 'http://admin:[email protected]:3000/api/datasources' -X POST -H 'Content-Type: application/json;charset=UTF-8' --data-binary '{"name":"prometheus.monitoring.svc.cluster.local","type":"prometheus","url":"http://prometheus.monitoring.svc.cluster.local:9090","access":"proxy","isDefault":true}'
-echo
+curl 'http://admin:[email protected]:3000/api/datasources' -X POST -H 'Content-Type: application/json;charset=UTF-8' --data-binary '{"name":"prometheus.monitoring.svc.cluster.local","type":"prometheus","url":"http://prometheus.monitoring.svc.cluster.local:9090","access":"proxy","isDefault":true}' 2> /dev/null 2>&1
 
 #check datasources
-echo -e "${GREEN}Checking datasource"
-tput sgr0
-curl 'http://admin:[email protected]:3000/api/datasources'
-
 echo
-
-echo -e "${GREEN}Datasource imported"
-tput sgr0
-
-echo
-
-echo -e "${RED}Killing background process."
+echo -e "${GREEN}Checking datasource"
 tput sgr0
+curl 'http://admin:[email protected]:3000/api/datasources' 2> /dev/null 2>&1
 
+# kill the backgrounded proxy process
 kill $!
-echo
 
+# set up proxy for the user
+echo
 read -r -p "Do you want to proxy Grafana to localhost now? [y/N] " response
 if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
 then

diff --git a/k8s/grafana/grafana.svc.deployment.yaml b/k8s/grafana/grafana.svc.deployment.yaml
@@ -36,3 +36,9 @@ spec:
             value: "true"
           - name: GF_AUTH_ANONYMOUS_ENABLED
             value: "false"
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 3000
+          initialDelaySeconds: 30
+          timeoutSeconds: 1
diff --git a/k8s/kube-state-metrics/deployment.yml b/k8s/kube-state-metrics/deployment.yml
@@ -17,3 +17,9 @@ spec:
         ports:
         - containerPort: 8080
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          initialDelaySeconds: 30
+          timeoutSeconds: 1
diff --git a/k8s/prometheus/00-alerts.cm.yaml b/k8s/prometheus/00-alerts.cm.yaml
@@ -175,7 +175,7 @@ data:
            description = "{{ $value }} K8s nodes (more than 10% of cluster {{ $labels.cluster }}) are in the NotReady state.",
          }
        ALERT HighDiskUsage
-         IF node_filesystem_free{device="/dev/xvda9"}<node_filesystem_size/10
+         IF node_filesystem_free{device="/dev/xvda1"}<node_filesystem_size/10
          FOR 1m
          ANNOTATIONS {
            summary = "Low disk space on {{ $labels.instance }}",