From 9c341ad4e7e7246024a2e7deec5d84ff5dacbef9 Mon Sep 17 00:00:00 2001 From: Dave K Date: Wed, 24 May 2017 20:28:55 -0400 Subject: [PATCH] init.sh improvements (#8) * updating init.sh to improve messaging and ease configuration workflow * moved namespace creation lower to ease iterative testing * improved question flow for basic-auth with a default user * improved SMTP config question flow as well as making it optional * same for slack config * updated read statements in AWS key config to ensure backspace works * removed superfluous messaging * add liveness probe to all containers --- .gitignore | 2 +- init.sh | 219 +++++++----------- k8s/grafana/grafana.svc.deployment.yaml | 6 + k8s/kube-state-metrics/deployment.yml | 6 + k8s/prometheus/00-alerts.cm.yaml | 2 +- .../02-prometheus.svc.statefulset.yaml | 6 + k8s/prometheus/03-alertmanager.configmap.yaml | 14 +- .../04-alertmanager.svc.deployment.yaml | 6 + .../05-node-exporter.svc.daemonset.yaml | 6 + k8s/rbac/02-nginx-ingress-rbac-config.yaml | 6 - 10 files changed, 118 insertions(+), 155 deletions(-) diff --git a/.gitignore b/.gitignore index 6ff6e1d..1377554 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.swp +*.swp diff --git a/init.sh b/init.sh index 1e3680f..50ef249 100755 --- a/init.sh +++ b/init.sh @@ -11,140 +11,116 @@ ORANGE='\033[0;33m' BLUE='\033[0;34m' PURPLE='\033[0;35m' -echo -e "${BLUE}Creating 'monitoring' namespace." -tput sgr0 - -#create a separate namespace for monitoring -kubectl create namespace monitoring - #Ask for grafana version or apply default echo read -p "Enter Grafana version [$GRAFANA_DEFAULT_VERSION]: " GRAFANA_VERSION GRAFANA_VERSION=${GRAFANA_VERSION:-$GRAFANA_DEFAULT_VERSION} #Ask for prometheus version or apply default -echo read -p "Enter Prometheus version [$PROMETHEUS_DEFAULT_VERSION]: " PROMETHEUS_VERSION PROMETHEUS_VERSION=${PROMETHEUS_VERSION:-$PROMETHEUS_DEFAULT_VERSION} #Ask for alertmanager version or apply default -echo read -p "Enter Alert Manager version [$ALERT_MANAGER_DEFAULT_VERSION]: " ALERT_MANAGER_VERSION ALERT_MANAGER_VERSION=${ALERT_MANAGER_VERSION:-$ALERT_MANAGER_DEFAULT_VERSION} #Ask for node exporter version or apply default -echo read -p "Enter Node Exporter version [$NODE_EXPORTER_DEFAULT_VERSION]: " NODE_EXPORTER_VERSION NODE_EXPORTER_VERSION=${NODE_EXPORTER_VERSION:-$NODE_EXPORTER_DEFAULT_VERSION} #Ask for dockerhub user or apply default of the current logged-in username -echo read -p "Enter Dockerhub username [$DOCKER_USER_DEFAULT]: " DOCKER_USER DOCKER_USER=${DOCKER_USER:-$DOCKER_USER_DEFAULT} #Set username and password for basic-auth echo -echo -e "${BLUE}Please set the username and password for basic-auth and [ENTER]:" +echo -e "${BLUE}Please set the username and password for basic-auth to prometheus and alertmanager:" tput sgr0 - -prompt="Set username:" -while IFS= read -p "$prompt" -r -s -n 1 char -do - if [[ $char == $'\0' ]] - then - break - fi - prompt=$char - username+="$char" -done -echo - -htpasswd -c auth $username +read -p "Set username [monitor]: " username +htpasswd -c auth ${username:-'monitor'} #base64 encode the basic-auth and set the secret BASIC_AUTH=$(cat ./auth | base64) - sed -i -e 's/htpasswd/'"$BASIC_AUTH"'/g' k8s/ingress/01-basic-auth.secret.yaml -echo - #Replace Dockerhub username in grafana deployment. - sed -i -e 's/DOCKER_USER/'"$DOCKER_USER"'/g' k8s/grafana/grafana.svc.deployment.yaml -#password for SMTP account in alertmanager ConfigMap. -echo -e "${BLUE}Insert the password for SMTP account and press [ENTER]:" +#Do you want to set up an SMTP relay? +echo +echo -e "${BLUE}Do you want to set up an SMTP relay?" tput sgr0 +read -p "Y/N [N]: " use_smtp + +#if so, fill out this form... +if [[ $use_smtp =~ ^([yY][eE][sS]|[yY])$ ]]; then + #smtp smarthost + read -p "SMTP smarthost: " smtp_smarthost + #smtp from address + read -p "SMTP from (user@domain.com): " smtp_from + #smtp to address + read -p "Email address to send alerts to (user@domain.com): " alert_email_address + #smtp username + read -p "SMTP auth username: " smtp_user + #smtp password + prompt="SMTP auth password: " + while IFS= read -p "$prompt" -r -s -n 1 char + do + if [[ $char == $'\0' ]] + then + break + fi + prompt='*' + smtp_password+="$char" + done -prompt="SMTP Password:" -while IFS= read -p "$prompt" -r -s -n 1 char -do - if [[ $char == $'\0' ]] - then - break - fi - prompt='*' - smtp_password+="$char" -done -echo + #update configmap with SMTP relay info + sed -i -e 's/your_smtp_smarthost/'"$smtp_smarthost"'/g' k8s/prometheus/03-alertmanager.configmap.yaml + sed -i -e 's/your_smtp_from/'"$smtp_from"'/g' k8s/prometheus/03-alertmanager.configmap.yaml + sed -i -e 's/your_smtp_user/'"$smtp_user"'/g' k8s/prometheus/03-alertmanager.configmap.yaml + sed -i -e 's/your_smtp_pass/'"$smtp_password"'/g' k8s/prometheus/03-alertmanager.configmap.yaml + sed -i -e 's/your_alert_email_address/'"$alert_email_address"'/g' k8s/prometheus/03-alertmanager.configmap.yaml +fi -sed -i -e 's/smtp_pass/'"$smtp_password"'/g' k8s/prometheus/03-alertmanager.configmap.yaml -echo -e "${BLUE}SMTP password set." -tput sgr0 +#Do you want to set up slack? echo +echo -e "${BLUE}Do you want to set up slack alerts?" +tput sgr0 +read -p "Y/N [N]: " use_slack + +#if so, fill out this form... +if [[ $use_slack =~ ^([yY][eE][sS]|[yY])$ ]]; then + + read -p "Slack api token: " slack_api_token + read -p "Slack channel: " slack_channel + + #again, our sed is funky due to slashes appearing in slack api tokens + sed -i -e 's,your_slack_api_token,'"$slack_api_token"',g' k8s/prometheus/03-alertmanager.configmap.yaml + sed -i -e 's/your_slack_channel/'"$slack_channel"'/g' k8s/prometheus/03-alertmanager.configmap.yaml +fi #try to figure out AWS credentials for EC2 monitoring, if not...ask. +echo echo -e "${BLUE}Detecting AWS access keys." tput sgr0 -echo if [ ! -z $AWS_ACCESS_KEY_ID ] && [ ! -z $AWS_SECRET_ACCESS_KEY ]; then aws_access_key=$AWS_ACCESS_KEY_ID aws_secret_key=$AWS_SECRET_ACCESS_KEY echo -e "${ORANGE}AWS_ACCESS_KEY_ID found, using $aws_access_key." tput sgr0 - echo elif [ ! -z $AWS_ACCESS_KEY ] && [ ! -z $AWS_SECRET_KEY ]; then aws_access_key=$AWS_ACCESS_KEY aws_secret_key=$AWS_SECRET_KEY echo -e "${ORANGE}AWS_ACCESS_KEY found, using $aws_access_key." tput sgr0 - echo else echo -e "${RED}Unable to determine AWS credetials from environment variables." - echo -e "${ORANGE}Insert your AWS Access Key ID and press [ENTER]:" tput sgr0 - #aws access key - prompt="AWS Access Key ID:" - tput sgr0 - while IFS= read -p "$prompt" -r -s -n 1 char - do - if [[ $char == $'\0' ]] - then - break - fi - prompt='*' - aws_access_key+="$char" - done - echo - + read -p "AWS Access Key ID: " aws_access_key #aws secret access key - echo -e "${ORANGE}Insert your AWS Secret Access Key and press [ENTER]:" - tput sgr0 - - prompt="AWS Secret Access Key:" - tput sgr0 - while IFS= read -p "$prompt" -r -s -n 1 char - do - if [[ $char == $'\0' ]] - then - break - fi - prompt='*' - aws_secret_key+="$char" - done - echo + read -p "AWS Secret Access Key: " aws_secret_key fi #sed in the AWS credentials. this looks odd because aws secret access keys can have '/' as a valid character @@ -152,34 +128,23 @@ fi sed -i -e 's/aws_access_key/'"$aws_access_key"'/g' k8s/prometheus/01-prometheus.configmap.yaml sed -i -e 's,aws_secret_key,'"$aws_secret_key"',g' k8s/prometheus/01-prometheus.configmap.yaml -#slack channel -echo -e "${PURPLE}Insert your slack channel name where you wish to receive alerts and press [ENTER]:" +echo +echo -e "${BLUE}Creating ${ORANGE}'monitoring' ${BLUE}and ${ORANGE}'nginx-ingress' ${BLUE}namespaces." tput sgr0 +#create a separate namespace for monitoring +kubectl create namespace monitoring +kubectl create namespace nginx-ingress -prompt="Slack channel:" -tput sgr0 -while IFS= read -p "$prompt" -r -s -n 1 char -do - if [[ $char == $'\0' ]] - then - break - fi - prompt=$char - slack_channel+="$char" -done -echo -sed -i -e 's/slack_channel/'"$slack_channel"'/g' k8s/prometheus/03-alertmanager.configmap.yaml echo - -read -r -p "Is the RBAC plugin enabled? [y/N] " response +read -r -p "Is the RBAC plugin enabled? [y/N]: " response if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]] then kubectl create -f ./k8s/rbac sed -i -e 's/default/'prometheus'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml else - echo -e "${GREEN}Skipping RBAC configuration." - tput sgr0 + echo -e "${GREEN}Skipping RBAC configuration" fi +tput sgr0 #set prometheus version sed -i -e 's/PROMETHEUS_VERSION/'"$PROMETHEUS_VERSION"'/g' k8s/prometheus/02-prometheus.svc.statefulset.yaml @@ -194,21 +159,14 @@ sed -i -e 's/ALERT_MANAGER_VERSION/'"$ALERT_MANAGER_VERSION"'/g' k8s/prometheus/ #set node-exporter version sed -i -e 's/NODE_EXPORTER_VERSION/'"$NODE_EXPORTER_VERSION"'/g' k8s/prometheus/05-node-exporter.svc.daemonset.yaml - #remove "sed" generated files -rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e && rm grafana/*-e +rm k8s/prometheus/*.yaml-e && rm k8s/ingress/*.yaml-e && rm k8s/grafana/*.yaml-e && rm grafana/*-e 2> /dev/null +#build grafana image, push to dockerhub echo - -#build grafana image -echo -e "${BLUE}Building Grafana Docker image" +echo -e "${BLUE}Building Grafana Docker image and pushing to dockerhub" tput sgr0 docker build -t $DOCKER_USER/grafana:$GRAFANA_VERSION ./grafana --no-cache - -echo - -echo -e "${BLUE}Pushing grafana docker image to DockerHub" -tput sgr0 docker push $DOCKER_USER/grafana:$GRAFANA_VERSION #upon failure, run docker login if [ $? -eq 1 ];then @@ -224,47 +182,41 @@ if [ $? -eq 1 ];then fi fi -echo #deploy grafana -echo -e "${RED}Deploying Grafana" +echo +echo -e "${ORANGE}Deploying Grafana" tput sgr0 kubectl create -f k8s/grafana -echo - #deploy prometheus +echo echo -e "${ORANGE}Deploying Prometheus" tput sgr0 kubectl create -R -f ./k8s/prometheus -echo - #deploy kube-state-metrics +echo echo -e "${ORANGE}Deploying Kube State Metrics exporter" tput sgr0 kubectl create -f ./k8s/kube-state-metrics -echo - #deploy ingress controller +echo echo -e "${BLUE}Deploying K8S Ingress Controller" tput sgr0 kubectl create -f ./k8s/ingress -echo - #wait for the ingress to become available. +echo echo -e "${BLUE}Waiting 10 seconds for the Ingress Controller to become available." tput sgr0 sleep 10 -#get ingress IP and hosts +#get ingress IP and hosts, display for user PROM_INGRESS=$(kubectl get ing --namespace=monitoring) - -echo 'Configure "/etc/hosts" or create DNS records for these hosts:' && printf "${RED}$PROM_INGRESS" - echo +echo 'Configure "/etc/hosts" or create DNS records for these hosts:' && printf "${RED}$PROM_INGRESS" echo #cleanup @@ -289,33 +241,20 @@ kubectl port-forward $GRAFANA_POD --namespace=monitoring 3000:3000 > /dev/null 2 echo -e "${GREEN}Importing Prometheus datasource." tput sgr0 - -echo -e "${GREEN}Waiting 5 seconds to establish the proxy connection" -tput sgr0 sleep 5 -echo - -curl 'http://admin:admin@127.0.0.1:3000/api/datasources' -X POST -H 'Content-Type: application/json;charset=UTF-8' --data-binary '{"name":"prometheus.monitoring.svc.cluster.local","type":"prometheus","url":"http://prometheus.monitoring.svc.cluster.local:9090","access":"proxy","isDefault":true}' -echo +curl 'http://admin:admin@127.0.0.1:3000/api/datasources' -X POST -H 'Content-Type: application/json;charset=UTF-8' --data-binary '{"name":"prometheus.monitoring.svc.cluster.local","type":"prometheus","url":"http://prometheus.monitoring.svc.cluster.local:9090","access":"proxy","isDefault":true}' 2> /dev/null 2>&1 #check datasources -echo -e "${GREEN}Checking datasource" -tput sgr0 -curl 'http://admin:admin@127.0.0.1:3000/api/datasources' - echo - -echo -e "${GREEN}Datasource imported" -tput sgr0 - -echo - -echo -e "${RED}Killing background process." +echo -e "${GREEN}Checking datasource" tput sgr0 +curl 'http://admin:admin@127.0.0.1:3000/api/datasources' 2> /dev/null 2>&1 +# kill the backgrounded proxy process kill $! -echo +# set up proxy for the user +echo read -r -p "Do you want to proxy Grafana to localhost now? [y/N] " response if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]] then diff --git a/k8s/grafana/grafana.svc.deployment.yaml b/k8s/grafana/grafana.svc.deployment.yaml index e4aec1e..69e6979 100644 --- a/k8s/grafana/grafana.svc.deployment.yaml +++ b/k8s/grafana/grafana.svc.deployment.yaml @@ -36,3 +36,9 @@ spec: value: "true" - name: GF_AUTH_ANONYMOUS_ENABLED value: "false" + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + timeoutSeconds: 1 diff --git a/k8s/kube-state-metrics/deployment.yml b/k8s/kube-state-metrics/deployment.yml index bd72593..0d4b591 100644 --- a/k8s/kube-state-metrics/deployment.yml +++ b/k8s/kube-state-metrics/deployment.yml @@ -17,3 +17,9 @@ spec: ports: - containerPort: 8080 imagePullPolicy: Always + livenessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 30 + timeoutSeconds: 1 diff --git a/k8s/prometheus/00-alerts.cm.yaml b/k8s/prometheus/00-alerts.cm.yaml index 9923edf..9d621a7 100644 --- a/k8s/prometheus/00-alerts.cm.yaml +++ b/k8s/prometheus/00-alerts.cm.yaml @@ -175,7 +175,7 @@ data: description = "{{ $value }} K8s nodes (more than 10% of cluster {{ $labels.cluster }}) are in the NotReady state.", } ALERT HighDiskUsage - IF node_filesystem_free{device="/dev/xvda9"}