diff --git a/README.md b/README.md index 90bad06..e9357ad 100644 --- a/README.md +++ b/README.md @@ -70,3 +70,8 @@ To remove everything, just execute the `./teardown` script. * **update grafana dashboards:** add or change the existing dashboards in `assets/grafana/` and execute `scripts/generate-dashboards-configmap.sh`. Then apply the changes using `kubectl apply -f manifests/grafana/grafana-dashboards.cm.yaml`. **Note:** all the Grafana dashboards should have names ending in `-dashboard.json`. + +## Custom Prometheus configuration + + The official documentation for Prometheus Operator custom configuration can be found here: [custom-configuration.md](https://github.com/coreos/prometheus-operator/blob/master/Documentation/custom-configuration.md) + If you wish, you can update the Prometheus configuration using the `./tools/custom-configuration/update_config` script. diff --git a/deploy b/deploy index 33cd5b1..218ebd9 100755 --- a/deploy +++ b/deploy @@ -3,8 +3,8 @@ ######################################################################################### #components default version ######################################################################################### -GRAFANA_DEFAULT_VERSION=4.6.0-beta2 -PROMETHEUS_DEFAULT_VERSION=v2.0.0-rc.1 +GRAFANA_DEFAULT_VERSION=4.6.0-beta3 +PROMETHEUS_DEFAULT_VERSION=v2.0.0-rc.2 PROMETHEUS_OPERATOR_DEFAULT_VERSION=v0.14.0 ALERTMANAGER_DEFAULT_VERSION=v0.9.1 NODE_EXPORTER_DEFAULT_VERSION=v0.15.0 diff --git a/tools/custom-configuration/prometheus-k8s-secret/configmaps.json b/tools/custom-configuration/prometheus-k8s-secret/configmaps.json new file mode 100644 index 0000000..58e3a30 --- /dev/null +++ b/tools/custom-configuration/prometheus-k8s-secret/configmaps.json @@ -0,0 +1 @@ +{"items":[{"key":"monitoring/prometheus-k8s-rules","checksum":"a228d0b1ffcaee89767e0763cb589c5341fbb504d75e5e005a16a053b44a7460"}]} diff --git a/tools/custom-configuration/prometheus-k8s-secret/prometheus.yaml b/tools/custom-configuration/prometheus-k8s-secret/prometheus.yaml new file mode 100644 index 0000000..0198f4f --- /dev/null +++ b/tools/custom-configuration/prometheus-k8s-secret/prometheus.yaml @@ -0,0 +1,463 @@ +global: + evaluation_interval: 30s + scrape_interval: 30s + external_labels: {} +rule_files: +- /etc/prometheus/rules/rules-0/* +scrape_configs: +- job_name: monitoring/alertmanager/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_alertmanager + regex: main + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: web + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - target_label: endpoint + replacement: web +- job_name: monitoring/kube-apiserver/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scrape_interval: 30s + scheme: https + tls_config: + insecure_skip_verify: false + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server_name: kubernetes + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_component + regex: apiserver + - action: keep + source_labels: + - __meta_kubernetes_service_label_provider + regex: kubernetes + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: https + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_component + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: https +- job_name: monitoring/kube-controller-manager/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kube-controller-manager + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics +- job_name: monitoring/kube-dns/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kube-dns + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics-skydns + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics-skydns +- job_name: monitoring/kube-dns/1 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kube-dns + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics-dnsmasq + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics-dnsmasq +- job_name: monitoring/kube-scheduler/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kube-scheduler + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics +- job_name: monitoring/kube-state-metrics/0 + honor_labels: true + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kube-state-metrics + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics +- job_name: monitoring/kubelet/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kubelet + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics +- job_name: monitoring/kubelet/1 + honor_labels: true + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - kube-system + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: kubelet + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: cadvisor + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: cadvisor +- job_name: monitoring/node-exporter/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: node-exporter + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http-metrics + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + regex: (.+) + replacement: ${1} + - target_label: endpoint + replacement: http-metrics +- job_name: monitoring/prometheus/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + scrape_interval: 30s + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_prometheus + regex: k8s + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: web + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - target_label: endpoint + replacement: web +- job_name: monitoring/prometheus-operator/0 + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_label_k8s_app + regex: prometheus-operator + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: http + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - target_label: endpoint + replacement: http +alerting: + alertmanagers: + - path_prefix: / + scheme: http + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - monitoring + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_service_name + regex: alertmanager-main + - action: keep + source_labels: + - __meta_kubernetes_endpoint_port_name + regex: web diff --git a/tools/custom-configuration/prometheus-k8s.yaml b/tools/custom-configuration/prometheus-k8s.yaml new file mode 100644 index 0000000..18b56b5 --- /dev/null +++ b/tools/custom-configuration/prometheus-k8s.yaml @@ -0,0 +1,34 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: k8s + labels: + prometheus: k8s +spec: + replicas: 2 + version: PROMETHEUS_VERSION + externalUrl: PROMETHEUS_EXTERNAL_URL + serviceAccountName: prometheus-k8s + serviceMonitorSelector: + ruleSelector: + matchLabels: + role: prometheus-rulefiles + prometheus: k8s + resources: + storage: + volumeClaimTemplate: + metadata: + annotations: + annotation1: prometheus + spec: + storageClassName: ssd + resources: + requests: + storage: 40Gi + requests: + memory: 2Gi + alerting: + alertmanagers: + - namespace: monitoring + name: alertmanager-main + port: web diff --git a/tools/custom-configuration/update-config b/tools/custom-configuration/update-config new file mode 100755 index 0000000..d31ea9f --- /dev/null +++ b/tools/custom-configuration/update-config @@ -0,0 +1,84 @@ +#!/bin/bash + +######################################################################################### +#environment configuration +######################################################################################### +RED='\033[0;31m' +GREEN='\033[0;32m' +ORANGE='\033[0;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +if [ -z "${KUBECONFIG}" ]; then + export KUBECONFIG=~/.kube/config +fi + +if [ -z "${NAMESPACE}" ]; then + NAMESPACE=monitoring +fi + +kctl() { + kubectl --namespace "$NAMESPACE" "$@" +} + +########################################################################################### +#set prometheus version and externalUrl +########################################################################################### +echo -e "${BLUE}Exporting existing configurations" +tput sgr0 +echo +if [ $(kctl get secret prometheus-k8s --no-headers | cut -d " " -f1) ] && [ $(kctl get prometheus k8s --no-headers | cut -d " " -f1) ]; then + PROMETHEUS_VERSION=$(kubectl get prometheus k8s -o 'jsonpath={.spec.version}' -n monitoring) + PROMETHEUS_EXTERNAL_URL=$(kubectl get prometheus k8s -o 'jsonpath={.spec.externalUrl}' -n monitoring) + sed -i -e 's/PROMETHEUS_VERSION/'"$PROMETHEUS_VERSION"'/g' ./prometheus-k8s.yaml + sed -i -e 's,PROMETHEUS_EXTERNAL_URL,'"$PROMETHEUS_EXTERNAL_URL"',g' ./prometheus-k8s.yaml +else + echo -e "${BLUE}Could not find the Prometheus object and secret. Please deploy Prometheus Operator before updating." + tput sgr0 + exit +fi + +########################################################################################### +#Remove "sed" generated files +########################################################################################### +echo +echo -e "${BLUE}Removing all the sed generated files" +find . -name "*.yaml-e" -exec rm -rf {} \; +tput sgr0 + +################################################################################################################################# +#export existing Prometheus configuration +########################################################################################################################################## +echo +echo -e "${BLUE}Exporting prometheus.yaml key to ./prometheus-k8s-secret/prometheus.yaml" +kctl get secret prometheus-k8s -o 'jsonpath={.data.prometheus\.yaml}' | base64 --decode > prometheus-k8s-secret/prometheus.yaml +tput sgr0 +echo +echo -e "${BLUE}Exporting configmaps.json key to ./prometheus-k8s-secret/configmaps.json" +kctl get secret prometheus-k8s -o 'jsonpath={.data.configmaps\.json}' | base64 --decode > prometheus-k8s-secret/configmaps.json +tput sgr0 +echo + +########################################################################################### +#delete existing Prometheus object and secret +########################################################################################### +echo -e "${BLUE}Delete prometheus object and secret" +tput sgr0 +kctl delete prometheus k8s +echo + +########################################################################################### +#Create new Prometheus secret and deploy new Prometheus +########################################################################################### +echo +echo -e "${BLUE}Please make the desired changes in ./prometheus-k8s-secret/prometheus.yaml and press Y to continue." +tput sgr0 +read -p "Y/N [Y]: " update_config + +if [[ $update_config =~ ^([yY]|[yY])$ ]]; then + kctl create secret generic prometheus-k8s --from-file=./prometheus-k8s-secret + kctl create -f ./prometheus-k8s.yaml +fi + +echo +echo -e "${GREEN}Done" +tput sgr0