From 65f6048dfe63a762103193398a71ca5cd7b23902 Mon Sep 17 00:00:00 2001 From: Lorenzo Garuti Date: Tue, 25 Jul 2023 17:30:54 +0200 Subject: [PATCH] Added example ECK deployment --- deployments/README.md | 141 +++++++++++++++++++++++++++ deployments/eck-elastic.yml | 21 ++++ deployments/eck-filebeat.yml | 95 ++++++++++++++++++ deployments/eck-kibana.yml | 10 ++ deployments/eck-metricbeat.yml | 173 +++++++++++++++++++++++++++++++++ 5 files changed, 440 insertions(+) create mode 100644 deployments/README.md create mode 100644 deployments/eck-elastic.yml create mode 100644 deployments/eck-filebeat.yml create mode 100644 deployments/eck-kibana.yml create mode 100644 deployments/eck-metricbeat.yml diff --git a/deployments/README.md b/deployments/README.md new file mode 100644 index 0000000..8ff46df --- /dev/null +++ b/deployments/README.md @@ -0,0 +1,141 @@ +## Deploy ECK on Kubernetes + +In this guide we will install [ECK](https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-deploy-eck.html) on Kubernetes. + +First of all install CRDs and ECK operator: + +``` +kubectl create -f https://download.elastic.co/downloads/eck/2.8.0/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.8.0/operator.yaml +``` + +Then we need to configure our Storage Class ([EFS CSI Driver with dynamic profisioning](https://github.com/kubernetes-sigs/aws-efs-csi-driver)) where elasticsearch will store and persist data: + +```yaml +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: efs-eck-sc # Don't change this name since is used on eck-elastic.yml deloyment +provisioner: efs.csi.aws.com +parameters: + provisioningMode: efs-ap + fileSystemId: # get the id form AWS console + directoryPerms: "755" + basePath: "/eck-storage-dynamic" # optional. Choose an appropriate name +``` + +Apply the above deployments and then we are ready to deploy elasticsearch: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-elastic.yml +``` + +Check the status of the newly created pods, pv and pvc: + +``` +kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 0/1 Init:0/2 0 2s +k8s-eck-es-default-1 0/1 Init:0/2 0 2s +k8s-eck-es-default-2 0/1 Init:0/2 0 2s + +root@i-097c1a2b2f1022439:~/eck# kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-0d766371-f9a4-4210-abe5-077748808643 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-0 efs-eck-sc 34s +pvc-6290aa54-f41b-4705-99fe-f69efddeb168 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-1 efs-eck-sc 34s +pvc-e8e7a076-f8c3-4a93-8239-44b5ca8696fa 20Gi RWO Delete Bound default/elasticsearch-data-k8s-eck-es-default-2 efs-eck-sc 34s + +root@i-097c1a2b2f1022439:~/eck# kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +elasticsearch-data-k8s-eck-es-default-0 Bound pvc-0d766371-f9a4-4210-abe5-077748808643 20Gi RWO efs-eck-sc 35s +elasticsearch-data-k8s-eck-es-default-1 Bound pvc-6290aa54-f41b-4705-99fe-f69efddeb168 20Gi RWO efs-eck-sc 35s +elasticsearch-data-k8s-eck-es-default-2 Bound pvc-e8e7a076-f8c3-4a93-8239-44b5ca8696fa 20Gi RWO efs-eck-sc 35s +``` + +Wait until the elasticsearch pods are ready: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 3m3s +k8s-eck-es-default-1 1/1 Running 0 3m3s +k8s-eck-es-default-2 1/1 Running 0 3m3s +``` + +Now we can deploy Kibana with: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-kibana.yml +``` + +Wait until kibana is up & running and check for the kibana service name: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 9m52s +k8s-eck-es-default-1 1/1 Running 0 9m52s +k8s-eck-es-default-2 1/1 Running 0 9m52s +k8s-eck-kibana-kb-56c4fb4bf8-vc9ct 1/1 Running 0 3m31s + +kubectl get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +k8s-eck-es-default ClusterIP None 9200/TCP 9m54s +k8s-eck-es-http ClusterIP 10.107.103.161 9200/TCP 9m55s +k8s-eck-es-internal-http ClusterIP 10.101.251.215 9200/TCP 9m55s +k8s-eck-es-transport ClusterIP None 9300/TCP 9m55s +k8s-eck-kibana-kb-http ClusterIP 10.102.152.26 5601/TCP 3m34s +kubernetes ClusterIP 10.96.0.1 443/TCP 51m +``` + +Now create an ingress rule with the above deployment and apply it: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: eck-kibana-ingress + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" +spec: + ingressClassName: nginx + rules: + - host: eck.yourdomain.com # FQDN in a domain that you manage. Create a CNAME record that point to the public LB DNS name + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: k8s-eck-kibana-kb-http + port: + number: 5601 +``` + +Now apply filebeat and metricbeat deployments to get some data into elasticsearch: + +``` +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-filebeat.yml +kubectl apply -f https://raw.githubusercontent.com/garutilorenzo/k8s-aws-terraform-cluster/master/deployments/eck-metricbeat.yml +``` + +And wait that all the pods are ready: + +``` +root@i-097c1a2b2f1022439:~/eck# kubectl get pods +NAME READY STATUS RESTARTS AGE +k8s-eck-es-default-0 1/1 Running 0 54m +k8s-eck-es-default-1 1/1 Running 0 54m +k8s-eck-es-default-2 1/1 Running 0 54m +k8s-eck-filebeat-beat-filebeat-76s9x 1/1 Running 4 (11m ago) 12m +k8s-eck-filebeat-beat-filebeat-pn77d 1/1 Running 4 (11m ago) 12m +k8s-eck-filebeat-beat-filebeat-wjkhm 1/1 Running 4 (11m ago) 12m +k8s-eck-kibana-kb-77d89694bc-vbp7s 1/1 Running 0 19m +k8s-eck-metricbeat-beat-metricbeat-8kpkl 1/1 Running 1 (7m36s ago) 8m1s +k8s-eck-metricbeat-beat-metricbeat-fl28t 1/1 Running 0 8m1s +k8s-eck-metricbeat-beat-metricbeat-knn2j 1/1 Running 1 (6m16s ago) 8m1s +``` + +Finally login to the Kibana UI on https://eck.yourdomain.com. Check [here](https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-deploy-kibana.html) how to get the elastic password. \ No newline at end of file diff --git a/deployments/eck-elastic.yml b/deployments/eck-elastic.yml new file mode 100644 index 0000000..b187f22 --- /dev/null +++ b/deployments/eck-elastic.yml @@ -0,0 +1,21 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: k8s-eck +spec: + version: 8.8.2 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false + volumeClaimTemplates: + - metadata: + name: elasticsearch-data # Do not change this name unless you set up a volume mount for the data path. + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: efs-eck-sc \ No newline at end of file diff --git a/deployments/eck-filebeat.yml b/deployments/eck-filebeat.yml new file mode 100644 index 0000000..148cbb8 --- /dev/null +++ b/deployments/eck-filebeat.yml @@ -0,0 +1,95 @@ +--- +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: k8s-eck-filebeat +spec: + type: filebeat + version: 8.8.2 + elasticsearchRef: + name: k8s-eck + kibanaRef: + name: k8s-eck-kibana + config: + filebeat.autodiscover.providers: + - node: ${NODE_NAME} + type: kubernetes + hints.default_config.enabled: "false" + templates: + - condition.equals.kubernetes.namespace: default + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + containers: + - name: filebeat + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat + namespace: default +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deployments/eck-kibana.yml b/deployments/eck-kibana.yml new file mode 100644 index 0000000..6f90311 --- /dev/null +++ b/deployments/eck-kibana.yml @@ -0,0 +1,10 @@ +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: k8s-eck-kibana +spec: + version: 8.8.2 + count: 1 + elasticsearchRef: + name: k8s-eck \ No newline at end of file diff --git a/deployments/eck-metricbeat.yml b/deployments/eck-metricbeat.yml new file mode 100644 index 0000000..0249643 --- /dev/null +++ b/deployments/eck-metricbeat.yml @@ -0,0 +1,173 @@ +--- +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: k8s-eck-metricbeat +spec: + type: metricbeat + version: 8.8.2 + elasticsearchRef: + name: k8s-eck + kibanaRef: + name: k8s-eck-kibana + config: + metricbeat: + autodiscover: + providers: + - hints: + default_config: {} + enabled: "true" + node: ${NODE_NAME} + type: kubernetes + modules: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + process: + include_top_n: + by_cpu: 5 + by_memory: 5 + processes: + - .* + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event: + when: + regexp: + system: + filesystem: + mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/) + - module: kubernetes + period: 10s + node: ${NODE_NAME} + hosts: + - https://${NODE_NAME}:10250 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl: + verification_mode: none + metricsets: + - node + - system + - pod + - container + - volume + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: metricbeat + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + containers: + - args: + - -e + - -c + - /etc/beat.yml + - -system.hostfs=/hostfs + name: metricbeat + volumeMounts: + - mountPath: /hostfs/sys/fs/cgroup + name: cgroup + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostfs/proc + name: proc + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /proc + name: proc +--- +# permissions needed for metricbeat +# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + namespace: default +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - events + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "extensions" + resources: + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - deployments + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat + namespace: default +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: default +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io \ No newline at end of file