From 2294c467bdd04d9d2ccf13b813681e8522209214 Mon Sep 17 00:00:00 2001 From: Rudraksh Pareek Date: Wed, 23 Oct 2024 18:55:13 +0530 Subject: [PATCH 1/2] feat: advanced kube-bench config Signed-off-by: Rudraksh Pareek --- cis-k8s-job/README.md | 23 ++-- cis-k8s-job/templates/_helpers.tpl | 141 ++++++++++++++++++++++++ cis-k8s-job/templates/cis-cron-job.yaml | 79 ++----------- cis-k8s-job/templates/cis-job.yaml | 90 +++------------ cis-k8s-job/values.yaml | 9 +- 5 files changed, 185 insertions(+), 157 deletions(-) diff --git a/cis-k8s-job/README.md b/cis-k8s-job/README.md index 86097a6..65bf669 100644 --- a/cis-k8s-job/README.md +++ b/cis-k8s-job/README.md @@ -6,14 +6,21 @@ Instruction to perform CIS Benchmark on Cluster via AccuKnox CIS K8s Job - [Helm (v3.13.1 or later)](https://v3.helm.sh/docs/intro/install/) ### Parameters: -| Variable | Sample Value | Description | -| -------------------- | ---------------------- | -------------------------- | -| accuknox.url | cspm.demo.accuknox.com | AccuKnox CSPM Endpoint URL | -| accuknox.tenantId | 2 | AccuKnox Tenant ID | -| accuknox.label | CIS | AccuKnox Label | -| accuknox.authToken | $token | AccuKnox Token | -| accuknox.clusterName | $clusterName | Cluster Name | -| accuknox.cronTab | 30 9 * * * | CronJob (UTC) | +| Variable | Sample Value | Description | +| -------------------- | -------------------------------- | ------------------------------------ | +| accuknox.url | cspm.demo.accuknox.com | AccuKnox CSPM Endpoint URL | +| accuknox.tenantId | 2 | AccuKnox Tenant ID | +| accuknox.label | CIS | AccuKnox Label | +| accuknox.authToken | $token | AccuKnox Token | +| accuknox.clusterName | $clusterName | Cluster Name | +| accuknox.cronTab | 30 9 * * * | CronJob (UTC) | +| toolConfig.platform | "GKE" OR "AKS" | Name of the platform. Default: empty | +| toolConfig.nodeType | master OR controlplane | For node selection | +| toolConfig.targets | "master,controlplane,node" | [Ref](https://github.com/aquasecurity/kube-bench/blob/main/docs/flags-and-commands.md#specifying-benchmark-sections) | +| toolConfig.benchmark | "gke-1.6.0" | [Ref](https://github.com/aquasecurity/kube-bench/blob/main/docs/platforms.md) | +| toolConfig.check | "1.1.1,1.2.3" | Control IDs to check | +| toolConfig.skip | "1.1.1,1.3.1" | Control IDs to skip | + ## Schedule CIS Job on cluster #### Clone GitHub and switch to CIS K8s Job folder diff --git a/cis-k8s-job/templates/_helpers.tpl b/cis-k8s-job/templates/_helpers.tpl index 5eaf546..8e9ab4d 100644 --- a/cis-k8s-job/templates/_helpers.tpl +++ b/cis-k8s-job/templates/_helpers.tpl @@ -60,3 +60,144 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{- define "volumes" }} +- name: datapath + emptyDir: {} +- name: var-lib-kubelet + hostPath: + path: "/var/lib/kubelet" +- name: etc-systemd + hostPath: + path: "/etc/systemd" +- name: etc-kubernetes + hostPath: + path: "/etc/kubernetes" +{{- if .platform | empty }} +- name: var-lib-cni + hostPath: + path: /var/lib/cni +- hostPath: + path: /var/lib/etcd + name: var-lib-etcd +- hostPath: + path: /var/lib/kube-scheduler + name: var-lib-kube-scheduler +- hostPath: + path: /var/lib/kube-controller-manager + name: var-lib-kube-controller-manager +- hostPath: + path: /lib/systemd + name: lib-systemd +- hostPath: + path: /srv/kubernetes + name: srv-kubernetes +- hostPath: + path: /usr/bin + name: usr-bin +- hostPath: + path: /etc/cni/net.d/ + name: etc-cni-netd +- hostPath: + path: /opt/cni/bin/ + name: opt-cni-bin +{{- else if eq .platform "GKE" }} +- name: home-kubernetes + hostPath: + path: "/home/kubernetes" +{{- else if eq .platform "AKS" }} +- name: etc-default + hostPath: + path: "/etc/default" +{{- end }} +{{- end }} + +{{- define "volumeMounts" }} +- mountPath: /data + name: datapath +- name: var-lib-kubelet + mountPath: /var/lib/kubelet + readOnly: true +- name: etc-systemd + mountPath: /etc/systemd + readOnly: true +- name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true +{{- if .platform | empty }} +- name: var-lib-cni + mountPath: /var/lib/cni + readOnly: true +- mountPath: /var/lib/etcd + name: var-lib-etcd + readOnly: true +- mountPath: /var/lib/kube-scheduler + name: var-lib-kube-scheduler + readOnly: true +- mountPath: /var/lib/kube-controller-manager + name: var-lib-kube-controller-manager + readOnly: true +- mountPath: /lib/systemd/ + name: lib-systemd + readOnly: true +- mountPath: /srv/kubernetes/ + name: srv-kubernetes + readOnly: true +- mountPath: /usr/local/mount-from-host/bin + name: usr-bin + readOnly: true +- mountPath: /etc/cni/net.d/ + name: etc-cni-netd + readOnly: true +- mountPath: /opt/cni/bin/ + name: opt-cni-bin + readOnly: true +{{- else if eq .platform "GKE" }} +- name: home-kubernetes + mountPath: /home/kubernetes + readOnly: true +{{- else if eq .platform "AKS" }} +- name: etc-default + mountPath: /etc/default + readOnly: true +{{- end }} +{{- end }} + +{{- define "cmd" }} +- kube-bench +- run +- --json +- --outputfile=/data/report.json +{{- if not (.targets | empty) }} +- --targets="{{ .targets }}" +{{- end }} +{{- if not (.benchmark | empty) }} +- --benchmark="{{ .benchmark }}" +{{- end }} +{{- if not (.check | empty) }} +- --check="{{ .check }}" +{{- end }} +{{- if not (.skip | empty) }} +- --skip="{{ .skip }}" +{{- end }} +{{- end }} + +{{- define "masterConfig" }} +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists +tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule +{{- end }} diff --git a/cis-k8s-job/templates/cis-cron-job.yaml b/cis-k8s-job/templates/cis-cron-job.yaml index bd3842b..cd7a81f 100644 --- a/cis-k8s-job/templates/cis-cron-job.yaml +++ b/cis-k8s-job/templates/cis-cron-job.yaml @@ -10,6 +10,9 @@ spec: spec: template: spec: + {{- if or (or (contains "master" .Values.toolConfig.nodeType) (contains "controlplane" .Values.toolConfig.nodeType)) (or (contains "master" .Values.toolConfig.targets) (contains "controlplane" .Values.toolConfig.targets)) }} + {{- include "masterConfig" .Values.toolConfig | trim | nindent 10 }} + {{- end }} {{- if .Values.imagePullSecrets.name }} imagePullSecrets: - name: {{ .Values.imagePullSecrets.name }} @@ -45,83 +48,15 @@ spec: name: datapath initContainers: - image: "{{ .Values.kubeBench.image.repository }}:{{ .Values.kubeBench.image.tag }}" - command: ["/bin/sh", "-c"] - args: ["kube-bench run --json > /data/report.json"] + command: + {{- include "cmd" .Values.toolConfig | trim | nindent 13 }} name: kube-bench volumeMounts: - - mountPath: /data - name: datapath - - mountPath: /var/lib/etcd - name: var-lib-etcd - readOnly: true - - mountPath: /var/lib/kubelet - name: var-lib-kubelet - readOnly: true - - mountPath: /var/lib/kube-scheduler - name: var-lib-kube-scheduler - readOnly: true - - mountPath: /var/lib/kube-controller-manager - name: var-lib-kube-controller-manager - readOnly: true - - mountPath: /etc/systemd - name: etc-systemd - readOnly: true - - mountPath: /lib/systemd/ - name: lib-systemd - readOnly: true - - mountPath: /srv/kubernetes/ - name: srv-kubernetes - readOnly: true - - mountPath: /etc/kubernetes - name: etc-kubernetes - readOnly: true - - mountPath: /usr/local/mount-from-host/bin - name: usr-bin - readOnly: true - - mountPath: /etc/cni/net.d/ - name: etc-cni-netd - readOnly: true - - mountPath: /opt/cni/bin/ - name: opt-cni-bin - readOnly: true + {{- include "volumeMounts" .Values.toolConfig | trim | nindent 13 }} hostPID: true restartPolicy: Never volumes: - - name: datapath - emptyDir: {} - - hostPath: - path: /var/lib/etcd - name: var-lib-etcd - - hostPath: - path: /var/lib/kubelet - name: var-lib-kubelet - - hostPath: - path: /var/lib/kube-scheduler - name: var-lib-kube-scheduler - - hostPath: - path: /var/lib/kube-controller-manager - name: var-lib-kube-controller-manager - - hostPath: - path: /etc/systemd - name: etc-systemd - - hostPath: - path: /lib/systemd - name: lib-systemd - - hostPath: - path: /srv/kubernetes - name: srv-kubernetes - - hostPath: - path: /etc/kubernetes - name: etc-kubernetes - - hostPath: - path: /usr/bin - name: usr-bin - - hostPath: - path: /etc/cni/net.d/ - name: etc-cni-netd - - hostPath: - path: /opt/cni/bin/ - name: opt-cni-bin + {{- include "volumes" .Values.toolConfig | trim | nindent 11 }} schedule: "{{ .Values.accuknox.cronTab }}" successfulJobsHistoryLimit: 1 diff --git a/cis-k8s-job/templates/cis-job.yaml b/cis-k8s-job/templates/cis-job.yaml index e4dd4a6..5402181 100644 --- a/cis-k8s-job/templates/cis-job.yaml +++ b/cis-k8s-job/templates/cis-job.yaml @@ -4,18 +4,24 @@ metadata: name: cis-k8s-job namespace: {{ .Release.Namespace }} spec: + # keep the job for 2 hours after successful completion + ttlSecondsAfterFinished: 7200 template: metadata: name: cis-k8s-job labels: - app: cis-k8s-job + app: cis-k8s-job spec: + {{- if or (or (contains "master" .Values.toolConfig.nodeType) (contains "controlplane" .Values.toolConfig.nodeType)) (or (contains "master" .Values.toolConfig.targets) (contains "controlplane" .Values.toolConfig.targets)) }} + {{- include "masterConfig" .Values.toolConfig | trim | nindent 6 }} + {{- end }} + {{- if .Values.imagePullSecrets.name }} imagePullSecrets: - name: {{ .Values.imagePullSecrets.name }} - {{- end }} + {{- end }} containers: - - image: "{{ .Values.accuknoxJob.image.repository }}:{{ .Values.accuknoxJob.image.tag }}" + - image: "{{ .Values.accuknoxJob.image.repository }}:{{ .Values.accuknoxJob.image.tag }}" command: ["/bin/sh", "-c"] args: ['/bin/sh entrypoint.sh && curl --location --request POST "https://${URL}/api/v1/artifact/?tenant_id=${TENANT_ID}&data_type=KB&label_id=${LABEL_NAME}&save_to_s3=true" --header "Tenant-Id: ${TENANT_ID}" --header "Authorization: Bearer ${AUTH_TOKEN}" --form "file=@\"./data/report.json\"" && cat /data/report.json'] name: cis-k8s-cronjob @@ -44,81 +50,13 @@ spec: - mountPath: /data name: datapath initContainers: - - image: "{{ .Values.kubeBench.image.repository }}:{{ .Values.kubeBench.image.tag }}" - command: ["/bin/sh", "-c"] - args: ["kube-bench run --json > /data/report.json"] + - image: "{{ .Values.kubeBench.image.repository }}:{{ .Values.kubeBench.image.tag }}" + command: + {{- include "cmd" .Values.toolConfig | trim | nindent 9 }} name: kube-bench volumeMounts: - - mountPath: /data - name: datapath - - mountPath: /var/lib/etcd - name: var-lib-etcd - readOnly: true - - mountPath: /var/lib/kubelet - name: var-lib-kubelet - readOnly: true - - mountPath: /var/lib/kube-scheduler - name: var-lib-kube-scheduler - readOnly: true - - mountPath: /var/lib/kube-controller-manager - name: var-lib-kube-controller-manager - readOnly: true - - mountPath: /etc/systemd - name: etc-systemd - readOnly: true - - mountPath: /lib/systemd/ - name: lib-systemd - readOnly: true - - mountPath: /srv/kubernetes/ - name: srv-kubernetes - readOnly: true - - mountPath: /etc/kubernetes - name: etc-kubernetes - readOnly: true - - mountPath: /usr/local/mount-from-host/bin - name: usr-bin - readOnly: true - - mountPath: /etc/cni/net.d/ - name: etc-cni-netd - readOnly: true - - mountPath: /opt/cni/bin/ - name: opt-cni-bin - readOnly: true + {{- include "volumeMounts" .Values.toolConfig | trim | nindent 9 }} hostPID: true restartPolicy: Never volumes: - - name: datapath - emptyDir: {} - - hostPath: - path: /var/lib/etcd - name: var-lib-etcd - - hostPath: - path: /var/lib/kubelet - name: var-lib-kubelet - - hostPath: - path: /var/lib/kube-scheduler - name: var-lib-kube-scheduler - - hostPath: - path: /var/lib/kube-controller-manager - name: var-lib-kube-controller-manager - - hostPath: - path: /etc/systemd - name: etc-systemd - - hostPath: - path: /lib/systemd - name: lib-systemd - - hostPath: - path: /srv/kubernetes - name: srv-kubernetes - - hostPath: - path: /etc/kubernetes - name: etc-kubernetes - - hostPath: - path: /usr/bin - name: usr-bin - - hostPath: - path: /etc/cni/net.d/ - name: etc-cni-netd - - hostPath: - path: /opt/cni/bin/ - name: opt-cni-bin + {{- include "volumes" .Values.toolConfig | trim | nindent 7 }} diff --git a/cis-k8s-job/values.yaml b/cis-k8s-job/values.yaml index 24e863a..371e374 100644 --- a/cis-k8s-job/values.yaml +++ b/cis-k8s-job/values.yaml @@ -11,7 +11,7 @@ accuknoxJob: kubeBench: image: repository: docker.io/aquasec/kube-bench - tag: "v0.6.19" + tag: "v0.9.1" # To use existing secret updated {imagePullSecrets.name} with your secret name. imagePullSecrets: @@ -20,6 +20,13 @@ imagePullSecrets: username: "" password: "" +toolConfig: + platform: "" + nodeType: "worker" + targets: "" + benchmark: "" + check: "" + skip: "" accuknox: authToken: "NO-TOKEN-SET" From be376cc184edc169f4e7af7762bfd0a75c4628fc Mon Sep 17 00:00:00 2001 From: Rudraksh Pareek Date: Thu, 24 Oct 2024 09:15:32 +0530 Subject: [PATCH 2/2] chore: update TTL in other jobs Signed-off-by: Rudraksh Pareek --- k8s-risk-assessment-job/templates/job.yaml | 2 ++ k8tls-job/templates/k8tls-job.yaml | 2 ++ kiem-job/templates/job.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/k8s-risk-assessment-job/templates/job.yaml b/k8s-risk-assessment-job/templates/job.yaml index f78bf66..8200398 100644 --- a/k8s-risk-assessment-job/templates/job.yaml +++ b/k8s-risk-assessment-job/templates/job.yaml @@ -4,6 +4,8 @@ metadata: name: k8s-risk-assessment-job namespace: {{ .Release.Namespace }} spec: + # keep the job for 2 hours after successful completion + ttlSecondsAfterFinished: 7200 template: metadata: labels: diff --git a/k8tls-job/templates/k8tls-job.yaml b/k8tls-job/templates/k8tls-job.yaml index 7aef681..2a4125d 100644 --- a/k8tls-job/templates/k8tls-job.yaml +++ b/k8tls-job/templates/k8tls-job.yaml @@ -4,6 +4,8 @@ metadata: name: k8tls-job namespace: {{ .Release.Namespace }} spec: + # keep the job for 2 hours after successful completion + ttlSecondsAfterFinished: 7200 template: metadata: name: k8tls-job diff --git a/kiem-job/templates/job.yaml b/kiem-job/templates/job.yaml index a612d73..11651df 100644 --- a/kiem-job/templates/job.yaml +++ b/kiem-job/templates/job.yaml @@ -4,6 +4,8 @@ metadata: name: kiem-job namespace: {{ .Release.Namespace }} spec: + # keep the job for 2 hours after successful completion + ttlSecondsAfterFinished: 7200 template: metadata: labels: