From 1dc2028d04c014888abf1f06431ac81ea6d93c06 Mon Sep 17 00:00:00 2001 From: Hien To Date: Tue, 20 Feb 2024 14:56:40 +0700 Subject: [PATCH] Change CICD for common chart and and auto-instrument with opentelemetry operator --- .github/workflows/helm-package.yaml | 7 +- charts/common/templates/deployment.yaml | 3 +- charts/common/templates/external-secret.yaml | 61 +++++++ charts/common/templates/hpa.yaml | 3 +- charts/common/templates/ingress.yaml | 2 +- charts/common/templates/instrumentation.yaml | 164 +++++++++++++++++++ charts/common/templates/keda-auto-scale.yaml | 3 +- charts/common/templates/security-pod.yaml | 3 +- charts/common/templates/service-preview.yaml | 4 +- charts/common/templates/service.yaml | 4 +- charts/common/templates/vpa.yaml | 28 ++++ charts/common/values.yaml | 61 +++++-- 12 files changed, 316 insertions(+), 27 deletions(-) create mode 100644 charts/common/templates/instrumentation.yaml create mode 100644 charts/common/templates/vpa.yaml diff --git a/.github/workflows/helm-package.yaml b/.github/workflows/helm-package.yaml index 9d7c3a1..7b2fde7 100644 --- a/.github/workflows/helm-package.yaml +++ b/.github/workflows/helm-package.yaml @@ -12,6 +12,8 @@ on: jobs: packing: + permissions: + contents: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -22,6 +24,7 @@ jobs: - uses: azure/setup-helm@v3 with: token: ${{ secrets.GITHUB_TOKEN }} # only needed if version is 'latest' + - name: check linting run: | helm lint charts/* @@ -39,7 +42,7 @@ jobs: - name: Helm repo login run: | - helm registry login ${{ secrets.ACR_URL }} --username ${{ secrets.ACR_USERNAME }} --password ${{ secrets.ACR_PASSWORD }} + helm registry login ghcr.io --username USERNAME --password ${{ secrets.GITHUB_TOKEN }} - name: 'Auto Increase Helm Version' run: | @@ -76,7 +79,7 @@ jobs: echo "$dir" helm package $dir version=$(cat $dir/Chart.yaml | grep "version: " | cut -d " " -f 2) - helm push $dir-$version.tgz oci://${{ secrets.ACR_URL }}/helm + helm push $dir-$version.tgz oci://ghcr.io/jan done - name: 'Commit new version to main and create tag' run: | diff --git a/charts/common/templates/deployment.yaml b/charts/common/templates/deployment.yaml index 8755821..60e2de3 100644 --- a/charts/common/templates/deployment.yaml +++ b/charts/common/templates/deployment.yaml @@ -12,7 +12,7 @@ {{- $chartAppVersion := .Chart.AppVersion -}} {{- range $key, $value := .Values.workloads }} ---- + apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: @@ -159,4 +159,5 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} +--- {{- end}} \ No newline at end of file diff --git a/charts/common/templates/external-secret.yaml b/charts/common/templates/external-secret.yaml index d5dde7c..1a292fc 100644 --- a/charts/common/templates/external-secret.yaml +++ b/charts/common/templates/external-secret.yaml @@ -39,4 +39,65 @@ spec: {{- end }} +{{- if eq .Values.externalSecret.backendType "secretsManager" }} +apiVersion: "kubernetes-client.io/v1" +kind: ExternalSecret +metadata: + name: {{ include "common.externalSecretName" . }} + labels: + {{- include "common.labels" . | nindent 4 }} + {{- with .Values.externalSecret.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + backendType: {{ .Values.externalSecret.backendType }} + {{- if .Values.externalSecret.roleArn }} + roleArn: {{ .Values.externalSecret.roleArn }} + {{- end }} + {{- if .Values.externalSecret.controllerId }} + controllerId: {{ .Values.externalSecret.controllerId }} + {{- end }} + {{- if .Values.externalSecret.region }} + region: {{ .Values.externalSecret.region }} + {{- end }} + {{- with .Values.externalSecret.data }} + data: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.externalSecret.dataFrom }} + dataFrom: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.externalSecret.template }} + template: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if eq .Values.externalSecret.backendType "vaults" }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "common.externalSecretName" . }} + labels: + {{- include "common.labels" . | nindent 4 }} + {{- with .Values.externalSecret.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + refreshInterval: 15s + secretStoreRef: + name: vault-secret-manager + kind: ClusterSecretStore + target: + name: {{ .Values.externalSecret.name }} + creationPolicy: Owner + dataFrom: + - extract: + key: {{ .Values.externalSecret.dataFrom }} + +{{- end }} + {{- end }} diff --git a/charts/common/templates/hpa.yaml b/charts/common/templates/hpa.yaml index ece388d..f527107 100644 --- a/charts/common/templates/hpa.yaml +++ b/charts/common/templates/hpa.yaml @@ -4,7 +4,7 @@ {{- range $key, $value := .Values.workloads }} ---- + {{- if $value.autoscaling.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -38,4 +38,5 @@ spec: type: Utilization {{- end }} {{- end }} +--- {{- end }} diff --git a/charts/common/templates/ingress.yaml b/charts/common/templates/ingress.yaml index 4e87a33..f2359f8 100644 --- a/charts/common/templates/ingress.yaml +++ b/charts/common/templates/ingress.yaml @@ -3,7 +3,6 @@ {{- $capabilities := .Capabilities -}} {{- range $key, $value := .Values.workloads }} ---- {{- if $value.ingress.enabled -}} {{- $svcPort := $value.service.port -}} @@ -66,4 +65,5 @@ spec: {{- end }} {{- end }} {{- end }} +--- {{- end }} diff --git a/charts/common/templates/instrumentation.yaml b/charts/common/templates/instrumentation.yaml new file mode 100644 index 0000000..2dfda27 --- /dev/null +++ b/charts/common/templates/instrumentation.yaml @@ -0,0 +1,164 @@ +{{- range $key, $value := .Values.workloads }} +--- +{{- if $value.instrumentation.enabled -}} +{{- if eq $value.instrumentation.type "python" }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: {{ $value.instrumentation.name }} +spec: + env: + - name: OTEL_EXPORTER_OTLP_TIMEOUT + value: "20" + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "{{ $value.instrumentation.sampler | default "1" }}" + - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED + value: "true" + exporter: + endpoint: {{ $value.instrumentation.endpoint }} + propagators: + {{- if $value.instrumentation.xray }} + - xray + {{- else }} + - tracecontext + - baggage + {{- end }} + sampler: + type: parentbased_traceidratio + argument: "{{ $value.instrumentation.sampler | default "1" }}" + python: + env: + {{- if $value.instrumentation.xray }} + - name: OTEL_PYTHON_ID_GENERATOR + value: xray + {{- end }} + - name: OTEL_PYTHON_INSTRUMENTATION_SANITIZE_REDIS + value: "true" + - name: OTEL_PYTHON_LOG_CORRELATION + value: "true" + - name: OTEL_LOG_LEVEL + value: "debug" + - name: OTEL_PYTHON_LOG_LEVEL + value: "debug" + - name: OTEL_TRACES_EXPORTER + value: otlp_proto_http + - name: OTEL_METRICS_EXPORTER + value: none + - name: OTEL_LOG_EXPORTER + value: otlp_proto_http + - name: OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + value: "http/protobuf" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $value.instrumentation.endpoint }} + - name: OTEL_SERVICE_NAME + value: {{ $value.instrumentation.servicename }} + image: {{ $value.instrumentation.image }} +{{- end }} + +{{- if eq $value.instrumentation.type "dotnet" }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: {{ $value.instrumentation.name }} +spec: + env: + - name: OTEL_EXPORTER_OTLP_TIMEOUT + value: "20" + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "{{ $value.instrumentation.sampler | default "1" }}" + - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED + value: "true" + exporter: + endpoint: {{ $value.instrumentation.endpoint }} + propagators: + {{- if $value.instrumentation.xray }} + - xray + {{- else }} + - tracecontext + - baggage + {{- end }} + sampler: + type: parentbased_traceidratio + argument: "{{ $value.instrumentation.sampler | default "1" }}" + dotnet: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $value.instrumentation.endpoint }} + - name: OTEL_SERVICE_NAME + value: {{ $value.instrumentation.servicename }} + image: {{ $value.instrumentation.image }} + +{{- end }} + +{{- if eq $value.instrumentation.type "nodejs" }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: {{ $value.instrumentation.name }} +spec: + env: + - name: OTEL_EXPORTER_OTLP_TIMEOUT + value: "20" + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "{{ $value.instrumentation.sampler | default "1" }}" + - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED + value: "true" + exporter: + endpoint: {{ $value.instrumentation.endpoint }} + propagators: + {{- if $value.instrumentation.xray }} + - xray + {{- else }} + - tracecontext + - baggage + {{- end }} + sampler: + type: parentbased_traceidratio + argument: "{{ $value.instrumentation.sampler | default "1" }}" + nodejs: + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $value.instrumentation.endpoint }} + - name: OTEL_SERVICE_NAME + value: {{ $value.instrumentation.servicename }} + +{{- end }} + +{{- if eq $value.instrumentation.type "nginx" }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: {{ $value.instrumentation.name }} +spec: + exporter: + endpoint: {{ $value.instrumentation.endpoint }} + propagators: + {{- if $value.instrumentation.xray }} + - xray + {{- else }} + - tracecontext + - baggage + {{- end }} + sampler: + type: parentbased_traceidratio + argument: "{{ $value.instrumentation.sampler | default "1" }}" + nginx: + configFile: {{ $value.instrumentation.nginxConfigPath }} + attrs: + - name: NginxModuleServiceName + value: {{ $value.instrumentation.servicename }} + - name: NginxModuleServiceNamespace + value: {{ $value.instrumentation.servicename }} + - name: NginxModuleServiceInstanceId + value: {{ $value.instrumentation.servicename }} + +{{- end }} + +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/common/templates/keda-auto-scale.yaml b/charts/common/templates/keda-auto-scale.yaml index e69f3f1..0baa7b1 100644 --- a/charts/common/templates/keda-auto-scale.yaml +++ b/charts/common/templates/keda-auto-scale.yaml @@ -4,7 +4,7 @@ {{- range $key, $value := .Values.workloads }} ---- + {{- if not $value.autoscaling.enabled }} {{- if $value.kedaScaling.enabled }} apiVersion: keda.sh/v1alpha1 @@ -32,4 +32,5 @@ spec: type: prometheus {{- end }} {{- end }} +--- {{- end }} diff --git a/charts/common/templates/security-pod.yaml b/charts/common/templates/security-pod.yaml index 829c6b8..009554d 100644 --- a/charts/common/templates/security-pod.yaml +++ b/charts/common/templates/security-pod.yaml @@ -3,7 +3,7 @@ {{- $selectorLabels := include "common.selectorLabels" . -}} {{- range $key, $value := .Values.workloads }} ---- + {{- if $value.podSecurityGroup.enabled -}} apiVersion: vpcresources.k8s.aws/v1beta1 kind: SecurityGroupPolicy @@ -19,4 +19,5 @@ spec: groupIds: {{- toYaml $value.podSecurityGroup.securitygroupid | nindent 6 }} {{- end}} +--- {{- end}} diff --git a/charts/common/templates/service-preview.yaml b/charts/common/templates/service-preview.yaml index 7eb6f8d..dd48345 100644 --- a/charts/common/templates/service-preview.yaml +++ b/charts/common/templates/service-preview.yaml @@ -4,7 +4,7 @@ {{- $selectorLabels := include "common.selectorLabels" . -}} {{- range $key, $value := .Values.workloads }} ---- + apiVersion: v1 kind: Service metadata: @@ -23,5 +23,5 @@ spec: {{- $selectorLabels | nindent 4 }} app.kubernetes.io/component: {{ $value.name }} app.kubernetes.io/name: {{ $name }}-{{ $value.name }} - +--- {{- end }} \ No newline at end of file diff --git a/charts/common/templates/service.yaml b/charts/common/templates/service.yaml index b83e1de..386b807 100644 --- a/charts/common/templates/service.yaml +++ b/charts/common/templates/service.yaml @@ -4,7 +4,7 @@ {{- $selectorLabels := include "common.selectorLabels" . -}} {{- range $key, $value := .Values.workloads }} ---- + apiVersion: v1 kind: Service metadata: @@ -26,5 +26,5 @@ spec: {{- $selectorLabels | nindent 4 }} app.kubernetes.io/component: {{ $value.name }} app.kubernetes.io/name: {{ $name }}-{{ $value.name }} - +--- {{- end }} diff --git a/charts/common/templates/vpa.yaml b/charts/common/templates/vpa.yaml new file mode 100644 index 0000000..61e4ba4 --- /dev/null +++ b/charts/common/templates/vpa.yaml @@ -0,0 +1,28 @@ + +{{- $name := include "common.name" . -}} +{{- $fullname := include "common.fullname" . -}} +{{- $labels := include "common.labels" . -}} +{{- $selectorLabels := include "common.selectorLabels" . -}} +{{- $chartName := .Chart.Name -}} + +{{- range $key, $value := .Values.workloads }} +--- +{{- if $value.vpa.enabled }} +apiVersion: autoscaling.k8s.io/v1beta2 +kind: VerticalPodAutoscaler +metadata: + name: {{ $name }}-vpa-{{ $value.name }} + labels: + app.kubernetes.io/name: {{ $name }}-{{ $value.name }} + {{- $labels | nindent 4 }} +spec: + targetRef: + apiVersion: argoproj.io/v1alpha1 + kind: Rollout + name: {{ $fullname }}-deployment-{{ $value.name }} + updatePolicy: + updateMode: {{ printf "%s" $value.vpa.updateMode }} + resourcePolicy: + {{- toYaml $value.vpa.resourcePolicy | nindent 6 }} +{{- end }} +{{ end -}} \ No newline at end of file diff --git a/charts/common/values.yaml b/charts/common/values.yaml index 06bd172..7289e0b 100644 --- a/charts/common/values.yaml +++ b/charts/common/values.yaml @@ -16,16 +16,28 @@ externalSecret: create: false name: "common-secret" annotations: {} - # Currently, this chart only support 1 backends azureKeyVaults - backendType: azureKeyVaults - userAssignedIdentityID: "" # Set the clientID of the user-assigned managed identity to use. Staging will be "564839d0-6cd1-45f7-aba7-c8ba2389ebab" - keyvaultName: "" # Name of the keyvault, staging will be "jankeyvaultstaging" - tenantId: "378d177e-843b-4c8f-891f-3dd3c23fe060" - data: - - key: FOO # The environment variable key that will be present inside pod - objectName: FOO # The Secret Object key on Azure Keyvaults - - key: FOO2 # The environment variable key that will be present inside pod - objectName: FOO2 # The Secret Object key on Azure Keyvaults + # Currently, this chart only support 3 backends: azureKeyVaults, vaults and secretsManager + + # Azure KeyVaults + # backendType: azureKeyVaults + # userAssignedIdentityID: "" # Set the clientID of the user-assigned managed identity to use. + # keyvaultName: "" # Name of the keyvault + # tenantId: "" + # data: + # - key: FOO # The environment variable key that will be present inside pod + # objectName: FOO # The Secret Object key on Azure Keyvaults + # - key: FOO2 # The environment variable key that will be present inside pod + # objectName: FOO2 # The Secret Object key on Azure Keyvaults + + # AWS Secrets Manager or HashiCorp Vault + # backendType: secretsManager/vaults + # roleArn: "" + # controllerId: "" + # region: "us-east-1" + # data: [] + # dataFrom: + # - prod/common_module/env + # template: {} nameOverride: "common-module" fullnameOverride: "common-module" @@ -73,6 +85,18 @@ workloads: pathType: Prefix tls: [] + instrumentation: + enabled: false + name: common-api-prod + endpoint: http://otel-collector.opentelemetry-operator:4318 + image: "" + servicename: common-api-prod + sampler: 1 + xray: false + # support types: python, dotnet, nodejs, nginx + type: nginx + nginxConfigPath: /etc/nginx/nginx.conf + podAnnotations: {} podSecurityContext: {} @@ -110,22 +134,22 @@ workloads: maxReplicas: 3 targetCPUUtilizationPercentage: 95 targetMemoryUtilizationPercentage: 95 - + kedaScaling: - enabled: false # ignore if autoscaling.enable = true + enabled: false # ignore if autoscaling.enable = true cooldownPeriod: 30 pollingInterval: 2 minReplicas: 1 maxReplicas: 5 metricName: celery_queue_length - query: celery_queue_length{queue_name="myqueue"} # change queue_name here + query: celery_queue_length{queue_name="myqueue"} # change queue_name here serverAddress: http://prometheus-stag-kube-prome-prometheus.monitoring.svc:9090 - threshold: '3' - + threshold: "3" + nodeSelector: {} tolerations: [] - + podSecurityGroup: enabled: false securitygroupid: [] @@ -133,3 +157,8 @@ workloads: # Reloader Option reloader: "true" + # VPA Option + vpa: + enabled: false + updateMode: "Off" + resourcePolicy: {}