Duplicate Kustomize Config for Tekton Log Collector

Until kubernetes-sigs/kustomize#5271 is resolved, we will need to duplicate this.
redhat-appstudio · Dec 18, 2024 · fa7f8f6 · fa7f8f6
1 parent bee1f9e
commit fa7f8f6
Show file tree

Hide file tree

Showing 16 changed files with 386 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,6 @@ cosign.pub
 tmp
 .idea/*
 components/pipeline-service/base/log-collector/charts/*
+components/pipeline-service/staging/stone-stage-p01/log-collector/charts/*
+components/pipeline-service/staging/stone-stg-rh01/log-collector/charts/*
+components/pipeline-service/development/log-collector/charts/*
diff --git a/components/pipeline-service/development/kustomization.yaml b/components/pipeline-service/development/kustomization.yaml
@@ -11,4 +11,4 @@ resources:
   - main-pipeline-service-configuration.yaml
   - dev-only-pipeline-service-storage-configuration.yaml
   - ../base/rbac
-  - ../base/log-collector
+  - ./log-collector
diff --git a/...ice/base/log-collector/kustomization.yaml → ...elopment/log-collector/kustomization.yaml b/...ice/base/log-collector/kustomization.yaml → ...elopment/log-collector/kustomization.yaml
diff --git a/.../log-collector/vector-helm-generator.yaml → .../log-collector/vector-helm-generator.yaml b/.../log-collector/vector-helm-generator.yaml → .../log-collector/vector-helm-generator.yaml
diff --git a/...ase/log-collector/vector-helm-values.yaml → ...ent/log-collector/vector-helm-values.yaml b/...ase/log-collector/vector-helm-values.yaml → ...ent/log-collector/vector-helm-values.yaml
diff --git a/...ervice/base/log-collector/vector-pre.yaml → ...development/log-collector/vector-pre.yaml b/...ervice/base/log-collector/vector-pre.yaml → ...development/log-collector/vector-pre.yaml
diff --git a/components/pipeline-service/staging/stone-stage-p01/kustomization.yaml b/components/pipeline-service/staging/stone-stage-p01/kustomization.yaml
@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - deploy.yaml
+  - ./log-collector
diff --git a/components/pipeline-service/staging/stone-stage-p01/log-collector/kustomization.yaml b/components/pipeline-service/staging/stone-stage-p01/log-collector/kustomization.yaml
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: tekton-logging
+# Skip applying the Tekton/PaC operands while the Tekton/PaC operator is being installed.
+# See more information about this option, here:
+# https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/#skip-dry-run-for-new-custom-resources-types
+commonAnnotations:
+  argocd.argoproj.io/sync-wave: "-1"
+
+generators:
+  - vector-helm-generator.yaml
+
+resources:
+  - vector-pre.yaml
+
diff --git a/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-helm-generator.yaml b/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-helm-generator.yaml
@@ -0,0 +1,10 @@
+apiVersion: builtin
+kind: HelmChartInflationGenerator
+metadata:
+  name: vector
+name: vector
+repo: https://helm.vector.dev
+version: 0.38.1
+releaseName: vector-tekton-logs-collector
+namespace: tekton-logging
+valuesFile: vector-helm-values.yaml
diff --git a/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-helm-values.yaml b/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-helm-values.yaml
@@ -0,0 +1,110 @@
+---
+role: Agent
+customConfig:
+  data_dir: /vector-data-dir
+  api:
+    enabled: true
+    address: 127.0.0.1:8686
+    playground: false
+  sources:
+    kubernetes_logs:
+      type: kubernetes_logs
+      rotate_wait_secs: 5
+      glob_minimum_cooldown_ms: 15000
+      auto_partial_merge: true
+      extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)"
+    internal_metrics:
+      type: internal_metrics
+  transforms:
+    remap_app_logs:
+      type: remap
+      inputs: [kubernetes_logs]
+      source: |-
+        .log_type = "application"
+        .kubernetes_namespace_name = .kubernetes.pod_namespace
+            if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") {
+              .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID")
+            } else {
+              .taskRunUID = "none"
+              }
+            if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") {
+              .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID")
+            .result = .pipelineRunUID
+            } else {
+               .result = .taskRunUID
+            }
+            if exists(.kubernetes.pod_labels."tekton.dev/task") {
+              .task = del(.kubernetes.pod_labels."tekton.dev/task")
+            } else {
+              .task = "none"
+            }
+            if exists(.kubernetes.pod_namespace) {
+              .namespace = del(.kubernetes.pod_namespace)
+            } else {
+              .namespace = "unlabeled"
+            }
+            .pod = .kubernetes.pod_name
+            .container = .kubernetes.container_name
+  sinks:
+    aws_s3:
+      type: "aws_s3"
+      bucket: ${BUCKET}
+      buffer:
+        type: "disk"
+        max_size: 1073741824
+      inputs: ["remap_app_logs"]
+      compression: "none"
+      endpoint: ${ENDPOINT}
+      encoding:
+        codec: "text"
+      key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}"
+      filename_time_format: ""
+      filename_append_uuid: false
+env:
+  - name: AWS_ACCESS_KEY_ID
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_access_key_id
+  - name: AWS_SECRET_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_secret_access_key
+  - name: AWS_DEFAULT_REGION
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_region
+  - name: BUCKET
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: bucket
+  - name: ENDPOINT
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: endpoint
+tolerations:
+  - effect: NoSchedule
+    key: konflux-ci.dev/workload
+    operator: Exists
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - CHOWN
+    - DAC_OVERRIDE
+    - FOWNER
+    - FSETID
+    - KILL
+    - NET_BIND_SERVICE
+    - SETGID
+    - SETPCAP
+    - SETUID
+  readOnlyRootFilesystem: true
+  seLinuxOptions:
+    type: spc_t
+  seccompProfile:
+    type: RuntimeDefault
diff --git a/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-pre.yaml b/components/pipeline-service/staging/stone-stage-p01/log-collector/vector-pre.yaml
@@ -0,0 +1,54 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tekton-logging
+---
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: false
+allowPrivilegedContainer: false
+allowedCapabilities: null
+apiVersion: security.openshift.io/v1
+defaultAddCapabilities: null
+defaultAllowPrivilegeEscalation: false
+forbiddenSysctls:
+- '*'
+fsGroup:
+  type: RunAsAny
+groups: []
+kind: SecurityContextConstraints
+metadata:
+  name: logging-scc
+  namespace: tekton-logging
+priority: null
+readOnlyRootFilesystem: true
+requiredDropCapabilities:
+- CHOWN
+- DAC_OVERRIDE
+- FSETID
+- FOWNER
+- SETGID
+- SETUID
+- SETPCAP
+- NET_BIND_SERVICE
+- KILL
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: RunAsAny
+seccompProfiles:
+- runtime/default
+supplementalGroups:
+  type: RunAsAny
+users:
+- system:serviceaccount:tekton-logging:vector-tekton-logs-collector
+volumes:
+- configMap
+- emptyDir
+- hostPath
+- projected
+- secret
diff --git a/components/pipeline-service/staging/stone-stg-rh01/kustomization.yaml b/components/pipeline-service/staging/stone-stg-rh01/kustomization.yaml
@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - deploy.yaml
+  - ./log-collector
diff --git a/components/pipeline-service/staging/stone-stg-rh01/log-collector/kustomization.yaml b/components/pipeline-service/staging/stone-stg-rh01/log-collector/kustomization.yaml
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: tekton-logging
+# Skip applying the Tekton/PaC operands while the Tekton/PaC operator is being installed.
+# See more information about this option, here:
+# https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/#skip-dry-run-for-new-custom-resources-types
+commonAnnotations:
+  argocd.argoproj.io/sync-wave: "-1"
+
+generators:
+  - vector-helm-generator.yaml
+
+resources:
+  - vector-pre.yaml
+
diff --git a/components/pipeline-service/staging/stone-stg-rh01/log-collector/vector-helm-generator.yaml b/components/pipeline-service/staging/stone-stg-rh01/log-collector/vector-helm-generator.yaml
@@ -0,0 +1,10 @@
+apiVersion: builtin
+kind: HelmChartInflationGenerator
+metadata:
+  name: vector
+name: vector
+repo: https://helm.vector.dev
+version: 0.38.1
+releaseName: vector-tekton-logs-collector
+namespace: tekton-logging
+valuesFile: vector-helm-values.yaml
diff --git a/components/pipeline-service/staging/stone-stg-rh01/log-collector/vector-helm-values.yaml b/components/pipeline-service/staging/stone-stg-rh01/log-collector/vector-helm-values.yaml
@@ -0,0 +1,110 @@
+---
+role: Agent
+customConfig:
+  data_dir: /vector-data-dir
+  api:
+    enabled: true
+    address: 127.0.0.1:8686
+    playground: false
+  sources:
+    kubernetes_logs:
+      type: kubernetes_logs
+      rotate_wait_secs: 5
+      glob_minimum_cooldown_ms: 15000
+      auto_partial_merge: true
+      extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)"
+    internal_metrics:
+      type: internal_metrics
+  transforms:
+    remap_app_logs:
+      type: remap
+      inputs: [kubernetes_logs]
+      source: |-
+        .log_type = "application"
+        .kubernetes_namespace_name = .kubernetes.pod_namespace
+            if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") {
+              .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID")
+            } else {
+              .taskRunUID = "none"
+              }
+            if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") {
+              .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID")
+            .result = .pipelineRunUID
+            } else {
+               .result = .taskRunUID
+            }
+            if exists(.kubernetes.pod_labels."tekton.dev/task") {
+              .task = del(.kubernetes.pod_labels."tekton.dev/task")
+            } else {
+              .task = "none"
+            }
+            if exists(.kubernetes.pod_namespace) {
+              .namespace = del(.kubernetes.pod_namespace)
+            } else {
+              .namespace = "unlabeled"
+            }
+            .pod = .kubernetes.pod_name
+            .container = .kubernetes.container_name
+  sinks:
+    aws_s3:
+      type: "aws_s3"
+      bucket: ${BUCKET}
+      buffer:
+        type: "disk"
+        max_size: 1073741824
+      inputs: ["remap_app_logs"]
+      compression: "none"
+      endpoint: ${ENDPOINT}
+      encoding:
+        codec: "text"
+      key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}"
+      filename_time_format: ""
+      filename_append_uuid: false
+env:
+  - name: AWS_ACCESS_KEY_ID
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_access_key_id
+  - name: AWS_SECRET_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_secret_access_key
+  - name: AWS_DEFAULT_REGION
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: aws_region
+  - name: BUCKET
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: bucket
+  - name: ENDPOINT
+    valueFrom:
+      secretKeyRef:
+        name: tekton-results-s3
+        key: endpoint
+tolerations:
+  - effect: NoSchedule
+    key: konflux-ci.dev/workload
+    operator: Exists
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - CHOWN
+    - DAC_OVERRIDE
+    - FOWNER
+    - FSETID
+    - KILL
+    - NET_BIND_SERVICE
+    - SETGID
+    - SETPCAP
+    - SETUID
+  readOnlyRootFilesystem: true
+  seLinuxOptions:
+    type: spc_t
+  seccompProfile:
+    type: RuntimeDefault