From e08c903438b25dfadff556eeacc7725e7cbf3401 Mon Sep 17 00:00:00 2001 From: Khurram Baig Date: Wed, 18 Dec 2024 08:15:26 +0530 Subject: [PATCH] Install Tekton Logs Collector using Kustomize generator ArgoCD CR's Application/ApplicationSet CR isn't available in every cluster. So we switched to using kustomize generator for vector installation. --- .github/workflows/kube-linter.yaml | 1 + .gitignore | 1 + .../infra-deployments/kustomization.yaml | 1 + .../kustomization.yaml | 6 + .../vector-tekton-logs-collector.yaml | 41 ++++ .../overlays/development/kustomization.yaml | 5 + .../kustomization.yaml | 5 + .../production-downstream/kustomization.yaml | 5 + .../main-pipeline-service-configuration.yaml | 190 --------------- .../staging/base/kustomization.yaml | 1 + .../main-pipeline-service-configuration.yaml | 229 +----------------- .../staging/stone-stage-p01/deploy.yaml | 200 +-------------- .../staging/stone-stg-rh01/deploy.yaml | 198 +-------------- .../base/kustomization.yaml | 16 ++ .../base/vector-helm-generator.yaml | 10 + .../base/vector-helm-values.yaml | 110 +++++++++ .../base/vector-pre.yaml | 54 +++++ .../development/kustomization.yaml | 4 + .../production/kustomization.yaml | 4 + .../staging/kustomization.yaml | 4 + hack/generate-deploy-config.sh | 2 +- 21 files changed, 279 insertions(+), 808 deletions(-) create mode 100644 argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/kustomization.yaml create mode 100644 argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/vector-tekton-logs-collector.yaml create mode 100644 components/vector-tekton-logs-collector/base/kustomization.yaml create mode 100644 components/vector-tekton-logs-collector/base/vector-helm-generator.yaml create mode 100644 components/vector-tekton-logs-collector/base/vector-helm-values.yaml create mode 100644 components/vector-tekton-logs-collector/base/vector-pre.yaml create mode 100644 components/vector-tekton-logs-collector/development/kustomization.yaml create mode 100644 components/vector-tekton-logs-collector/production/kustomization.yaml create mode 100644 components/vector-tekton-logs-collector/staging/kustomization.yaml diff --git a/.github/workflows/kube-linter.yaml b/.github/workflows/kube-linter.yaml index 1b4e2a21606..56ef9432df8 100644 --- a/.github/workflows/kube-linter.yaml +++ b/.github/workflows/kube-linter.yaml @@ -24,6 +24,7 @@ jobs: - name: Run kustomize build run: | + rm -rf components/pipeline-service/base/log-collector/charts && \ find argo-cd-apps components -name 'kustomization.yaml' \ ! -path '*/k-components/*' \ ! -path 'components/repository-validator/staging/*' \ diff --git a/.gitignore b/.gitignore index 88418819536..7c5b98666ed 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ cosign.pub .tmp/ tmp .idea/* +components/pipeline-service/base/log-collector/charts/* diff --git a/argo-cd-apps/base/member/infra-deployments/kustomization.yaml b/argo-cd-apps/base/member/infra-deployments/kustomization.yaml index eaeb8b46306..e4816af4eeb 100644 --- a/argo-cd-apps/base/member/infra-deployments/kustomization.yaml +++ b/argo-cd-apps/base/member/infra-deployments/kustomization.yaml @@ -29,5 +29,6 @@ resources: - konflux-ui - konflux-rbac - konflux-info + - vector-tekton-logs-collector components: - ../../../k-components/inject-infra-deployments-repo-details diff --git a/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/kustomization.yaml b/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/kustomization.yaml new file mode 100644 index 00000000000..cdd7692e940 --- /dev/null +++ b/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- vector-tekton-logs-collector.yaml +components: + - ../../../../k-components/deploy-to-member-cluster-merge-generator diff --git a/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/vector-tekton-logs-collector.yaml b/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/vector-tekton-logs-collector.yaml new file mode 100644 index 00000000000..aadedd33051 --- /dev/null +++ b/argo-cd-apps/base/member/infra-deployments/vector-tekton-logs-collector/vector-tekton-logs-collector.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: vector-tekton-logs-collector +spec: + generators: + - merge: + mergeKeys: + - nameNormalized + generators: + - clusters: + values: + sourceRoot: components/vector-tekton-logs-collector + environment: staging + clusterDir: "" + - list: + elements: [] + template: + metadata: + name: vector-tekton-logs-collector-{{nameNormalized}} + spec: + project: default + source: + path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}' + repoURL: https://github.com/redhat-appstudio/infra-deployments.git + targetRevision: main + destination: + namespace: tekton-logging + server: '{{server}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + retry: + limit: -1 + backoff: + duration: 10s + factor: 2 + maxDuration: 3m diff --git a/argo-cd-apps/overlays/development/kustomization.yaml b/argo-cd-apps/overlays/development/kustomization.yaml index ab2a83c08f0..6c64f51e096 100644 --- a/argo-cd-apps/overlays/development/kustomization.yaml +++ b/argo-cd-apps/overlays/development/kustomization.yaml @@ -184,3 +184,8 @@ patches: kind: ApplicationSet version: v1alpha1 name: crossplane-control-plane + - path: development-overlay-patch.yaml + target: + kind: ApplicationSet + version: v1alpha1 + name: vector-tekton-logs-collector diff --git a/argo-cd-apps/overlays/konflux-public-production/kustomization.yaml b/argo-cd-apps/overlays/konflux-public-production/kustomization.yaml index 3facfcc3f27..cc5541d03c9 100644 --- a/argo-cd-apps/overlays/konflux-public-production/kustomization.yaml +++ b/argo-cd-apps/overlays/konflux-public-production/kustomization.yaml @@ -201,3 +201,8 @@ patches: kind: ApplicationSet version: v1alpha1 name: konflux-info + - path: production-overlay-patch.yaml + target: + kind: ApplicationSet + version: v1alpha1 + name: vector-tekton-logs-collector diff --git a/argo-cd-apps/overlays/production-downstream/kustomization.yaml b/argo-cd-apps/overlays/production-downstream/kustomization.yaml index 3b165f2918a..283689a0150 100644 --- a/argo-cd-apps/overlays/production-downstream/kustomization.yaml +++ b/argo-cd-apps/overlays/production-downstream/kustomization.yaml @@ -201,3 +201,8 @@ patches: kind: ApplicationSet version: v1alpha1 name: konflux-info + - path: production-overlay-patch.yaml + target: + kind: ApplicationSet + version: v1alpha1 + name: vector-tekton-logs-collector diff --git a/components/pipeline-service/development/main-pipeline-service-configuration.yaml b/components/pipeline-service/development/main-pipeline-service-configuration.yaml index 1da5039be16..016105adb6b 100644 --- a/components/pipeline-service/development/main-pipeline-service-configuration.yaml +++ b/components/pipeline-service/development/main-pipeline-service-configuration.yaml @@ -1767,196 +1767,6 @@ spec: - name: AUTOINSTALL_COMPONENTS value: "false" --- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: vectors-tekton-logs-collector - namespace: openshift-gitops -spec: - destination: - namespace: tekton-logging - server: https://kubernetes.default.svc - project: default - source: - path: charts/vector - repoURL: 'https://github.com/vectordotdev/helm-charts' - targetRevision: "08506fdc01c7cc3fcf2dd83102add7b44980ee23" - helm: - valueFiles: - - values.yaml - values: |- - role: Agent - customConfig: - data_dir: /vector-data-dir - api: - enabled: true - address: 127.0.0.1:8686 - playground: false - sources: - kubernetes_logs: - type: kubernetes_logs - rotate_wait_secs: 5 - glob_minimum_cooldown_ms: 15000 - auto_partial_merge: true - extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)" - internal_metrics: - type: internal_metrics - transforms: - remap_app_logs: - type: remap - inputs: [kubernetes_logs] - source: |- - .log_type = "application" - .kubernetes_namespace_name = .kubernetes.pod_namespace - if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") { - .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID") - } else { - .taskRunUID = "none" - } - if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { - .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") - .result = .pipelineRunUID - } else { - .result = .taskRunUID - } - if exists(.kubernetes.pod_labels."tekton.dev/task") { - .task = del(.kubernetes.pod_labels."tekton.dev/task") - } else { - .task = "none" - } - if exists(.kubernetes.pod_namespace) { - .namespace = del(.kubernetes.pod_namespace) - } else { - .namespace = "unlabeled" - } - .pod = .kubernetes.pod_name - .container = .kubernetes.container_name - sinks: - aws_s3: - type: "aws_s3" - bucket: ${BUCKET} - buffer: - type: "disk" - max_size: 1073741824 - inputs: ["remap_app_logs"] - compression: "none" - endpoint: ${ENDPOINT} - encoding: - codec: "text" - key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" - filename_time_format: "" - filename_append_uuid: false - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_access_key_id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_secret_access_key - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_region - - name: BUCKET - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: bucket - - name: ENDPOINT - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: endpoint - tolerations: - - effect: NoSchedule - key: konflux-ci.dev/workload - operator: Exists - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - CHOWN - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - NET_BIND_SERVICE - - SETGID - - SETPCAP - - SETUID - readOnlyRootFilesystem: true - seLinuxOptions: - type: spc_t - seccompProfile: - type: RuntimeDefault - syncPolicy: - automated: - prune: true - selfHeal: true - retry: - backoff: - duration: 10s - factor: 2 - maxDuration: 3m - limit: -1 - syncOptions: - - CreateNamespace=true - - Validate=false ---- -allowHostDirVolumePlugin: true -allowHostIPC: false -allowHostNetwork: false -allowHostPID: false -allowHostPorts: false -allowPrivilegeEscalation: false -allowPrivilegedContainer: false -allowedCapabilities: null -apiVersion: security.openshift.io/v1 -defaultAddCapabilities: null -defaultAllowPrivilegeEscalation: false -forbiddenSysctls: -- '*' -fsGroup: - type: RunAsAny -groups: [] -kind: SecurityContextConstraints -metadata: - name: logging-scc - namespace: tekton-logging -priority: null -readOnlyRootFilesystem: true -requiredDropCapabilities: -- CHOWN -- DAC_OVERRIDE -- FSETID -- FOWNER -- SETGID -- SETUID -- SETPCAP -- NET_BIND_SERVICE -- KILL -runAsUser: - type: RunAsAny -seLinuxContext: - type: RunAsAny -seccompProfiles: -- runtime/default -supplementalGroups: - type: RunAsAny -users: -- system:serviceaccount:tekton-logging:vectors-tekton-logs-collector -volumes: -- configMap -- emptyDir -- hostPath -- projected -- secret ---- apiVersion: route.openshift.io/v1 kind: Route metadata: diff --git a/components/pipeline-service/staging/base/kustomization.yaml b/components/pipeline-service/staging/base/kustomization.yaml index 225a0a245a2..8b290c6a620 100644 --- a/components/pipeline-service/staging/base/kustomization.yaml +++ b/components/pipeline-service/staging/base/kustomization.yaml @@ -7,6 +7,7 @@ kind: Kustomization commonAnnotations: argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + resources: - main-pipeline-service-configuration.yaml - pipelines-as-code-secret.yaml diff --git a/components/pipeline-service/staging/base/main-pipeline-service-configuration.yaml b/components/pipeline-service/staging/base/main-pipeline-service-configuration.yaml index 29f1e3819fa..8bd556aa68a 100644 --- a/components/pipeline-service/staging/base/main-pipeline-service-configuration.yaml +++ b/components/pipeline-service/staging/base/main-pipeline-service-configuration.yaml @@ -19,16 +19,6 @@ metadata: name: tekton-results --- apiVersion: v1 -kind: Namespace -metadata: - annotations: - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true - argocd.argoproj.io/sync-wave: "-1" - labels: - argocd.argoproj.io/managed-by: openshift-gitops - name: tekton-logging ---- -apiVersion: v1 kind: ServiceAccount metadata: annotations: @@ -1692,224 +1682,15 @@ spec: - name: AUTOINSTALL_COMPONENTS value: "false" --- -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret +apiVersion: v1 +kind: Namespace metadata: annotations: argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true argocd.argoproj.io/sync-wave: "-1" - name: s3-conf - namespace: tekton-logging -spec: - dataFrom: - - extract: - key: integrations-output/terraform-resources/appsres07ue1/stonesoup-infra-stage/redhat-stg-plnsvc-s3 - refreshInterval: 1h - secretStoreRef: - kind: ClusterSecretStore - name: appsre-vault - target: - creationPolicy: Owner - deletionPolicy: Delete - name: tekton-results-s3 - template: - data: - aws_access_key_id: '{{ .aws_access_key_id }}' - aws_region: '{{ .aws_region }}' - aws_secret_access_key: '{{ .aws_secret_access_key }}' - bucket: '{{ .bucket }}' - endpoint: https://{{ .endpoint }} ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: vectors-tekton-logs-collector - namespace: openshift-gitops -spec: - destination: - namespace: tekton-logging - server: https://kubernetes.default.svc - project: default - source: - path: charts/vector - repoURL: 'https://github.com/vectordotdev/helm-charts' - targetRevision: "08506fdc01c7cc3fcf2dd83102add7b44980ee23" - helm: - valueFiles: - - values.yaml - values: |- - role: Agent - customConfig: - data_dir: /vector-data-dir - api: - enabled: true - address: 127.0.0.1:8686 - playground: false - sources: - kubernetes_logs: - type: kubernetes_logs - rotate_wait_secs: 5 - glob_minimum_cooldown_ms: 15000 - auto_partial_merge: true - extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)" - internal_metrics: - type: internal_metrics - transforms: - remap_app_logs: - type: remap - inputs: [kubernetes_logs] - source: |- - .log_type = "application" - .kubernetes_namespace_name = .kubernetes.pod_namespace - if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") { - .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID") - } else { - .taskRunUID = "none" - } - if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { - .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") - .result = .pipelineRunUID - } else { - .result = .taskRunUID - } - if exists(.kubernetes.pod_labels."tekton.dev/task") { - .task = del(.kubernetes.pod_labels."tekton.dev/task") - } else { - .task = "none" - } - if exists(.kubernetes.pod_namespace) { - .namespace = del(.kubernetes.pod_namespace) - } else { - .namespace = "unlabeled" - } - .pod = .kubernetes.pod_name - .container = .kubernetes.container_name - sinks: - aws_s3: - type: "aws_s3" - bucket: ${BUCKET} - buffer: - type: "disk" - max_size: 1073741824 - inputs: ["remap_app_logs"] - compression: "none" - endpoint: ${ENDPOINT} - encoding: - codec: "text" - key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" - filename_time_format: "" - filename_append_uuid: false - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_access_key_id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_secret_access_key - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_region - - name: BUCKET - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: bucket - - name: ENDPOINT - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: endpoint - tolerations: - - effect: NoSchedule - key: konflux-ci.dev/workload - operator: Equal - value: konflux-tenants - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - CHOWN - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - NET_BIND_SERVICE - - SETGID - - SETPCAP - - SETUID - readOnlyRootFilesystem: true - seLinuxOptions: - type: spc_t - seccompProfile: - type: RuntimeDefault - syncPolicy: - automated: - prune: true - selfHeal: true - retry: - backoff: - duration: 10s - factor: 2 - maxDuration: 3m - limit: -1 - syncOptions: - - CreateNamespace=false - - Validate=false ---- -allowHostDirVolumePlugin: true -allowHostIPC: false -allowHostNetwork: false -allowHostPID: false -allowHostPorts: false -allowPrivilegeEscalation: false -allowPrivilegedContainer: false -allowedCapabilities: null -apiVersion: security.openshift.io/v1 -defaultAddCapabilities: null -defaultAllowPrivilegeEscalation: false -forbiddenSysctls: -- '*' -fsGroup: - type: RunAsAny -groups: [] -kind: SecurityContextConstraints -metadata: - name: logging-scc - namespace: tekton-logging -priority: null -readOnlyRootFilesystem: true -requiredDropCapabilities: -- CHOWN -- DAC_OVERRIDE -- FSETID -- FOWNER -- SETGID -- SETUID -- SETPCAP -- NET_BIND_SERVICE -- KILL -runAsUser: - type: RunAsAny -seLinuxContext: - type: RunAsAny -seccompProfiles: -- runtime/default -supplementalGroups: - type: RunAsAny -users: -- system:serviceaccount:tekton-logging:vectors-tekton-logs-collector -volumes: -- configMap -- emptyDir -- hostPath -- projected -- secret + labels: + argocd.argoproj.io/managed-by: openshift-gitops + name: tekton-logging --- apiVersion: route.openshift.io/v1 kind: Route diff --git a/components/pipeline-service/staging/stone-stage-p01/deploy.yaml b/components/pipeline-service/staging/stone-stage-p01/deploy.yaml index 989395707ac..8b43e8686d7 100644 --- a/components/pipeline-service/staging/stone-stage-p01/deploy.yaml +++ b/components/pipeline-service/staging/stone-stage-p01/deploy.yaml @@ -1711,150 +1711,6 @@ spec: secret: secretName: tekton-results-tls --- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - annotations: - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true - name: vectors-tekton-logs-collector - namespace: openshift-gitops -spec: - destination: - namespace: tekton-logging - server: https://kubernetes.default.svc - project: default - source: - helm: - valueFiles: - - values.yaml - values: |- - role: Agent - customConfig: - data_dir: /vector-data-dir - api: - enabled: true - address: 127.0.0.1:8686 - playground: false - sources: - kubernetes_logs: - type: kubernetes_logs - rotate_wait_secs: 5 - glob_minimum_cooldown_ms: 15000 - auto_partial_merge: true - extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)" - internal_metrics: - type: internal_metrics - transforms: - remap_app_logs: - type: remap - inputs: [kubernetes_logs] - source: |- - .log_type = "application" - .kubernetes_namespace_name = .kubernetes.pod_namespace - if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") { - .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID") - } else { - .taskRunUID = "none" - } - if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { - .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") - .result = .pipelineRunUID - } else { - .result = .taskRunUID - } - if exists(.kubernetes.pod_labels."tekton.dev/task") { - .task = del(.kubernetes.pod_labels."tekton.dev/task") - } else { - .task = "none" - } - if exists(.kubernetes.pod_namespace) { - .namespace = del(.kubernetes.pod_namespace) - } else { - .namespace = "unlabeled" - } - .pod = .kubernetes.pod_name - .container = .kubernetes.container_name - sinks: - aws_s3: - type: "aws_s3" - bucket: ${BUCKET} - buffer: - type: "disk" - max_size: 1073741824 - inputs: ["remap_app_logs"] - compression: "none" - endpoint: ${ENDPOINT} - encoding: - codec: "text" - key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" - filename_time_format: "" - filename_append_uuid: false - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_access_key_id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_secret_access_key - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_region - - name: BUCKET - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: bucket - - name: ENDPOINT - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: endpoint - tolerations: - - effect: NoSchedule - key: konflux-ci.dev/workload - operator: Equal - value: konflux-tenants - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - CHOWN - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - NET_BIND_SERVICE - - SETGID - - SETPCAP - - SETUID - readOnlyRootFilesystem: true - seLinuxOptions: - type: spc_t - seccompProfile: - type: RuntimeDefault - path: charts/vector - repoURL: https://github.com/vectordotdev/helm-charts - targetRevision: 08506fdc01c7cc3fcf2dd83102add7b44980ee23 - syncPolicy: - automated: - prune: true - selfHeal: true - retry: - backoff: - duration: 10s - factor: 2 - maxDuration: 3m - limit: -1 - syncOptions: - - CreateNamespace=false - - Validate=false ---- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -1932,12 +1788,12 @@ metadata: annotations: argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true argocd.argoproj.io/sync-wave: "-1" - name: s3-conf + name: tekton-results-s3 namespace: tekton-logging spec: dataFrom: - extract: - key: integrations-output/terraform-resources/appsres07ue1/stonesoup-infra-stage/redhat-stg-plnsvc-s3 + key: integrations-output/terraform-resources/appsres09ue1/stone-stage-p01/stone-stage-p01-plnsvc-s3 refreshInterval: 1h secretStoreRef: kind: ClusterSecretStore @@ -1953,6 +1809,7 @@ spec: aws_secret_access_key: '{{ .aws_secret_access_key }}' bucket: '{{ .bucket }}' endpoint: https://{{ .endpoint }} + s3_url: s3://{{ .bucket }} --- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -2401,57 +2258,6 @@ spec: weight: 100 wildcardPolicy: None --- -allowHostDirVolumePlugin: true -allowHostIPC: false -allowHostNetwork: false -allowHostPID: false -allowHostPorts: false -allowPrivilegeEscalation: false -allowPrivilegedContainer: false -allowedCapabilities: null -apiVersion: security.openshift.io/v1 -defaultAddCapabilities: null -defaultAllowPrivilegeEscalation: false -forbiddenSysctls: -- '*' -fsGroup: - type: RunAsAny -groups: [] -kind: SecurityContextConstraints -metadata: - annotations: - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true - name: logging-scc - namespace: tekton-logging -priority: null -readOnlyRootFilesystem: true -requiredDropCapabilities: -- CHOWN -- DAC_OVERRIDE -- FSETID -- FOWNER -- SETGID -- SETUID -- SETPCAP -- NET_BIND_SERVICE -- KILL -runAsUser: - type: RunAsAny -seLinuxContext: - type: RunAsAny -seccompProfiles: -- runtime/default -supplementalGroups: - type: RunAsAny -users: -- system:serviceaccount:tekton-logging:vectors-tekton-logs-collector -volumes: -- configMap -- emptyDir -- hostPath -- projected -- secret ---- allowHostDirVolumePlugin: false allowHostIPC: false allowHostNetwork: false diff --git a/components/pipeline-service/staging/stone-stg-rh01/deploy.yaml b/components/pipeline-service/staging/stone-stg-rh01/deploy.yaml index 9b247d770a3..b381f116c99 100644 --- a/components/pipeline-service/staging/stone-stg-rh01/deploy.yaml +++ b/components/pipeline-service/staging/stone-stg-rh01/deploy.yaml @@ -1711,150 +1711,6 @@ spec: secret: secretName: tekton-results-tls --- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - annotations: - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true - name: vectors-tekton-logs-collector - namespace: openshift-gitops -spec: - destination: - namespace: tekton-logging - server: https://kubernetes.default.svc - project: default - source: - helm: - valueFiles: - - values.yaml - values: |- - role: Agent - customConfig: - data_dir: /vector-data-dir - api: - enabled: true - address: 127.0.0.1:8686 - playground: false - sources: - kubernetes_logs: - type: kubernetes_logs - rotate_wait_secs: 5 - glob_minimum_cooldown_ms: 15000 - auto_partial_merge: true - extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)" - internal_metrics: - type: internal_metrics - transforms: - remap_app_logs: - type: remap - inputs: [kubernetes_logs] - source: |- - .log_type = "application" - .kubernetes_namespace_name = .kubernetes.pod_namespace - if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") { - .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID") - } else { - .taskRunUID = "none" - } - if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { - .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") - .result = .pipelineRunUID - } else { - .result = .taskRunUID - } - if exists(.kubernetes.pod_labels."tekton.dev/task") { - .task = del(.kubernetes.pod_labels."tekton.dev/task") - } else { - .task = "none" - } - if exists(.kubernetes.pod_namespace) { - .namespace = del(.kubernetes.pod_namespace) - } else { - .namespace = "unlabeled" - } - .pod = .kubernetes.pod_name - .container = .kubernetes.container_name - sinks: - aws_s3: - type: "aws_s3" - bucket: ${BUCKET} - buffer: - type: "disk" - max_size: 1073741824 - inputs: ["remap_app_logs"] - compression: "none" - endpoint: ${ENDPOINT} - encoding: - codec: "text" - key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" - filename_time_format: "" - filename_append_uuid: false - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_access_key_id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_secret_access_key - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: aws_region - - name: BUCKET - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: bucket - - name: ENDPOINT - valueFrom: - secretKeyRef: - name: tekton-results-s3 - key: endpoint - tolerations: - - effect: NoSchedule - key: konflux-ci.dev/workload - operator: Equal - value: konflux-tenants - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - CHOWN - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - NET_BIND_SERVICE - - SETGID - - SETPCAP - - SETUID - readOnlyRootFilesystem: true - seLinuxOptions: - type: spc_t - seccompProfile: - type: RuntimeDefault - path: charts/vector - repoURL: https://github.com/vectordotdev/helm-charts - targetRevision: 08506fdc01c7cc3fcf2dd83102add7b44980ee23 - syncPolicy: - automated: - prune: true - selfHeal: true - retry: - backoff: - duration: 10s - factor: 2 - maxDuration: 3m - limit: -1 - syncOptions: - - CreateNamespace=false - - Validate=false ---- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -1932,7 +1788,7 @@ metadata: annotations: argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true argocd.argoproj.io/sync-wave: "-1" - name: s3-conf + name: tekton-results-s3 namespace: tekton-logging spec: dataFrom: @@ -1953,6 +1809,7 @@ spec: aws_secret_access_key: '{{ .aws_secret_access_key }}' bucket: '{{ .bucket }}' endpoint: https://{{ .endpoint }} + s3_url: s3://{{ .bucket }} --- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -2412,57 +2269,6 @@ spec: weight: 100 wildcardPolicy: None --- -allowHostDirVolumePlugin: true -allowHostIPC: false -allowHostNetwork: false -allowHostPID: false -allowHostPorts: false -allowPrivilegeEscalation: false -allowPrivilegedContainer: false -allowedCapabilities: null -apiVersion: security.openshift.io/v1 -defaultAddCapabilities: null -defaultAllowPrivilegeEscalation: false -forbiddenSysctls: -- '*' -fsGroup: - type: RunAsAny -groups: [] -kind: SecurityContextConstraints -metadata: - annotations: - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true - name: logging-scc - namespace: tekton-logging -priority: null -readOnlyRootFilesystem: true -requiredDropCapabilities: -- CHOWN -- DAC_OVERRIDE -- FSETID -- FOWNER -- SETGID -- SETUID -- SETPCAP -- NET_BIND_SERVICE -- KILL -runAsUser: - type: RunAsAny -seLinuxContext: - type: RunAsAny -seccompProfiles: -- runtime/default -supplementalGroups: - type: RunAsAny -users: -- system:serviceaccount:tekton-logging:vectors-tekton-logs-collector -volumes: -- configMap -- emptyDir -- hostPath -- projected -- secret ---- allowHostDirVolumePlugin: false allowHostIPC: false allowHostNetwork: false diff --git a/components/vector-tekton-logs-collector/base/kustomization.yaml b/components/vector-tekton-logs-collector/base/kustomization.yaml new file mode 100644 index 00000000000..1b75ee19635 --- /dev/null +++ b/components/vector-tekton-logs-collector/base/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: tekton-logging +# Skip applying the Tekton/PaC operands while the Tekton/PaC operator is being installed. +# See more information about this option, here: +# https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/#skip-dry-run-for-new-custom-resources-types +commonAnnotations: + argocd.argoproj.io/sync-wave: "-1" + +generators: + - vector-helm-generator.yaml + +resources: + - vector-pre.yaml + diff --git a/components/vector-tekton-logs-collector/base/vector-helm-generator.yaml b/components/vector-tekton-logs-collector/base/vector-helm-generator.yaml new file mode 100644 index 00000000000..3ef35209bdf --- /dev/null +++ b/components/vector-tekton-logs-collector/base/vector-helm-generator.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: HelmChartInflationGenerator +metadata: + name: vector +name: vector +repo: https://helm.vector.dev +version: 0.38.1 +releaseName: vector-tekton-logs-collector +namespace: tekton-logging +valuesFile: vector-helm-values.yaml \ No newline at end of file diff --git a/components/vector-tekton-logs-collector/base/vector-helm-values.yaml b/components/vector-tekton-logs-collector/base/vector-helm-values.yaml new file mode 100644 index 00000000000..f8e22673b2d --- /dev/null +++ b/components/vector-tekton-logs-collector/base/vector-helm-values.yaml @@ -0,0 +1,110 @@ +--- +role: Agent +customConfig: + data_dir: /vector-data-dir + api: + enabled: true + address: 127.0.0.1:8686 + playground: false + sources: + kubernetes_logs: + type: kubernetes_logs + rotate_wait_secs: 5 + glob_minimum_cooldown_ms: 15000 + auto_partial_merge: true + extra_label_selector: "app.kubernetes.io/managed-by in (tekton-pipelines,pipelinesascode.tekton.dev)" + internal_metrics: + type: internal_metrics + transforms: + remap_app_logs: + type: remap + inputs: [kubernetes_logs] + source: |- + .log_type = "application" + .kubernetes_namespace_name = .kubernetes.pod_namespace + if exists(.kubernetes.pod_labels."tekton.dev/taskRunUID") { + .taskRunUID = del(.kubernetes.pod_labels."tekton.dev/taskRunUID") + } else { + .taskRunUID = "none" + } + if exists(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") { + .pipelineRunUID = del(.kubernetes.pod_labels."tekton.dev/pipelineRunUID") + .result = .pipelineRunUID + } else { + .result = .taskRunUID + } + if exists(.kubernetes.pod_labels."tekton.dev/task") { + .task = del(.kubernetes.pod_labels."tekton.dev/task") + } else { + .task = "none" + } + if exists(.kubernetes.pod_namespace) { + .namespace = del(.kubernetes.pod_namespace) + } else { + .namespace = "unlabeled" + } + .pod = .kubernetes.pod_name + .container = .kubernetes.container_name + sinks: + aws_s3: + type: "aws_s3" + bucket: ${BUCKET} + buffer: + type: "disk" + max_size: 1073741824 + inputs: ["remap_app_logs"] + compression: "none" + endpoint: ${ENDPOINT} + encoding: + codec: "text" + key_prefix: "/logs/{{ `{{ .namespace }}` }}/{{`{{ .result }}`}}/{{`{{ .taskRunUID }}`}}/{{`{{ .container }}`}}" + filename_time_format: "" + filename_append_uuid: false +env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: tekton-results-s3 + key: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: tekton-results-s3 + key: aws_secret_access_key + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: tekton-results-s3 + key: aws_region + - name: BUCKET + valueFrom: + secretKeyRef: + name: tekton-results-s3 + key: bucket + - name: ENDPOINT + valueFrom: + secretKeyRef: + name: tekton-results-s3 + key: endpoint +tolerations: + - effect: NoSchedule + key: konflux-ci.dev/workload + operator: Exists +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - NET_BIND_SERVICE + - SETGID + - SETPCAP + - SETUID + readOnlyRootFilesystem: true + seLinuxOptions: + type: spc_t + seccompProfile: + type: RuntimeDefault diff --git a/components/vector-tekton-logs-collector/base/vector-pre.yaml b/components/vector-tekton-logs-collector/base/vector-pre.yaml new file mode 100644 index 00000000000..c8b05bf298a --- /dev/null +++ b/components/vector-tekton-logs-collector/base/vector-pre.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-logging +--- +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: false +allowPrivilegedContainer: false +allowedCapabilities: null +apiVersion: security.openshift.io/v1 +defaultAddCapabilities: null +defaultAllowPrivilegeEscalation: false +forbiddenSysctls: +- '*' +fsGroup: + type: RunAsAny +groups: [] +kind: SecurityContextConstraints +metadata: + name: logging-scc + namespace: tekton-logging +priority: null +readOnlyRootFilesystem: true +requiredDropCapabilities: +- CHOWN +- DAC_OVERRIDE +- FSETID +- FOWNER +- SETGID +- SETUID +- SETPCAP +- NET_BIND_SERVICE +- KILL +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- runtime/default +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:tekton-logging:vector-tekton-logs-collector +volumes: +- configMap +- emptyDir +- hostPath +- projected +- secret diff --git a/components/vector-tekton-logs-collector/development/kustomization.yaml b/components/vector-tekton-logs-collector/development/kustomization.yaml new file mode 100644 index 00000000000..dd624e504b3 --- /dev/null +++ b/components/vector-tekton-logs-collector/development/kustomization.yaml @@ -0,0 +1,4 @@ +kind: Kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +resources: + - ../base diff --git a/components/vector-tekton-logs-collector/production/kustomization.yaml b/components/vector-tekton-logs-collector/production/kustomization.yaml new file mode 100644 index 00000000000..dd624e504b3 --- /dev/null +++ b/components/vector-tekton-logs-collector/production/kustomization.yaml @@ -0,0 +1,4 @@ +kind: Kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +resources: + - ../base diff --git a/components/vector-tekton-logs-collector/staging/kustomization.yaml b/components/vector-tekton-logs-collector/staging/kustomization.yaml new file mode 100644 index 00000000000..dd624e504b3 --- /dev/null +++ b/components/vector-tekton-logs-collector/staging/kustomization.yaml @@ -0,0 +1,4 @@ +kind: Kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +resources: + - ../base diff --git a/hack/generate-deploy-config.sh b/hack/generate-deploy-config.sh index cb7caa357e9..bd24acdd240 100755 --- a/hack/generate-deploy-config.sh +++ b/hack/generate-deploy-config.sh @@ -42,7 +42,7 @@ function main() { for DIR in $(find "$COMPONENT" -name resources); do TARGET=$(dirname "$DIR")/deploy.yaml echo "$DIR: $TARGET" - kustomize build "$DIR" >"$TARGET" + kustomize build --enable-helm "$DIR" >"$TARGET" done }