PVCORE-4214 - Otel tracing support in staging environments (#3917)

* PVCORE-4214 - Otel tracing support in staging environments * StageTake3
redhat-appstudio · Jun 28, 2024 · 0180a0b · 0180a0b
1 parent ecca2b8
commit 0180a0b
Show file tree

Hide file tree

Showing 21 changed files with 146 additions and 122 deletions.
diff --git a/argo-cd-apps/base/all-clusters/infra-deployments/kustomization.yaml b/argo-cd-apps/base/all-clusters/infra-deployments/kustomization.yaml
@@ -12,6 +12,5 @@ resources:
   - enable-dvo-for-all-cluster
   - kubesaw-common
   - tracing-workload-otel-collector
-  - enable-tekton-tracing
 components:
   - ../../../k-components/inject-infra-deployments-repo-details
diff --git a/...ps/base/all-clusters/infra-deployments/tracing-workload-otel-collector/kustomization.yaml b/...ps/base/all-clusters/infra-deployments/tracing-workload-otel-collector/kustomization.yaml
@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - tracing-workload-otel-collector.yaml
+- tracing-workload-tracing.yaml
diff --git a/...rs/infra-deployments/tracing-workload-otel-collector/tracing-workload-otel-collector.yaml b/...rs/infra-deployments/tracing-workload-otel-collector/tracing-workload-otel-collector.yaml
@@ -3,118 +3,42 @@ kind: ApplicationSet
 metadata:
   name: tracing-workload-otel-collector
   labels:
-    ### Prevent the repoURL from being transformed to a local fork name.
     noSourceTransform: "true"
 spec:
   generators:
-    - merge:
-        mergeKeys:
-          - nameNormalized
-        generators:
-          - clusters:
-              values:
-                sourceRoot: components/tracing/otel-collector
-                environment: staging
-                clusterDir: ""
-          - list:
-              elements: []
+  - merge:
+      mergeKeys:
+        - nameNormalized
+      generators:
+        - clusters:
+            values:
+              sourceRoot: components/tracing/otel-collector
+              environment: staging
+              clusterDir: ""
+        - list:
+            elements: []
   template:
     metadata:
       name: tracing-workload-otel-collector-{{nameNormalized}}
     spec:
       project: default
-      source:
-        chart: opentelemetry-collector
-        path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}'
-        repoURL: https://open-telemetry.github.io/opentelemetry-helm-charts
-        targetRevision: 0.90.1
-        helm:
-          parameters:
-            - name: "image.repository"
-              value: otel/opentelemetry-collector-k8s
-            - name: "mode"
-              value: deployment
-          releaseName: open-telemetry
-          values: |
-            config:
-              exporters:
-                debug:
-                  verbosity: basic
-              #             ### To be configured in later work for stage and prod clusters..
-              #                otlphttp:
-              #                  endpoint: https://api.honeycomb.io:443
-              #                  headers:
-              #                    "x-honeycomb-team": ${HONEYCOMB_API_TOKEN}
-              extensions:
-                # The health_check extension is mandatory for this chart.
-                # Without the health_check extension the collector will fail the readiness and liveliness probes.
-                # The health_check extension can be modified, but should never be removed.
-                health_check: {}
-                memory_ballast: {}
-              processors:
-                batch: {}
-                # If set to null, will be overridden with values based on k8s resource limits
-                memory_limiter:
-                  check_interval: 2s
-                  limit_mib: 512
-                  spike_limit_percentage: 100
-                attributes/collector-info:
-                  actions:
-                    - key: collector.hostname
-                      action: insert
-                      value: ${env:HOSTNAME}
-                    - key: collector.clustername
-                      action: insert
-                      value: dev
-              receivers:
-                jaeger: null
-                prometheus: null
-                zipkin: null
-                otlp:
-                  protocols:
-                    grpc:
-                      endpoint: 0.0.0.0:4317
-                      max_recv_msg_size_mib: 999999999
-                    http:
-                      endpoint: 0.0.0.0:4318
-              service:
-                extensions:
-                  - health_check
-                  - memory_ballast
-                pipelines:
-                  traces:
-                    exporters:
-                      - debug
-                    processors:
-                      - memory_limiter
-                      - attributes/collector-info
-                      - batch
-                    receivers:
-                      - otlp
-                  metrics: null
-            # Configuration for ports
-            ports:
-              otlp:
-                enabled: true
-                containerPort: 4317
-                servicePort: 4317
-                hostPort: 4317
-                protocol: TCP
-              otlp-http:
-                enabled: true
-                containerPort: 4318
-                servicePort: 4318
-                hostPort: 4318
-                protocol: TCP
-              jaeger-compact:
-                enabled: false
-              jaeger-thrift:
-                enabled: false
-              jaeger-grpc:
-                enabled: false
-              zipkin:
-                enabled: false
-
+      sources:
+          # Chart from OTel Project
+        - chart: opentelemetry-collector
+          repoURL: https://open-telemetry.github.io/opentelemetry-helm-charts
+          targetRevision: 0.90.1
+          helm:
+            parameters:
+              - name: "image.repository"
+                value: otel/opentelemetry-collector-k8s
+              - name: "mode"
+                value: deployment
+            valueFiles:
+            - $values/{{values.sourceRoot}}/{{values.environment}}/otel-collector-helm-values.yaml
+          # Values for chart from git
+        - repoURL: https://github.com/redhat-appstudio/infra-deployments.git
+          targetRevision: main
+          ref: values
       destination:
         namespace: konflux-otel
         server: '{{server}}'

diff --git a/...tekton-tracing/enable-tekton-tracing.yaml → ...l-collector/tracing-workload-tracing.yaml b/...tekton-tracing/enable-tekton-tracing.yaml → ...l-collector/tracing-workload-tracing.yaml
@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: enable-tekton-tracing
+  name: tracing-workload-tracing
 spec:
   generators:
     - merge:
@@ -10,28 +10,31 @@ spec:
         generators:
           - clusters:
               values:
-                sourceRoot: configs/enable-tekton-tracing
+                sourceRoot: components/tracing/otel-collector
                 environment: staging
                 clusterDir: ""
           - list:
               elements: []
   template:
     metadata:
-      name: enable-tekton-tracing-{{nameNormalized}}
+      name: tracing-workload-tracing-{{nameNormalized}}
     spec:
       project: default
       source:
-        path: configs/enable-tekton-tracing
+        path: '{{values.sourceRoot}}/{{values.environment}}/{{values.clusterDir}}'
         repoURL: https://github.com/redhat-appstudio/infra-deployments.git
         targetRevision: main
       destination:
+        namespace: konflux-otel
         server: '{{server}}'
       syncPolicy:
         automated:
           prune: true
           selfHeal: true
+        syncOptions:
+        - CreateNamespace=true
         retry:
-          limit: -1
+          limit: 50
           backoff:
             duration: 10s
             factor: 2

diff --git a/...omponents/inject-infra-deployments-repo-details/application-set-multisrc-src-1-patch.yaml b/...omponents/inject-infra-deployments-repo-details/application-set-multisrc-src-1-patch.yaml
@@ -0,0 +1,7 @@
+---
+- op: replace
+  path: /spec/template/spec/sources/1/repoURL
+  value: https://github.com/redhat-appstudio/infra-deployments.git
+- op: replace
+  path: /spec/template/spec/sources/1/targetRevision
+  value: main
diff --git a/argo-cd-apps/k-components/inject-infra-deployments-repo-details/kustomization.yaml b/argo-cd-apps/k-components/inject-infra-deployments-repo-details/kustomization.yaml
@@ -13,4 +13,10 @@ patches:
       group: argoproj.io
       version: v1alpha1
       kind: Application
-      labelSelector: noSourceTransform != true
+      labelSelector: noSourceTransform != true
+  - path: application-set-multisrc-src-1-patch.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: ApplicationSet
+      name: tracing-workload-otel-collector
diff --git a/argo-cd-apps/overlays/development/kustomization.yaml b/argo-cd-apps/overlays/development/kustomization.yaml
@@ -168,4 +168,9 @@ patches:
     target:
       kind: ApplicationSet
       version: v1alpha1
-      name: tracing-workload-otel-collector
+      name: tracing-workload-tracing
+  - path: development-overlay-patch.yaml
+    target:
+      kind: ApplicationSet
+      version: v1alpha1
+      name: tracing-workload-otel-collector
diff --git a/argo-cd-apps/overlays/konflux-public-production/delete-applications.yaml b/argo-cd-apps/overlays/konflux-public-production/delete-applications.yaml
@@ -5,9 +5,16 @@ metadata:
   name: gitops
 $patch: delete
 ---
-# otel-collector is dev only.
+# otel-collector is dev & staging only.
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
   name: tracing-workload-otel-collector
 $patch: delete
+---
+# otel-collector is dev & staging only.
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: tracing-workload-tracing
+$patch: delete
diff --git a/argo-cd-apps/overlays/konflux-public-staging/delete-applications.yaml b/argo-cd-apps/overlays/konflux-public-staging/delete-applications.yaml
@@ -5,9 +5,4 @@ metadata:
   name: gitops
 $patch: delete
 ---
-# otel-collector is dev only.
-apiVersion: argoproj.io/v1alpha1
-kind: ApplicationSet
-metadata:
-  name: tracing-workload-otel-collector
-$patch: delete
+
diff --git a/argo-cd-apps/overlays/production-downstream/delete-applications.yaml b/argo-cd-apps/overlays/production-downstream/delete-applications.yaml
@@ -29,3 +29,18 @@ kind: ApplicationSet
 metadata:
   name: quality-dashboard
 $patch: delete
+---
+# otel-collector is dev & staging only.
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: tracing-workload-otel-collector
+$patch: delete
+---
+# otel-collector is dev & staging only.
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: tracing-workload-tracing
+$patch: delete
+---
diff --git a/components/tracing/otel-collector/base/enable-tekton-tracing/enable-tekton-tracing.yaml b/components/tracing/otel-collector/base/enable-tekton-tracing/enable-tekton-tracing.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-tracing
+  namespace: openshift-pipelines
+data:
+  enabled: "true"
+  endpoint: "http://open-telemetry-opentelemetry-collector.konflux-otel.svc.cluster.local:4318/v1/traces"
diff --git a/.../enable-tekton-tracing/kustomization.yaml → .../enable-tekton-tracing/kustomization.yaml b/.../enable-tekton-tracing/kustomization.yaml → .../enable-tekton-tracing/kustomization.yaml
diff --git a/components/tracing/otel-collector/base/external-secrets/honeycomb-api-key.yaml b/components/tracing/otel-collector/base/external-secrets/honeycomb-api-key.yaml
@@ -0,0 +1,19 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: honeycomb-api-key
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  dataFrom:
+    - extract:
+        key: # will be added by the overlays
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: appsre-stonesoup-vault
+  target:
+    creationPolicy: Owner
+    deletionPolicy: Delete
+    name: honeycomb-api-key
diff --git a/components/tracing/otel-collector/base/external-secrets/kustomization.yaml b/components/tracing/otel-collector/base/external-secrets/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - honeycomb-api-key.yaml
+namespace: konflux-otel
diff --git a/.../enable-tekton-tracing/kustomization.yaml → ...ng/otel-collector/base/kustomization.yaml b/.../enable-tekton-tracing/kustomization.yaml → ...ng/otel-collector/base/kustomization.yaml
@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 resources:
-  - enable-tekton-tracing.yaml
+  - external-secrets
+  - enable-tekton-tracing
diff --git a/components/tracing/otel-collector/development/kustomization.yaml b/components/tracing/otel-collector/development/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+patchesStrategicMerge:
+  - patch-delete-external-secrets.yaml
diff --git a/components/tracing/otel-collector/development/patch-delete-external-secrets.yaml b/components/tracing/otel-collector/development/patch-delete-external-secrets.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: honeycomb-api-key
+$patch: delete
+---
diff --git a/components/tracing/otel-collector/staging/honeycomb-api-key-secret-path.yaml b/components/tracing/otel-collector/staging/honeycomb-api-key-secret-path.yaml
@@ -0,0 +1,5 @@
+---
+- op: add
+  path: /spec/dataFrom/0/extract/key
+  value: staging/tracing/honeycomb
+
diff --git a/components/tracing/otel-collector/staging/kustomization.yaml b/components/tracing/otel-collector/staging/kustomization.yaml
@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+patches:
+  - path: honeycomb-api-key-secret-path.yaml
+    target:
+      name: honeycomb-api-key
+      kind: ExternalSecret
+      group: external-secrets.io
+      version: v1beta1
diff --git a/configs/enable-tekton-tracing/enable-tekton-tracing.yaml b/configs/enable-tekton-tracing/enable-tekton-tracing.yaml
@@ -5,5 +5,5 @@ metadata:
   name: config-tracing
   namespace: openshift-pipelines
 data:
-  enabled: "false"
+  enabled: "true"
   endpoint: "http://open-telemetry-opentelemetry-collector.konflux-otel.svc.cluster.local:4318/v1/traces"
diff --git a/hack/preview.sh b/hack/preview.sh
@@ -124,6 +124,7 @@ update_patch_file () {
 }
 update_patch_file "${ROOT}/argo-cd-apps/k-components/inject-infra-deployments-repo-details/application-patch.yaml"
 update_patch_file "${ROOT}/argo-cd-apps/k-components/inject-infra-deployments-repo-details/application-set-patch.yaml"
+update_patch_file "${ROOT}/argo-cd-apps/k-components/inject-infra-deployments-repo-details/application-set-multisrc-src-1-patch.yaml"
 
 if $OBO ; then
   echo "Adding Observability operator and Prometheus for federation"