feat: add ovn_network_logging_parser support (#1481)

depends-on: #1541
vexxhost · Jul 11, 2024 · 7170e79 · 7170e79
1 parent a1f4fbd
commit 7170e79
Show file tree

Hide file tree

Showing 12 changed files with 625 additions and 3 deletions.
diff --git a/charts/ovn/templates/bin/_ovn-network-logging-parser.sh.tpl b/charts/ovn/templates/bin/_ovn-network-logging-parser.sh.tpl
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+  exec uwsgi --ini /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND
diff --git a/charts/ovn/templates/configmap-bin.yaml b/charts/ovn/templates/configmap-bin.yaml
@@ -26,4 +26,6 @@ data:
 {{- end }}
   ovn-controller-init.sh: |
 {{ tuple "bin/_ovn-controller-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ovn-network-logging-parser.sh: |
+{{ tuple "bin/_ovn-network-logging-parser.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
diff --git a/charts/ovn/templates/configmap-etc.yaml b/charts/ovn/templates/configmap-etc.yaml
@@ -17,6 +17,12 @@ limitations under the License.
 {{- $envAll := index . 1 }}
 {{- with $envAll }}
 
+{{- if empty (index .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket") -}}
+{{- $http_socket_port := tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | toString }}
+{{- $http_socket := printf "0.0.0.0:%s" $http_socket_port }}
+{{- $_ := set .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket" $http_socket -}}
+{{- end -}}
+
 ---
 apiVersion: v1
 kind: Secret
@@ -25,7 +31,7 @@ metadata:
 type: Opaque
 data:
   auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }}
-
+  neutron-ovn-network-logging-parser-uwsgi.ini: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.ovn_network_logging_parser_uwsgi | b64enc }}
 {{- end }}
 {{- end }}
 

diff --git a/charts/ovn/templates/daemonset-controller-gw.yaml b/charts/ovn/templates/daemonset-controller-gw.yaml
@@ -112,6 +112,54 @@ spec:
               mountPath: /var/run/ovn
             - name: run-openvswitch
               mountPath: /var/run/openvswitch
+            - name: shared
+              mountPath: /var/log/ovn/
+        {{- if .Values.pod.sidecars.vector }}
+        - name: vector
+{{ tuple $envAll "vector" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.ovn_controller_gw.vector | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "ovn_controller_gw" "container" "vector" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - vector
+            - --config
+            - /etc/vector/vector.toml
+          volumeMounts:
+            - name: vector-config
+              mountPath: /etc/vector
+            - name: shared
+              mountPath: /logs
+            - name: vector-data
+              mountPath: /var/lib/vector
+        {{- end }}
+        {{- if .Values.pod.sidecars.ovn_logging_parser }}
+        - name: log-parser
+{{ tuple $envAll "ovn_logging_parser" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.ovn_controller_gw.ovn_logging_parser | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "ovn_controller_gw" "container" "ovn_logging_parser" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/ovn-network-logging-parser.sh
+            - start
+          env:
+            - name: VECTOR_HTTP_ENDPOINT
+              value: http://localhost:5001
+          ports:
+            - name: http
+              containerPort: {{ tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              protocol: TCP
+          volumeMounts:
+            - name: neutron-etc
+              mountPath: /etc/neutron/neutron.conf
+              subPath: neutron.conf
+              readOnly: true
+            - name: ovn-bin
+              mountPath: /tmp/ovn-network-logging-parser.sh
+              subPath: ovn-network-logging-parser.sh
+              readOnly: true
+            - name: ovn-etc
+              mountPath: /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini
+              subPath: neutron-ovn-network-logging-parser-uwsgi.ini
+              readOnly: true
+        {{- end }}
       volumes:
         - name: ovn-bin
           configMap:
@@ -125,4 +173,19 @@ spec:
           secret:
             secretName: {{ $configMapName }}
             defaultMode: 0444
+        - name: shared
+          emptyDir: {}
+        {{- if .Values.pod.sidecars.vector }}
+        - name: vector-config
+          secret:
+            secretName: ovn-vector-config
+        - name: vector-data
+          emptyDir: {}
+        {{- end }}
+        {{- if .Values.pod.sidecars.ovn_logging_parser }}
+        - name: neutron-etc
+          secret:
+            secretName: neutron-etc
+            defaultMode: 0444
+        {{- end }}
 {{- end }}
diff --git a/charts/ovn/templates/secret-vector.yaml b/charts/ovn/templates/secret-vector.yaml
@@ -0,0 +1,56 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.pod.sidecars.vector }}
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ovn-vector-config
+type: Opaque
+stringData:
+  vector.toml: |
+    [sources.file_logs]
+    type = "file"
+    include = [ "/logs/ovn-controller.log" ]
+
+    [sinks.ovn_log_parser_in]
+    type = "http"
+    inputs = ["file_logs"]
+    uri = "{{ tuple "ovn_logging_parser" "default" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}"
+    encoding.codec = "json"
+    method = "post"
+
+    [sources.ovn_log_parser_out]
+    type = "http_server"
+    address = "0.0.0.0:5001"
+    encoding = "json"
+
+    [transforms.parse_log_message]
+    type = "remap"
+    inputs = ["ovn_log_parser_out"]
+    source = '''
+      del(.source_type)
+      del(.path)
+    '''
+
+    [sinks.loki_sink]
+    type = "loki"
+    labels.event_source = "network_logs"
+    inputs = ["parse_log_message"]
+    endpoint = "http://loki.monitoring:3100"
+    encoding.codec = "json"
+    tenant_id = "{{`{{ project_id }}`}}"
+{{- end }}
diff --git a/charts/ovn/values.yaml b/charts/ovn/values.yaml
@@ -26,6 +26,8 @@ images:
     ovn_controller: docker.io/openstackhelm/ovn:latest-ubuntu_focal
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/library/docker:17.07.0
+    vector: docker.io/timberio/vector:0.39.0-debian
+    ovn_logging_parser: docker.io/openstackhelm/neutron:2024.1-ubuntu_jammy
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -86,6 +88,24 @@ conf:
   use_fqdn:
     compute: true
 
+  ovn_network_logging_parser_uwsgi:
+    uwsgi:
+      add-header: "Connection: close"
+      buffer-size: 65535
+      die-on-term: true
+      enable-threads: true
+      exit-on-reload: false
+      hook-master-start: unix_signal:15 gracefully_kill_them_all
+      lazy-apps: true
+      log-x-forwarded-for: true
+      master: true
+      processes: 1
+      procname-prefix-spaced: "neutron-ovn-network-logging-parser:"
+      route-user-agent: '^kube-probe.* donotlog:'
+      thunder-lock: true
+      worker-reload-mercy: 80
+      wsgi-file: /var/lib/openstack/bin/neutron-ovn-network-logging-parser-wsgi
+
 pod:
   security_context:
     ovn_northd:
@@ -112,6 +132,12 @@ pod:
           capabilities:
             add:
               - SYS_NICE
+        ovn_logging_parser:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        vector:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
   tolerations:
     ovn_ovsdb_nb:
       enabled: false
@@ -240,6 +266,21 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+    ovn_controller_gw:
+      ovn_logging_parser:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "256Mi"
+          cpu: "500m"
+      vector:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "256Mi"
+          cpu: "500m"
     jobs:
       image_repo_sync:
         requests:
@@ -248,6 +289,9 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+  sidecars:
+    ovn_logging_parser: false
+    vector: false
 
 secrets:
   oci_image_registry:
@@ -311,6 +355,22 @@ endpoints:
         default: 6642
       raft:
         default: 6644
+  ovn_logging_parser:
+    name: ovn-logging-parser
+    namespace: null
+    hosts:
+      default: localhost
+    host_fqdn_override:
+      default: localhost
+    scheme:
+      default: 'http'
+      service: 'http'
+    path:
+      default: "/logs"
+    port:
+      api:
+        default: 9697
+        service: 9697
 
 network_policy:
   ovn_ovsdb_nb: