From a299b5497109fe138a41c76890ce03fcb73d79f8 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Wed, 17 Jan 2024 17:54:46 -0500 Subject: [PATCH 1/2] chore: refactor OVN_DATABASE_NAME --- charts/ovn/templates/bin/_ovsdb-server.sh.tpl | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/charts/ovn/templates/bin/_ovsdb-server.sh.tpl b/charts/ovn/templates/bin/_ovsdb-server.sh.tpl index e023505be..5f67ff77e 100644 --- a/charts/ovn/templates/bin/_ovsdb-server.sh.tpl +++ b/charts/ovn/templates/bin/_ovsdb-server.sh.tpl @@ -32,6 +32,15 @@ if [[ ! $HOSTNAME == *-0 && $OVSDB_HOST =~ (.+)-([0-9]+)\. ]]; then ) fi +if [[ $OVS_DATABASE == "nb" ]]; then + OVN_DATABASE_NAME="OVN_Northbound" +elif [[ $OVS_DATABASE == "sb" ]]; then + OVN_DATABASE_NAME="OVN_Southbound" +else + echo "OVS_DATABASE must be nb or sb" + exit 1 +fi + function start () { /usr/share/ovn/scripts/ovn-ctl start_${OVS_DATABASE}_ovsdb ${ARGS[@]} @@ -40,33 +49,16 @@ function start () { function stop () { /usr/share/ovn/scripts/ovn-ctl stop_${OVS_DATABASE}_ovsdb + pkill tail } function liveness () { - if [[ $OVS_DATABASE == "nb" ]]; then - OVN_DATABASE="Northbound" - elif [[ $OVS_DATABASE == "sb" ]]; then - OVN_DATABASE="Southbound" - else - echo "OVS_DATABASE must be nb or sb" - exit 1 - fi - - ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status OVN_${OVN_DATABASE} + ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status ${OVN_DATABASE_NAME} } function readiness () { - if [[ $OVS_DATABASE == "nb" ]]; then - OVN_DATABASE="Northbound" - elif [[ $OVS_DATABASE == "sb" ]]; then - OVN_DATABASE="Southbound" - else - echo "OVS_DATABASE must be nb or sb" - exit 1 - fi - - ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status OVN_${OVN_DATABASE} + ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status ${OVN_DATABASE_NAME} } $COMMAND From f2c1aeec2bb0997a39c165afd4156914e050df96 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Thu, 18 Jan 2024 20:15:07 -0500 Subject: [PATCH 2/2] chore: refactor ovn to use ovn-kubernetes code --- .../bin/_neutron-metadata-agent-init.sh.tpl | 3 + .../daemonset-ovn-metadata-agent.yaml | 2 + .../ovn/templates/bin/_ovn-controller.sh.tpl | 39 ----- charts/ovn/templates/bin/_ovn-northd.sh.tpl | 57 ------- charts/ovn/templates/bin/_ovsdb-server.sh.tpl | 64 ------- charts/ovn/templates/configmap-bin.yaml | 6 - .../templates/daemonset-controller-gw.yaml | 45 +++-- .../ovn/templates/daemonset-controller.yaml | 45 +++-- charts/ovn/templates/deployment-northd.yaml | 55 +++--- charts/ovn/templates/role-controller.yaml | 11 ++ charts/ovn/templates/role-northd.yaml | 11 ++ charts/ovn/templates/role-ovsdb.yaml | 19 +++ .../ovn/templates/rolebinding-controller.yaml | 13 ++ charts/ovn/templates/rolebinding-northd.yaml | 11 ++ charts/ovn/templates/rolebinding-ovsdb.yaml | 13 ++ charts/ovn/templates/service-ovsdb-nb.yaml | 1 + charts/ovn/templates/service-ovsdb-sb.yaml | 1 + .../ovn/templates/statefulset-ovsdb-nb.yaml | 61 ++++--- .../ovn/templates/statefulset-ovsdb-sb.yaml | 63 ++++--- charts/ovn/values.yaml | 46 ++++- docs/operator.md | 35 ++++ images/ovn/Earthfile | 21 ++- ...-to-using-OVN_KUBERNETES_STATEFULSET.patch | 161 ++++++++++++++++++ ...actor-to-being-able-to-use-split-svc.patch | 144 ++++++++++++++++ roles/defaults/vars/main.yml | 8 +- 25 files changed, 651 insertions(+), 284 deletions(-) delete mode 100644 charts/ovn/templates/bin/_ovn-controller.sh.tpl delete mode 100644 charts/ovn/templates/bin/_ovn-northd.sh.tpl delete mode 100644 charts/ovn/templates/bin/_ovsdb-server.sh.tpl create mode 100644 charts/ovn/templates/role-controller.yaml create mode 100644 charts/ovn/templates/role-northd.yaml create mode 100644 charts/ovn/templates/role-ovsdb.yaml create mode 100644 charts/ovn/templates/rolebinding-controller.yaml create mode 100644 charts/ovn/templates/rolebinding-northd.yaml create mode 100644 charts/ovn/templates/rolebinding-ovsdb.yaml create mode 100644 images/ovn/patches/ovn-kubernetes/0001-chore-refactor-to-using-OVN_KUBERNETES_STATEFULSET.patch create mode 100644 images/ovn/patches/ovn-kubernetes/0002-chore-northd-refactor-to-being-able-to-use-split-svc.patch diff --git a/charts/neutron/templates/bin/_neutron-metadata-agent-init.sh.tpl b/charts/neutron/templates/bin/_neutron-metadata-agent-init.sh.tpl index 5b6ce43e1..1cb25e30e 100644 --- a/charts/neutron/templates/bin/_neutron-metadata-agent-init.sh.tpl +++ b/charts/neutron/templates/bin/_neutron-metadata-agent-init.sh.tpl @@ -17,6 +17,9 @@ limitations under the License. set -ex chown ${NEUTRON_USER_UID} /var/lib/neutron/openstack-helm +{{- if (has "ovn" .Values.network.backend) }} +chown ${NEUTRON_USER_UID} /run/openvswitch/db.sock +{{- end }} {{- if and ( empty .Values.conf.neutron.DEFAULT.host ) ( .Values.pod.use_fqdn.neutron_agent ) }} mkdir -p /tmp/pod-shared diff --git a/charts/neutron/templates/daemonset-ovn-metadata-agent.yaml b/charts/neutron/templates/daemonset-ovn-metadata-agent.yaml index 1fa7f52bb..598c74c08 100644 --- a/charts/neutron/templates/daemonset-ovn-metadata-agent.yaml +++ b/charts/neutron/templates/daemonset-ovn-metadata-agent.yaml @@ -100,6 +100,8 @@ spec: command: - /tmp/neutron-metadata-agent-init.sh volumeMounts: + - name: run + mountPath: /run - name: pod-tmp mountPath: /tmp - name: neutron-bin diff --git a/charts/ovn/templates/bin/_ovn-controller.sh.tpl b/charts/ovn/templates/bin/_ovn-controller.sh.tpl deleted file mode 100644 index ecb659d26..000000000 --- a/charts/ovn/templates/bin/_ovn-controller.sh.tpl +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -xe - -# Copyright 2023 VEXXHOST, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMMAND="${@:-start}" - -function start () { - /usr/share/ovn/scripts/ovn-ctl start_controller \ - --ovn-manage-ovsdb=no - - tail --follow=name /var/log/ovn/ovn-controller.log -} - -function stop () { - /usr/share/ovn/scripts/ovn-ctl stop_controller - pkill tail -} - -function liveness () { - ovs-appctl -t /var/run/ovn/ovn-controller.$(cat /var/run/ovn/ovn-controller.pid).ctl status -} - -function readiness () { - ovs-appctl -t /var/run/ovn/ovn-controller.$(cat /var/run/ovn/ovn-controller.pid).ctl status -} - -$COMMAND diff --git a/charts/ovn/templates/bin/_ovn-northd.sh.tpl b/charts/ovn/templates/bin/_ovn-northd.sh.tpl deleted file mode 100644 index fefd793cc..000000000 --- a/charts/ovn/templates/bin/_ovn-northd.sh.tpl +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -xe - -# Copyright 2023 VEXXHOST, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMMAND="${@:-start}" - -{{- $nb_svc_name := "ovn-ovsdb-nb" -}} -{{- $nb_svc := (tuple $nb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} -{{- $nb_port := (tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} -{{- $nb_service_list := list -}} -{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_nb | int) -}} - {{- $nb_service_list = printf "tcp:%s-%d.%s:%s" $nb_svc_name $i $nb_svc $nb_port | append $nb_service_list -}} -{{- end -}} - -{{- $sb_svc_name := "ovn-ovsdb-sb" -}} -{{- $sb_svc := (tuple $sb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} -{{- $sb_port := (tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} -{{- $sb_service_list := list -}} -{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_sb | int) -}} - {{- $sb_service_list = printf "tcp:%s-%d.%s:%s" $sb_svc_name $i $sb_svc $sb_port | append $sb_service_list -}} -{{- end }} - -function start () { - /usr/share/ovn/scripts/ovn-ctl start_northd \ - --ovn-manage-ovsdb=no \ - --ovn-northd-nb-db={{ include "helm-toolkit.utils.joinListWithComma" $nb_service_list }} \ - --ovn-northd-sb-db={{ include "helm-toolkit.utils.joinListWithComma" $sb_service_list }} - - tail --follow=name /var/log/ovn/ovn-northd.log -} - -function stop () { - /usr/share/ovn/scripts/ovn-ctl stop_northd - pkill tail -} - -function liveness () { - ovs-appctl -t /var/run/ovn/ovn-northd.$(cat /var/run/ovn/ovn-northd.pid).ctl status -} - -function readiness () { - ovs-appctl -t /var/run/ovn/ovn-northd.$(cat /var/run/ovn/ovn-northd.pid).ctl status -} - -$COMMAND diff --git a/charts/ovn/templates/bin/_ovsdb-server.sh.tpl b/charts/ovn/templates/bin/_ovsdb-server.sh.tpl deleted file mode 100644 index 5f67ff77e..000000000 --- a/charts/ovn/templates/bin/_ovsdb-server.sh.tpl +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -xe - -# Copyright 2023 VEXXHOST, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -COMMAND="${@:-start}" - -OVSDB_HOST=$(hostname -f) -ARGS=( - --db-${OVS_DATABASE}-create-insecure-remote=yes - --db-${OVS_DATABASE}-cluster-local-proto=tcp - --db-${OVS_DATABASE}-cluster-local-addr=$(hostname -f) -) - -if [[ ! $HOSTNAME == *-0 && $OVSDB_HOST =~ (.+)-([0-9]+)\. ]]; then - OVSDB_BOOTSTRAP_HOST="${BASH_REMATCH[1]}-0.${OVSDB_HOST#*.}" - - ARGS+=( - --db-${OVS_DATABASE}-cluster-remote-proto=tcp - --db-${OVS_DATABASE}-cluster-remote-addr=${OVSDB_BOOTSTRAP_HOST} - ) -fi - -if [[ $OVS_DATABASE == "nb" ]]; then - OVN_DATABASE_NAME="OVN_Northbound" -elif [[ $OVS_DATABASE == "sb" ]]; then - OVN_DATABASE_NAME="OVN_Southbound" -else - echo "OVS_DATABASE must be nb or sb" - exit 1 -fi - -function start () { - /usr/share/ovn/scripts/ovn-ctl start_${OVS_DATABASE}_ovsdb ${ARGS[@]} - - tail --follow=name /var/log/ovn/ovsdb-server-${OVS_DATABASE}.log -} - -function stop () { - /usr/share/ovn/scripts/ovn-ctl stop_${OVS_DATABASE}_ovsdb - - pkill tail -} - -function liveness () { - ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status ${OVN_DATABASE_NAME} -} - -function readiness () { - ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status ${OVN_DATABASE_NAME} -} - -$COMMAND diff --git a/charts/ovn/templates/configmap-bin.yaml b/charts/ovn/templates/configmap-bin.yaml index a849dd8ae..82001f990 100644 --- a/charts/ovn/templates/configmap-bin.yaml +++ b/charts/ovn/templates/configmap-bin.yaml @@ -24,12 +24,6 @@ data: image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} - ovsdb-server.sh: | -{{ tuple "bin/_ovsdb-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - ovn-northd.sh: | -{{ tuple "bin/_ovn-northd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} ovn-controller-init.sh: | {{ tuple "bin/_ovn-controller-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - ovn-controller.sh: | -{{ tuple "bin/_ovn-controller.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/ovn/templates/daemonset-controller-gw.yaml b/charts/ovn/templates/daemonset-controller-gw.yaml index 5701a879a..eb309c5e5 100644 --- a/charts/ovn/templates/daemonset-controller-gw.yaml +++ b/charts/ovn/templates/daemonset-controller-gw.yaml @@ -12,6 +12,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "controllerGatewayReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovn-controller +{{- end }} + {{- if .Values.manifests.daemonset_ovn_controller_gw }} {{- $envAll := . }} @@ -76,25 +85,33 @@ spec: readOnly: true containers: - name: controller + command: + - /root/ovnkube.sh + - ovn-controller {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "ovn_controller_gw" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - command: - - /tmp/ovn-controller.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/ovn-controller.sh - - stop +{{ dict "envAll" . "component" "ovn_controller_gw" "container" "controller" "type" "readiness" "probeTemplate" (include "controllerGatewayReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_CONTROLLER + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_NB_STATEFULSET + value: ovn-ovsdb-nb + - name: OVN_KUBERNETES_SB_STATEFULSET + value: ovn-ovsdb-sb + - name: OVN_SSL_ENABLE + value: "no" volumeMounts: - - name: ovn-bin - mountPath: /tmp/ovn-controller.sh - subPath: ovn-controller.sh - readOnly: true - name: run-openvswitch - mountPath: /run/openvswitch + mountPath: /var/run/ovn + - name: run-openvswitch + mountPath: /var/run/openvswitch volumes: - name: ovn-bin configMap: diff --git a/charts/ovn/templates/daemonset-controller.yaml b/charts/ovn/templates/daemonset-controller.yaml index 580d5aed4..b6b0b048d 100644 --- a/charts/ovn/templates/daemonset-controller.yaml +++ b/charts/ovn/templates/daemonset-controller.yaml @@ -12,6 +12,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "controllerReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovn-controller +{{- end }} + {{- if .Values.manifests.daemonset_ovn_controller }} {{- $envAll := . }} @@ -76,25 +85,33 @@ spec: readOnly: true containers: - name: controller + command: + - /root/ovnkube.sh + - ovn-controller {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "ovn_controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - command: - - /tmp/ovn-controller.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/ovn-controller.sh - - stop +{{ dict "envAll" . "component" "ovn_controller" "container" "controller" "type" "readiness" "probeTemplate" (include "controllerReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_CONTROLLER + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_NB_STATEFULSET + value: ovn-ovsdb-nb + - name: OVN_KUBERNETES_SB_STATEFULSET + value: ovn-ovsdb-sb + - name: OVN_SSL_ENABLE + value: "no" volumeMounts: - - name: ovn-bin - mountPath: /tmp/ovn-controller.sh - subPath: ovn-controller.sh - readOnly: true - name: run-openvswitch - mountPath: /run/openvswitch + mountPath: /var/run/ovn + - name: run-openvswitch + mountPath: /var/run/openvswitch volumes: - name: ovn-bin configMap: diff --git a/charts/ovn/templates/deployment-northd.yaml b/charts/ovn/templates/deployment-northd.yaml index e3afdd05b..ae31b357a 100644 --- a/charts/ovn/templates/deployment-northd.yaml +++ b/charts/ovn/templates/deployment-northd.yaml @@ -12,18 +12,13 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- define "livenessProbeTemplate" }} +{{- define "northdReadinessProbeTemplate" }} exec: command: - - /tmp/ovn-northd.sh - - liveness -{{- end }} - -{{- define "readinessProbeTemplate" }} -exec: - command: - - /tmp/ovn-northd.sh - - readiness + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovn-northd {{- end }} {{- if .Values.manifests.deployment_northd }} @@ -60,28 +55,26 @@ spec: {{- tuple $envAll "ovn_northd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: northd + command: + - /root/ovnkube.sh + - run-ovn-northd {{ tuple $envAll "ovn_northd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "ovn_northd" "container" "northd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} -{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} -{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} - command: - - /tmp/ovn-northd.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/ovn-northd.sh - - stop - volumeMounts: - - name: ovn-bin - mountPath: /tmp/ovn-northd.sh - subPath: ovn-northd.sh - readOnly: true - volumes: - - name: ovn-bin - configMap: - name: ovn-bin - defaultMode: 0555 +{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "readiness" "probeTemplate" (include "northdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NORTHD + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_NB_STATEFULSET + value: ovn-ovsdb-nb + - name: OVN_KUBERNETES_SB_STATEFULSET + value: ovn-ovsdb-sb + - name: OVN_SSL_ENABLE + value: "no" {{- end }} diff --git a/charts/ovn/templates/role-controller.yaml b/charts/ovn/templates/role-controller.yaml new file mode 100644 index 000000000..de3cfa6d3 --- /dev/null +++ b/charts/ovn/templates/role-controller.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-controller +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list diff --git a/charts/ovn/templates/role-northd.yaml b/charts/ovn/templates/role-northd.yaml new file mode 100644 index 000000000..ca02fae67 --- /dev/null +++ b/charts/ovn/templates/role-northd.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-northd +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list diff --git a/charts/ovn/templates/role-ovsdb.yaml b/charts/ovn/templates/role-ovsdb.yaml new file mode 100644 index 000000000..10e0e2390 --- /dev/null +++ b/charts/ovn/templates/role-ovsdb.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-ovsdb +rules: +- apiGroups: + - "apps" + resources: + - statefulsets + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - endpoints + verbs: + - list + - get diff --git a/charts/ovn/templates/rolebinding-controller.yaml b/charts/ovn/templates/rolebinding-controller.yaml new file mode 100644 index 000000000..7973c7e26 --- /dev/null +++ b/charts/ovn/templates/rolebinding-controller.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-controller +subjects: +- kind: ServiceAccount + name: ovn-controller +- kind: ServiceAccount + name: ovn-controller-gw diff --git a/charts/ovn/templates/rolebinding-northd.yaml b/charts/ovn/templates/rolebinding-northd.yaml new file mode 100644 index 000000000..428a47075 --- /dev/null +++ b/charts/ovn/templates/rolebinding-northd.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-northd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-northd +subjects: +- kind: ServiceAccount + name: ovn-northd diff --git a/charts/ovn/templates/rolebinding-ovsdb.yaml b/charts/ovn/templates/rolebinding-ovsdb.yaml new file mode 100644 index 000000000..f32382bc3 --- /dev/null +++ b/charts/ovn/templates/rolebinding-ovsdb.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-ovsdb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-ovsdb +subjects: +- kind: ServiceAccount + name: ovn-ovsdb-nb +- kind: ServiceAccount + name: ovn-ovsdb-sb diff --git a/charts/ovn/templates/service-ovsdb-nb.yaml b/charts/ovn/templates/service-ovsdb-nb.yaml index b93da9b8b..56f7cd096 100644 --- a/charts/ovn/templates/service-ovsdb-nb.yaml +++ b/charts/ovn/templates/service-ovsdb-nb.yaml @@ -20,6 +20,7 @@ kind: Service metadata: name: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: + publishNotReadyAddresses: true ports: - name: ovsdb port: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} diff --git a/charts/ovn/templates/service-ovsdb-sb.yaml b/charts/ovn/templates/service-ovsdb-sb.yaml index 70f62c6e4..4a6b5864d 100644 --- a/charts/ovn/templates/service-ovsdb-sb.yaml +++ b/charts/ovn/templates/service-ovsdb-sb.yaml @@ -20,6 +20,7 @@ kind: Service metadata: name: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: + publishNotReadyAddresses: true ports: - name: ovsdb port: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} diff --git a/charts/ovn/templates/statefulset-ovsdb-nb.yaml b/charts/ovn/templates/statefulset-ovsdb-nb.yaml index 93805156f..8b784f64e 100644 --- a/charts/ovn/templates/statefulset-ovsdb-nb.yaml +++ b/charts/ovn/templates/statefulset-ovsdb-nb.yaml @@ -12,6 +12,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "ovnnbReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovnnb-db-raft +{{- end }} + {{- if .Values.manifests.statefulset_ovn_ovsdb_nb }} {{- $envAll := . }} @@ -28,6 +37,7 @@ metadata: {{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: serviceName: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: Parallel replicas: {{ .Values.pod.replicas.ovn_ovsdb_nb }} selector: matchLabels: @@ -49,41 +59,50 @@ spec: {{- tuple $envAll "ovn_ovsdb_nb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: ovsdb + command: + - /root/ovnkube.sh + - nb-ovsdb-raft {{ tuple $envAll "ovn_ovsdb_nb" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" . "component" "ovn_ovsdb_nb" "container" "ovsdb" "type" "readiness" "probeTemplate" (include "ovnnbReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} ports: - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: - - name: OVS_DATABASE - value: nb - - name: OVS_PORT + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NB + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_STATEFULSET + value: ovn-ovsdb-nb + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OVN_SSL_ENABLE + value: "no" + - name: ENABLE_IPSEC + value: "false" + - name: OVN_NB_RAFT_ELECTION_TIMER + value: "1000" + - name: OVN_NB_PORT value: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - command: - - /tmp/ovsdb-server.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/ovsdb-server.sh - - stop + - name: OVN_NB_RAFT_PORT + value: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} volumeMounts: - - name: ovn-bin - mountPath: /tmp/ovsdb-server.sh - subPath: ovsdb-server.sh - readOnly: true - name: run-openvswitch - mountPath: /run/openvswitch + mountPath: /var/run/openvswitch + - name: run-openvswitch + mountPath: /var/run/ovn - name: data mountPath: /var/lib/ovn volumes: - name: run-openvswitch emptyDir: {} - - name: ovn-bin - configMap: - name: ovn-bin - defaultMode: 0555 {{- if not .Values.volume.ovn_ovsdb_nb.enabled }} - name: data emptyDir: {} diff --git a/charts/ovn/templates/statefulset-ovsdb-sb.yaml b/charts/ovn/templates/statefulset-ovsdb-sb.yaml index 400b127cc..b5c5398dd 100644 --- a/charts/ovn/templates/statefulset-ovsdb-sb.yaml +++ b/charts/ovn/templates/statefulset-ovsdb-sb.yaml @@ -12,6 +12,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "ovnsbReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovnsb-db-raft +{{- end }} + {{- if .Values.manifests.statefulset_ovn_ovsdb_sb }} {{- $envAll := . }} @@ -28,6 +37,7 @@ metadata: {{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: serviceName: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: Parallel replicas: {{ .Values.pod.replicas.ovn_ovsdb_sb }} selector: matchLabels: @@ -49,41 +59,50 @@ spec: {{- tuple $envAll "ovn_ovsdb_sb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: ovsdb + command: + - /root/ovnkube.sh + - sb-ovsdb-raft {{ tuple $envAll "ovn_ovsdb_sb" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" . "component" "ovn_ovsdb_sb" "container" "ovsdb" "type" "readiness" "probeTemplate" (include "ovnsbReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} ports: - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: - - name: OVS_DATABASE - value: sb - - name: OVS_PORT + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_SB + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_STATEFULSET + value: ovn-ovsdb-sb + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OVN_SSL_ENABLE + value: "no" + - name: ENABLE_IPSEC + value: "false" + - name: OVN_SB_RAFT_ELECTION_TIMER + value: "1000" + - name: OVN_SB_PORT value: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - command: - - /tmp/ovsdb-server.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/ovsdb-server.sh - - stop + - name: OVN_SB_RAFT_PORT + value: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} volumeMounts: - - name: ovn-bin - mountPath: /tmp/ovsdb-server.sh - subPath: ovsdb-server.sh - readOnly: true - name: run-openvswitch - mountPath: /run/openvswitch + mountPath: /var/run/openvswitch + - name: run-openvswitch + mountPath: /var/run/ovn - name: data mountPath: /var/lib/ovn volumes: - name: run-openvswitch emptyDir: {} - - name: ovn-bin - configMap: - name: ovn-bin - defaultMode: 0555 {{- if not .Values.volume.ovn_ovsdb_sb.enabled }} - name: data emptyDir: {} @@ -93,10 +112,10 @@ spec: name: data spec: accessModes: ["ReadWriteOnce"] + storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_sb.class_name }} resources: requests: storage: {{ $envAll.Values.volume.ovn_ovsdb_sb.size }} - storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_sb.class_name }} {{- end }} {{- end }} diff --git a/charts/ovn/values.yaml b/charts/ovn/values.yaml index cc0eea444..214dd16f7 100644 --- a/charts/ovn/values.yaml +++ b/charts/ovn/values.yaml @@ -138,13 +138,41 @@ pod: readiness: enabled: true params: - initialDelaySeconds: 5 - timeoutSeconds: 10 - liveness: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_ovsdb_nb: + ovsdb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_ovsdb_sb: + ovsdb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_controller: + controller: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_controller_gw: + controller: + readiness: enabled: true params: - initialDelaySeconds: 5 - timeoutSeconds: 10 + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 dns_policy: "ClusterFirstWithHostNet" replicas: ovn_ovsdb_nb: 1 @@ -179,18 +207,18 @@ pod: ovs: ovn_ovsdb_nb: requests: - memory: "128Mi" + memory: "384Mi" cpu: "100m" limits: memory: "1024Mi" - cpu: "2000m" + cpu: "1000m" ovn_ovsdb_sb: requests: - memory: "128Mi" + memory: "384Mi" cpu: "100m" limits: memory: "1024Mi" - cpu: "2000m" + cpu: "1000m" ovn_northd: requests: memory: "128Mi" diff --git a/docs/operator.md b/docs/operator.md index a9a8c90dd..5f8d7c80e 100644 --- a/docs/operator.md +++ b/docs/operator.md @@ -1,5 +1,40 @@ # Operator Documentation +## OVN + +## Recovering cluster + +If any of the OVN database pods fail, they will no longer be ready. You can +recover the cluster by deleting the pods and allowing them to be recreated. + +For example, if the `ovn-ovsdb-nb-0` pod fails, you can recover the cluster by +deleting the pod: + +```bash +kubectl -n openstack delete pods/ovn-ovsdb-nb-0 +``` + +If the entire cluster fails, you can recover the cluster by deleting all of the +pods. For example, if the southbound database fails, you can recover the +cluster with this command: + +```bash +kubectl -n openstack delete pods -lcomponent=ovn-ovsdb-sb +``` + +If the state of Neutron is lost from the cluster, you can recover it by running +the repair command: + +```bash +kubectl -n openstack exec deploy/neutron-server -- \ + neutron-ovn-db-sync-util \ + --debug \ + --config-file /etc/neutron/neutron.conf \ + --config-file /tmp/pod-shared/ovn.ini \ + --config-file /etc/neutron/plugins/ml2/ml2_conf.ini \ + --ovn-neutron_sync_mode repair +``` + ## Octavia ### Accessing Amphorae diff --git a/images/ovn/Earthfile b/images/ovn/Earthfile index 436535ae8..ce6689c77 100644 --- a/images/ovn/Earthfile +++ b/images/ovn/Earthfile @@ -3,14 +3,29 @@ VERSION 0.7 ARG --global SERIES=23.03 ARG --global VERSION=23.03.0-69 +ovn-kubernetes: + FROM golang:1.20 + GIT CLONE --branch master https://github.com/ovn-org/ovn-kubernetes /src + WORKDIR /src + COPY patches/ovn-kubernetes /patches + DO ../+APT_INSTALL --PACKAGES "git" + RUN git apply --verbose /patches/*.patch + SAVE ARTIFACT /src/dist/images/ovndb-raft-functions.sh + SAVE ARTIFACT /src/dist/images/ovnkube.sh + RUN \ + cd /src/go-controller && \ + go build -o /build/ovn-kube-util ./cmd/ovn-kube-util + SAVE ARTIFACT /build/ovn-kube-util + component-image: FROM ../openvswitch+platform-image DO ../+DNF_INSTALL --PACKAGES "firewalld-filesystem hostname ovn${SERIES}-${VERSION}.el9s procps-ng" ARG --required NAME DO ../+DNF_INSTALL --PACKAGES "ovn${SERIES}-${NAME}-${VERSION}.el9s" - IF [ "${NAME}" = "host" ] - COPY ../kubernetes+image/kubectl /usr/local/bin/kubectl - END + COPY ../kubernetes+image/kubectl /usr/local/bin/kubectl + COPY +ovn-kubernetes/ovndb-raft-functions.sh /root + COPY +ovn-kubernetes/ovnkube.sh /root + COPY +ovn-kubernetes/ovn-kube-util /usr/bin SAVE IMAGE --push \ ghcr.io/vexxhost/atmosphere/ovn-${NAME}:${SERIES} \ ghcr.io/vexxhost/atmosphere/ovn-${NAME}:${VERSION} diff --git a/images/ovn/patches/ovn-kubernetes/0001-chore-refactor-to-using-OVN_KUBERNETES_STATEFULSET.patch b/images/ovn/patches/ovn-kubernetes/0001-chore-refactor-to-using-OVN_KUBERNETES_STATEFULSET.patch new file mode 100644 index 000000000..807119207 --- /dev/null +++ b/images/ovn/patches/ovn-kubernetes/0001-chore-refactor-to-using-OVN_KUBERNETES_STATEFULSET.patch @@ -0,0 +1,161 @@ +From 0227559bb404c3d7d5f32737809c38024bbabef0 Mon Sep 17 00:00:00 2001 +From: Mohammed Naser +Date: Thu, 18 Jan 2024 12:42:24 -0500 +Subject: [PATCH 1/2] chore: refactor to using OVN_KUBERNETES_STATEFULSET + +--- + dist/images/ovndb-raft-functions.sh | 32 ++++++++++++++--------------- + dist/images/ovnkube.sh | 5 ++++- + 2 files changed, 20 insertions(+), 17 deletions(-) + +diff --git a/dist/images/ovndb-raft-functions.sh b/dist/images/ovndb-raft-functions.sh +index dceb4ec2e..65e9b6d9b 100644 +--- a/dist/images/ovndb-raft-functions.sh ++++ b/dist/images/ovndb-raft-functions.sh +@@ -10,7 +10,7 @@ verify-ovsdb-raft() { + fi + + replicas=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get statefulset -n ${ovn_kubernetes_namespace} ovnkube-db -o=jsonpath='{.spec.replicas}') ++ get statefulset -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset} -o=jsonpath='{.spec.replicas}') + if [[ ${replicas} -lt 3 || $((${replicas} % 2)) -eq 0 ]]; then + echo "at least 3 nodes need to be configured, and it must be odd number of nodes" + exit 1 +@@ -45,14 +45,14 @@ db_part_of_cluster() { + } + + # Checks if cluster has already been initialized. +-# If not it returns false and sets init_ip to ovnkube-db-0 ++# If not it returns false and sets init_ip to ${ovn_kubernetes_statefulset}-0 + cluster_exists() { + # See if ep is available ... + local db=${1} + local port=${2} + + db_pods=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get pod -n ${ovn_kubernetes_namespace} -o=jsonpath='{.items[*].metadata.name}' | egrep -o 'ovnkube-db[^ ]+') ++ get pod -n ${ovn_kubernetes_namespace} -o=jsonpath='{.items[*].metadata.name}' | egrep -o "${ovn_kubernetes_statefulset}[^ ]+") + + for db_pod in $db_pods; do + if db_part_of_cluster $db_pod $db $port; then +@@ -63,7 +63,7 @@ cluster_exists() { + + # if we get here there is no cluster, set init_ip and get out + init_ip="$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get pod -n ${ovn_kubernetes_namespace} ovnkube-db-0 -o=jsonpath='{.status.podIP}')" ++ get pod -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset}-0 -o=jsonpath='{.status.podIP}')" + if [[ $? != 0 ]]; then + return 1 + fi +@@ -90,17 +90,17 @@ check_and_apply_ovnkube_db_ep() { + + # return if ovn db service endpoint already exists + result=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get ep -n ${ovn_kubernetes_namespace} ovnkube-db 2>&1) ++ get ep -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset} 2>&1) + test $? -eq 0 && return + if ! echo ${result} | grep -q "NotFound"; then +- echo "Failed to find ovnkube-db endpoint: ${result}, Exiting..." ++ echo "Failed to find ${ovn_kubernetes_statefulset} endpoint: ${result}, Exiting..." + exit 12 + fi +- # Get IPs of all ovnkube-db PODs ++ # Get IPs of all ${ovn_kubernetes_statefulset} PODs + ips=() + for ((i = 0; i < ${replicas}; i++)); do + ip=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get pod -n ${ovn_kubernetes_namespace} ovnkube-db-${i} -o=jsonpath='{.status.podIP}') ++ get pod -n ${ovn_kubernetes_namespace} ${ovn_kubernetes_statefulset}-${i} -o=jsonpath='{.status.podIP}') + if [[ ${ip} == "" ]]; then + break + fi +@@ -108,7 +108,7 @@ check_and_apply_ovnkube_db_ep() { + done + + if [[ ${i} -eq ${replicas} ]]; then +- # Number of POD IPs is same as number of statefulset replicas. Now, if the number of ovnkube-db endpoints ++ # Number of POD IPs is same as number of statefulset replicas. Now, if the number of ${ovn_kubernetes_statefulset} endpoints + # is 0, then we are applying the endpoint for the first time. So, we need to make sure that each of the + # pod IP responds to the `ovsdb-client list-dbs` call before we set the endpoint. If they don't, retry several + # times and then give up. +@@ -170,7 +170,7 @@ set_election_timer() { + return 0 + } + +-# set_connection() will be called for ovnkube-db-0 pod when : ++# set_connection() will be called for ${ovn_kubernetes_statefulset}-0 pod when : + # 1. it is first started or + # 2. it restarts after the initial start has failed or + # 3. subsequent restarts during the lifetime of the pod +@@ -307,7 +307,7 @@ ovsdb-raft() { + --ovn-${db}-log="${ovn_loglevel_db}" & + else + # either we need to initialize a new cluster or wait for db-0 to create it +- if [[ "${POD_NAME}" == "ovnkube-db-0" ]]; then ++ if [[ "${POD_NAME}" == "${ovn_kubernetes_statefulset}-0" ]]; then + echo "Cluster does not exist for DB: ${db}, creating new raft cluster" + run_as_ovs_user_if_needed \ + ${OVNCTL_PATH} run_${db}_ovsdb --no-monitor \ +@@ -317,7 +317,7 @@ ovsdb-raft() { + ${db_ssl_opts} \ + --ovn-${db}-log="${ovn_loglevel_db}" & + else +- echo "Cluster does not exist for DB: ${db}, waiting for ovnkube-db-0 pod to create it" ++ echo "Cluster does not exist for DB: ${db}, waiting for ${ovn_kubernetes_statefulset}-0 pod to create it" + # all non pod-0 pods will be blocked here till connection is set + wait_for_event cluster_exists ${db} ${port} + run_as_ovs_user_if_needed \ +@@ -356,8 +356,8 @@ ovsdb-raft() { + fi + echo "=============== ${db}-ovsdb-raft ========== RUNNING" + +- if [[ "${POD_NAME}" == "ovnkube-db-0" ]]; then +- # post raft create work has to be done only once and in ovnkube-db-0 while it is still ++ if [[ "${POD_NAME}" == "${ovn_kubernetes_statefulset}-0" ]]; then ++ # post raft create work has to be done only once and in ${ovn_kubernetes_statefulset}-0 while it is still + # a single-node cluster, additional protection against the case when pod-0 isn't a leader + # is needed in the cases of sudden pod-0 initialization logic restarts + current_raft_role=$(ovs-appctl -t ${OVN_RUNDIR}/ovn${db}_db.ctl cluster/status ${database} 2>&1 | grep "^Role") +@@ -381,9 +381,9 @@ ovsdb-raft() { + fi + + last_node_index=$(expr ${replicas} - 1) +- # Create endpoints only if all ovnkube-db pods have started and are running. We do this ++ # Create endpoints only if all ${ovn_kubernetes_statefulset} pods have started and are running. We do this + # from the last pod of the statefulset. +- if [[ ${db} == "nb" && "${POD_NAME}" == "ovnkube-db-"${last_node_index} ]]; then ++ if [[ ${db} == "nb" && "${POD_NAME}" == "${ovn_kubernetes_statefulset}-"${last_node_index} ]]; then + check_and_apply_ovnkube_db_ep ${port} + fi + +diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh +index bf3989e37..720b3e14d 100755 +--- a/dist/images/ovnkube.sh ++++ b/dist/images/ovnkube.sh +@@ -40,6 +40,7 @@ fi + # OVN_NET_CIDR - the network cidr - v3 + # OVN_SVC_CIDR - the cluster-service-cidr - v3 + # OVN_KUBERNETES_NAMESPACE - k8s namespace - v3 ++# OVN_KUBERNETES_STATEFULSET - k8s statefulset - v3 + # K8S_NODE - hostname of the node - v3 + # + # OVN_DAEMONSET_VERSION - version match daemonset and image - v3 +@@ -198,12 +199,14 @@ metrics_bind_port=${OVN_METRICS_BIND_PORT:-9476} + metrics_exporter_port=${OVN_METRICS_EXPORTER_PORT:-9310} + + ovn_kubernetes_namespace=${OVN_KUBERNETES_NAMESPACE:-ovn-kubernetes} ++ovn_kubernetes_statefulset=${OVN_KUBERNETES_STATEFULSET:-ovnkube-db} ++ + # namespace used for classifying host network traffic + ovn_host_network_namespace=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network} + + # host on which ovnkube-db POD is running and this POD contains both + # OVN NB and SB DB running in their own container. +-ovn_db_host=${K8S_NODE_IP:-""} ++ovn_db_host=${K8S_NODE_IP:-$(hostname -f)} + + # OVN_NB_PORT - ovn north db port (default 6641) + ovn_nb_port=${OVN_NB_PORT:-6641} +-- +2.42.0 + diff --git a/images/ovn/patches/ovn-kubernetes/0002-chore-northd-refactor-to-being-able-to-use-split-svc.patch b/images/ovn/patches/ovn-kubernetes/0002-chore-northd-refactor-to-being-able-to-use-split-svc.patch new file mode 100644 index 000000000..c615f63fa --- /dev/null +++ b/images/ovn/patches/ovn-kubernetes/0002-chore-northd-refactor-to-being-able-to-use-split-svc.patch @@ -0,0 +1,144 @@ +From 90851bd77718bc834446ebe2ddf34b8e9383dee8 Mon Sep 17 00:00:00 2001 +From: Mohammed Naser +Date: Thu, 18 Jan 2024 16:16:11 -0500 +Subject: [PATCH 2/2] chore(northd): refactor to being able to use split svcs + +--- + dist/images/ovnkube.sh | 75 +++++++++++++++++++++++++++++++++--------- + 1 file changed, 59 insertions(+), 16 deletions(-) + +diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh +index 720b3e14d..1d3059cf8 100755 +--- a/dist/images/ovnkube.sh ++++ b/dist/images/ovnkube.sh +@@ -200,6 +200,8 @@ metrics_exporter_port=${OVN_METRICS_EXPORTER_PORT:-9310} + + ovn_kubernetes_namespace=${OVN_KUBERNETES_NAMESPACE:-ovn-kubernetes} + ovn_kubernetes_statefulset=${OVN_KUBERNETES_STATEFULSET:-ovnkube-db} ++ovn_kubernetes_nb_statefulset=${OVN_KUBERNETES_NB_STATEFULSET:-ovnkube-db} ++ovn_kubernetes_sb_statefulset=${OVN_KUBERNETES_SB_STATEFULSET:-ovnkube-db} + + # namespace used for classifying host network traffic + ovn_host_network_namespace=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network} +@@ -374,6 +376,24 @@ wait_for_event() { + done + } + ++wait_for_db () { ++ local db=$1 ++ local ep=$(get_ovnkube_zone_db_ep ${db}) ++ ++ echo "Getting the ${ep} ep" ++ # See if ep is available ... ++ IFS=" " read -a ep_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ ++ get endpointslice -n ${ovn_kubernetes_namespace} -l kubernetes.io/service-name=${ep} -o=jsonpath='{range .items[0].endpoints[*]}{.addresses[0]} ')" ++ if [[ ${#ep_hosts[@]} == 0 ]]; then ++ return 1 ++ fi ++ ++ ep_hosts_string="${ep_hosts[*]}" ++ declare -g -a "ovn_${db}db_hosts=($ep_hosts_string)" ++ ++ return 0 ++} ++ + # The ovnkube-db kubernetes service must be populated with OVN DB service endpoints + # before various OVN K8s containers can come up. This functions checks for that. + # If OVN dbs are configured to listen only on unix sockets, then there will not be +@@ -384,15 +404,18 @@ ready_to_start_node() { + return 0 + fi + +- ovnkube_db_ep=$(get_ovnkube_zone_db_ep) +- echo "Getting the ${ovnkube_db_ep} ep" +- # See if ep is available ... +- IFS=" " read -a ovn_db_hosts <<<"$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} \ +- get ep -n ${ovn_kubernetes_namespace} ${ovnkube_db_ep} -o=jsonpath='{range .subsets[0].addresses[*]}{.ip}{" "}')" +- if [[ ${#ovn_db_hosts[@]} == 0 ]]; then ++ wait_for_db nb ++ if [[ $? != 0 ]]; then ++ return 1 ++ fi ++ ++ wait_for_db sb ++ if [[ $? != 0 ]]; then + return 1 + fi ++ + get_ovn_db_vars ++ + return 0 + } + # wait_for_event ready_to_start_node +@@ -410,17 +433,29 @@ check_ovn_daemonset_version() { + } + + get_ovn_db_vars() { ++ ++ index=0 + ovn_nbdb_str="" +- ovn_sbdb_str="" +- for i in "${ovn_db_hosts[@]}"; do ++ for i in "${ovn_nbdb_hosts[@]}"; do + if [ -n "$ovn_nbdb_str" ]; then + ovn_nbdb_str=${ovn_nbdb_str}"," ++ fi ++ host="${ovn_kubernetes_nb_statefulset}-${index}.${ovn_kubernetes_nb_statefulset}.${ovn_kubernetes_namespace}.svc.cluster.local" ++ ovn_nbdb_str=${ovn_nbdb_str}${transport}://${host}:${ovn_nb_port} ++ index=$((index + 1)) ++ done ++ ++ index=0 ++ ovn_sbdb_str="" ++ for i in "${ovn_sbdb_hosts[@]}"; do ++ if [ -n "$ovn_sbdb_str" ]; then + ovn_sbdb_str=${ovn_sbdb_str}"," + fi +- ip=$(bracketify $i) +- ovn_nbdb_str=${ovn_nbdb_str}${transport}://${ip}:${ovn_nb_port} +- ovn_sbdb_str=${ovn_sbdb_str}${transport}://${ip}:${ovn_sb_port} ++ host="${ovn_kubernetes_sb_statefulset}-${index}.${ovn_kubernetes_sb_statefulset}.${ovn_kubernetes_namespace}.svc.cluster.local" ++ ovn_sbdb_str=${ovn_sbdb_str}${transport}://${host}:${ovn_sb_port} ++ index=$((index + 1)) + done ++ + # OVN_NORTH and OVN_SOUTH override derived host + ovn_nbdb=${OVN_NORTH:-$ovn_nbdb_str} + ovn_sbdb=${OVN_SOUTH:-$ovn_sbdb_str} +@@ -730,7 +765,7 @@ set_ovnkube_db_ep() { + ips=("$@") + + ovn_zone=$(get_node_zone) +- ovnkube_db_ep=$(get_ovnkube_zone_db_ep) ++ ovnkube_db_ep=$(get_ovnkube_zone_db_ep sb) + echo "=============== setting ${ovnkube_db_ep} endpoints to ${ips[@]}" + # create a new endpoint for the headless onvkube-db service without selectors + kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${K8S_CACERT} apply -f - <