Multus deployment with nmstate operator (red-hat-storage#9777)

* Added NMState deployment

Signed-off-by: oviner <[email protected]>

conf/README.md

@@ -301,6 +301,10 @@ higher priority).
             * `private_gw` - GW for the private interface
             * `root_disk_id` - ID of the root disk
             * `root_disk_sn` - Serial number of the root disk
+            * `node_network_configuration_policy_name` - The NodeNetworkConfigurationPolicy CR name
+            * `node_network_configuration_policy_ip` - The ip address of NodeNetworkConfigurationPolicy CR
+            * `node_network_configuration_policy_prefix_length` - The subnetmask of NodeNetworkConfigurationPolicy CR
+            * `node_network_configuration_policy_destination_route` - The destination route of NodeNetworkConfigurationPolicy CR
 * `hcp_version` - version of HCP client to be deployed on machine running the tests
 * `metallb_version` - MetalLB operator version to install
 * `install_hypershift_upstream` - Install hypershift from upstream or not (Default: false). Necessary for unreleased OCP/CNV versions

ocs_ci/deployment/deployment.py

@@ -1012,6 +1012,51 @@ def deploy_ocs_via_operator(self, image=None):
 
         # Create Multus Networks
         if config.ENV_DATA.get("is_multus_enabled"):
+            from ocs_ci.deployment.nmstate import NMStateInstaller
+
+            logger.info("Install NMState operator and create an instance")
+            nmstate_obj = NMStateInstaller()
+            nmstate_obj.running_nmstate()
+            logger.info("Configure NodeNetworkConfigurationPolicy on all worker nodes")
+            worker_node_names = get_worker_nodes()
+            for worker_node_name in worker_node_names:
+                worker_network_configuration = config.ENV_DATA["baremetal"]["servers"][
+                    worker_node_name
+                ]
+                node_network_configuration_policy = templating.load_yaml(
+                    constants.NODE_NETWORK_CONFIGURATION_POLICY
+                )
+                node_network_configuration_policy["spec"]["nodeSelector"][
+                    "kubernetes.io/hostname"
+                ] = worker_node_name
+                node_network_configuration_policy["metadata"][
+                    "name"
+                ] = worker_network_configuration[
+                    "node_network_configuration_policy_name"
+                ]
+                node_network_configuration_policy["spec"]["desiredState"]["interfaces"][
+                    0
+                ]["ipv4"]["address"][0]["ip"] = worker_network_configuration[
+                    "node_network_configuration_policy_ip"
+                ]
+                node_network_configuration_policy["spec"]["desiredState"]["interfaces"][
+                    0
+                ]["ipv4"]["address"][0]["prefix-length"] = worker_network_configuration[
+                    "node_network_configuration_policy_prefix_length"
+                ]
+                node_network_configuration_policy["spec"]["desiredState"]["routes"][
+                    "config"
+                ][0]["destination"] = worker_network_configuration[
+                    "node_network_configuration_policy_destination_route"
+                ]
+                public_net_yaml = tempfile.NamedTemporaryFile(
+                    mode="w+", prefix="multus_public", delete=False
+                )
+                templating.dump_data_to_temp_yaml(
+                    node_network_configuration_policy, public_net_yaml.name
+                )
+                run_cmd(f"oc create -f {public_net_yaml.name}")
+
             create_public_net = config.ENV_DATA["multus_create_public_net"]
             create_cluster_net = config.ENV_DATA["multus_create_cluster_net"]
             interfaces = set()

ocs_ci/deployment/nmstate.py

@@ -0,0 +1,147 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+from ocs_ci.ocs import constants
+from ocs_ci.utility import templating
+from ocs_ci.ocs import exceptions
+from ocs_ci.ocs.resources.ocs import OCS
+from ocs_ci.utility.utils import TimeoutSampler
+from ocs_ci.ocs.resources.csv import CSV, get_csvs_start_with_prefix
+from ocs_ci.ocs.ocp import OCP
+from ocs_ci.ocs.exceptions import TimeoutExpiredError
+
+
+class NMStateInstaller(object):
+    """
+    NMState Installer class for NMState deployment
+
+    """
+
+    def __init__(self):
+        self.namespace = constants.NMSTATE_NAMESPACE
+
+    def create_nmstate_operator_namespace(self):
+        """
+        Creates the namespace for NMState resources
+
+        Raises:
+            CommandFailed: If the 'oc create' command fails.
+
+        """
+        try:
+            logger.info(f"Creating namespace {self.namespace} for NMState resources")
+            namespace_yaml_file = templating.load_yaml(constants.NMSTATE_NAMESPACE_YAML)
+            namespace_yaml = OCS(**namespace_yaml_file)
+            namespace_yaml.create()
+            logger.info(f"NMState namespace {self.namespace} was created successfully")
+        except exceptions.CommandFailed as ef:
+            if (
+                f'project.project.openshift.io "{self.namespace}" already exists'
+                in str(ef)
+            ):
+                logger.info(f"Namespace {self.namespace} already present")
+                raise ef
+
+    def create_nmstate_operatorgroup(self):
+        """
+        Creates an OperatorGroup for NMState
+
+        """
+        logger.info("Creating OperatorGroup for NMState")
+        operatorgroup_yaml_file = templating.load_yaml(
+            constants.NMSTATE_OPERATORGROUP_YAML
+        )
+        operatorgroup_yaml = OCS(**operatorgroup_yaml_file)
+        operatorgroup_yaml.create()
+        logger.info("NMState OperatorGroup created successfully")
+
+    def create_nmstate_subscription(self):
+        """
+        Creates subscription for NMState operator
+
+        """
+        logger.info("Creating Subscription for NMState")
+        subscription_yaml_file = templating.load_yaml(
+            constants.NMSTATE_SUBSCRIPTION_YAML
+        )
+        subscription_yaml = OCS(**subscription_yaml_file)
+        subscription_yaml.create()
+        logger.info("NMState Subscription created successfully")
+
+    def verify_nmstate_csv_status(self):
+        """
+        Verify the CSV status for the nmstate Operator deployment equals Succeeded
+
+        """
+        for csv in TimeoutSampler(
+            timeout=900,
+            sleep=15,
+            func=get_csvs_start_with_prefix,
+            csv_prefix=constants.NMSTATE_CSV_NAME,
+            namespace=self.namespace,
+        ):
+            if csv:
+                break
+        csv_name = csv[0]["metadata"]["name"]
+        csv_obj = CSV(resource_name=csv_name, namespace=self.namespace)
+        csv_obj.wait_for_phase(phase="Succeeded", timeout=720)
+
+    def create_nmstate_instance(self):
+        """
+        Create an instance of the nmstate Operator
+
+        """
+        logger.info("Creating NMState Instance")
+        subscription_yaml_file = templating.load_yaml(constants.NMSTATE_INSTANCE_YAML)
+        subscription_yaml = OCS(**subscription_yaml_file)
+        subscription_yaml.create()
+        logger.info("NMState Instance created successfully")
+
+    def verify_nmstate_pods_running(self):
+        """
+        Verify the pods for NMState Operator are running
+
+        """
+        sample = TimeoutSampler(
+            timeout=300,
+            sleep=10,
+            func=self.count_nmstate_pods_running,
+            count=10,
+        )
+        if not sample.wait_for_func_status(result=True):
+            raise TimeoutExpiredError(
+                "Not all nmstate pods in Running state after 300 seconds"
+            )
+
+    def count_nmstate_pods_running(self, count):
+        """
+        Count the pods for NMState Operator are running
+
+        Returns:
+            bool:
+
+        """
+        count_running_nmstate_pods = 0
+        ocp_pod = OCP(kind=constants.POD, namespace=self.namespace)
+        pod_items = ocp_pod.get().get("items")
+        # Check if nmstate pods are in running state
+        for nmstate_pod in pod_items:
+            nmstate_pod_name = nmstate_pod.get("metadata").get("name")
+            status = ocp_pod.get_resource_status(nmstate_pod_name)
+            if status == constants.STATUS_RUNNING:
+                logger.info(f"NMState pod {nmstate_pod_name} in running state")
+                count_running_nmstate_pods += 1
+        return count_running_nmstate_pods >= count
+
+    def running_nmstate(self):
+        """
+        Install NMState operator and create an instance
+
+        """
+        self.create_nmstate_operator_namespace()
+        self.create_nmstate_operatorgroup()
+        self.create_nmstate_subscription()
+        self.verify_nmstate_csv_status()
+        self.create_nmstate_instance()
+        self.verify_nmstate_pods_running()

ocs_ci/ocs/constants.py

@@ -32,6 +32,7 @@
 TEMPLATE_MULTICLUSTER_DIR = os.path.join(TEMPLATE_DEPLOYMENT_DIR, "multicluster")
 TEMPLATE_DEPLOYMENT_DIR_CNV = os.path.join(TEMPLATE_DIR, "cnv-deployment")
 TEMPLATE_DEPLOYMENT_DIR_METALLB = os.path.join(TEMPLATE_DIR, "metallb-deployment")
+TEMPLATE_DEPLOYMENT_DIR_NMSTATE = os.path.join(TEMPLATE_DIR, "nmstate-deployment")
 TEMPLATE_CEPH_DIR = os.path.join(TEMPLATE_DIR, "ceph")
 TEMPLATE_CSI_DIR = os.path.join(TEMPLATE_DIR, "CSI")
 TEMPLATE_CSI_LVM_DIR = os.path.join(TEMPLATE_CSI_DIR, "lvm")
@@ -887,11 +888,30 @@
     TEMPLATE_DEPLOYMENT_DIR_OCP, "qe-app-registry-catalog-source.yaml"
 )
 
+# NMState deployment
+NMSTATE_NAMESPACE_YAML = os.path.join(TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "namespace.yaml")
+NMSTATE_OPERATORGROUP_YAML = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "operatorgroup.yaml"
+)
+NMSTATE_SUBSCRIPTION_YAML = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "subscription.yaml"
+)
+NMSTATE_INSTANCE_YAML = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "nmstate_instance.yaml"
+)
+NMSTATE_NAMESPACE = "openshift-nmstate"
+NMSTATE_CSV_NAME = "kubernetes-nmstate-operator"
+
 # Multus Networks
 MULTUS_PUBLIC_NET_YAML = os.path.join(TEMPLATE_DEPLOYMENT_DIR, "multus-public-net.yaml")
 MULTUS_CLUSTER_NET_YAML = os.path.join(
     TEMPLATE_DEPLOYMENT_DIR, "multus-cluster-net.yaml"
 )
+NODE_NETWORK_CONFIGURATION_POLICY = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy.yaml"
+)
+NETWORK_ATTACHEMENT_DEFINITION = "network-attachment-definitions.k8s.cni.cncf.io"
+
 
 OPERATOR_SOURCE_NAME = "ocs-operatorsource"
 

ocs_ci/templates/nmstate-deployment/namespace.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    kubernetes.io/metadata.name: openshift-nmstate
+    name: openshift-nmstate
+  name: openshift-nmstate
+spec:
+  finalizers:
+  - kubernetes

ocs_ci/templates/nmstate-deployment/nmstate_instance.yaml

@@ -0,0 +1,4 @@
+apiVersion: nmstate.io/v1
+kind: NMState
+metadata:
+  name: nmstate

ocs_ci/templates/nmstate-deployment/operatorgroup.yaml

@@ -0,0 +1,10 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  annotations:
+    olm.providedAPIs: NMState.v1.nmstate.io
+  name: openshift-nmstate
+  namespace: openshift-nmstate
+spec:
+  targetNamespaces:
+  - openshift-nmstate

ocs_ci/templates/nmstate-deployment/subscription.yaml

@@ -0,0 +1,13 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  labels:
+    operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: ""
+  name: kubernetes-nmstate-operator
+  namespace: openshift-nmstate
+spec:
+  channel: stable
+  installPlanApproval: Automatic
+  name: kubernetes-nmstate-operator
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace

ocs_ci/templates/ocs-deployment/multus-public-net.yaml

@@ -13,6 +13,7 @@ spec:
       "mode": "bridge",
       "ipam": {
             "type": "whereabouts",
-            "range": "192.168.20.0/24"
+            "range": "192.168.20.0/24",
+            "routes": [{"dst": "192.168.252.0/24"}]
       }
   }'

ocs_ci/templates/ocs-deployment/node_network_configuration_policy.yaml

@@ -0,0 +1,29 @@
+apiVersion: nmstate.io/v1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+ name: ceph-public-net-shim-worker-node
+ namespace: openshift-storage
+spec:
+ nodeSelector:
+   node-role.kubernetes.io/worker: ""
+   kubernetes.io/hostname: worker-node
+ desiredState:
+   interfaces:
+     - name: odf-pub-shim
+       description: Shim interface used to connect host to OpenShift Data Foundation public Multus network
+       type: mac-vlan
+       state: up
+       mac-vlan:
+         base-iface: enp1s0f1
+         mode: bridge
+         promiscuous: true
+       ipv4:
+         enabled: true
+         dhcp: false
+         address:
+           - ip: 192.168.252.1 # STATIC IP FOR worker node
+             prefix-length: 24
+   routes:
+     config:
+       - destination: 192.168.20.0/24
+         next-hop-interface: odf-pub-shim