diff --git a/conf/README.md b/conf/README.md index f5a5ee07a50..1cf5bac2b63 100644 --- a/conf/README.md +++ b/conf/README.md @@ -301,6 +301,10 @@ higher priority). * `private_gw` - GW for the private interface * `root_disk_id` - ID of the root disk * `root_disk_sn` - Serial number of the root disk + * `node_network_configuration_policy_name` - The NodeNetworkConfigurationPolicy CR name + * `node_network_configuration_policy_ip` - The ip address of NodeNetworkConfigurationPolicy CR + * `node_network_configuration_policy_prefix_length` - The subnetmask of NodeNetworkConfigurationPolicy CR + * `node_network_configuration_policy_destination_route` - The destination route of NodeNetworkConfigurationPolicy CR * `hcp_version` - version of HCP client to be deployed on machine running the tests * `metallb_version` - MetalLB operator version to install * `install_hypershift_upstream` - Install hypershift from upstream or not (Default: false). Necessary for unreleased OCP/CNV versions diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 15e4df13a9a..7d61bfcf6c4 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -1012,6 +1012,51 @@ def deploy_ocs_via_operator(self, image=None): # Create Multus Networks if config.ENV_DATA.get("is_multus_enabled"): + from ocs_ci.deployment.nmstate import NMStateInstaller + + logger.info("Install NMState operator and create an instance") + nmstate_obj = NMStateInstaller() + nmstate_obj.running_nmstate() + logger.info("Configure NodeNetworkConfigurationPolicy on all worker nodes") + worker_node_names = get_worker_nodes() + for worker_node_name in worker_node_names: + worker_network_configuration = config.ENV_DATA["baremetal"]["servers"][ + worker_node_name + ] + node_network_configuration_policy = templating.load_yaml( + constants.NODE_NETWORK_CONFIGURATION_POLICY + ) + node_network_configuration_policy["spec"]["nodeSelector"][ + "kubernetes.io/hostname" + ] = worker_node_name + node_network_configuration_policy["metadata"][ + "name" + ] = worker_network_configuration[ + "node_network_configuration_policy_name" + ] + node_network_configuration_policy["spec"]["desiredState"]["interfaces"][ + 0 + ]["ipv4"]["address"][0]["ip"] = worker_network_configuration[ + "node_network_configuration_policy_ip" + ] + node_network_configuration_policy["spec"]["desiredState"]["interfaces"][ + 0 + ]["ipv4"]["address"][0]["prefix-length"] = worker_network_configuration[ + "node_network_configuration_policy_prefix_length" + ] + node_network_configuration_policy["spec"]["desiredState"]["routes"][ + "config" + ][0]["destination"] = worker_network_configuration[ + "node_network_configuration_policy_destination_route" + ] + public_net_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="multus_public", delete=False + ) + templating.dump_data_to_temp_yaml( + node_network_configuration_policy, public_net_yaml.name + ) + run_cmd(f"oc create -f {public_net_yaml.name}") + create_public_net = config.ENV_DATA["multus_create_public_net"] create_cluster_net = config.ENV_DATA["multus_create_cluster_net"] interfaces = set() diff --git a/ocs_ci/deployment/nmstate.py b/ocs_ci/deployment/nmstate.py new file mode 100644 index 00000000000..9e7514c764e --- /dev/null +++ b/ocs_ci/deployment/nmstate.py @@ -0,0 +1,147 @@ +import logging + +logger = logging.getLogger(__name__) + +from ocs_ci.ocs import constants +from ocs_ci.utility import templating +from ocs_ci.ocs import exceptions +from ocs_ci.ocs.resources.ocs import OCS +from ocs_ci.utility.utils import TimeoutSampler +from ocs_ci.ocs.resources.csv import CSV, get_csvs_start_with_prefix +from ocs_ci.ocs.ocp import OCP +from ocs_ci.ocs.exceptions import TimeoutExpiredError + + +class NMStateInstaller(object): + """ + NMState Installer class for NMState deployment + + """ + + def __init__(self): + self.namespace = constants.NMSTATE_NAMESPACE + + def create_nmstate_operator_namespace(self): + """ + Creates the namespace for NMState resources + + Raises: + CommandFailed: If the 'oc create' command fails. + + """ + try: + logger.info(f"Creating namespace {self.namespace} for NMState resources") + namespace_yaml_file = templating.load_yaml(constants.NMSTATE_NAMESPACE_YAML) + namespace_yaml = OCS(**namespace_yaml_file) + namespace_yaml.create() + logger.info(f"NMState namespace {self.namespace} was created successfully") + except exceptions.CommandFailed as ef: + if ( + f'project.project.openshift.io "{self.namespace}" already exists' + in str(ef) + ): + logger.info(f"Namespace {self.namespace} already present") + raise ef + + def create_nmstate_operatorgroup(self): + """ + Creates an OperatorGroup for NMState + + """ + logger.info("Creating OperatorGroup for NMState") + operatorgroup_yaml_file = templating.load_yaml( + constants.NMSTATE_OPERATORGROUP_YAML + ) + operatorgroup_yaml = OCS(**operatorgroup_yaml_file) + operatorgroup_yaml.create() + logger.info("NMState OperatorGroup created successfully") + + def create_nmstate_subscription(self): + """ + Creates subscription for NMState operator + + """ + logger.info("Creating Subscription for NMState") + subscription_yaml_file = templating.load_yaml( + constants.NMSTATE_SUBSCRIPTION_YAML + ) + subscription_yaml = OCS(**subscription_yaml_file) + subscription_yaml.create() + logger.info("NMState Subscription created successfully") + + def verify_nmstate_csv_status(self): + """ + Verify the CSV status for the nmstate Operator deployment equals Succeeded + + """ + for csv in TimeoutSampler( + timeout=900, + sleep=15, + func=get_csvs_start_with_prefix, + csv_prefix=constants.NMSTATE_CSV_NAME, + namespace=self.namespace, + ): + if csv: + break + csv_name = csv[0]["metadata"]["name"] + csv_obj = CSV(resource_name=csv_name, namespace=self.namespace) + csv_obj.wait_for_phase(phase="Succeeded", timeout=720) + + def create_nmstate_instance(self): + """ + Create an instance of the nmstate Operator + + """ + logger.info("Creating NMState Instance") + subscription_yaml_file = templating.load_yaml(constants.NMSTATE_INSTANCE_YAML) + subscription_yaml = OCS(**subscription_yaml_file) + subscription_yaml.create() + logger.info("NMState Instance created successfully") + + def verify_nmstate_pods_running(self): + """ + Verify the pods for NMState Operator are running + + """ + sample = TimeoutSampler( + timeout=300, + sleep=10, + func=self.count_nmstate_pods_running, + count=10, + ) + if not sample.wait_for_func_status(result=True): + raise TimeoutExpiredError( + "Not all nmstate pods in Running state after 300 seconds" + ) + + def count_nmstate_pods_running(self, count): + """ + Count the pods for NMState Operator are running + + Returns: + bool: + + """ + count_running_nmstate_pods = 0 + ocp_pod = OCP(kind=constants.POD, namespace=self.namespace) + pod_items = ocp_pod.get().get("items") + # Check if nmstate pods are in running state + for nmstate_pod in pod_items: + nmstate_pod_name = nmstate_pod.get("metadata").get("name") + status = ocp_pod.get_resource_status(nmstate_pod_name) + if status == constants.STATUS_RUNNING: + logger.info(f"NMState pod {nmstate_pod_name} in running state") + count_running_nmstate_pods += 1 + return count_running_nmstate_pods >= count + + def running_nmstate(self): + """ + Install NMState operator and create an instance + + """ + self.create_nmstate_operator_namespace() + self.create_nmstate_operatorgroup() + self.create_nmstate_subscription() + self.verify_nmstate_csv_status() + self.create_nmstate_instance() + self.verify_nmstate_pods_running() diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 9802dfe5086..a3e565d53fe 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -32,6 +32,7 @@ TEMPLATE_MULTICLUSTER_DIR = os.path.join(TEMPLATE_DEPLOYMENT_DIR, "multicluster") TEMPLATE_DEPLOYMENT_DIR_CNV = os.path.join(TEMPLATE_DIR, "cnv-deployment") TEMPLATE_DEPLOYMENT_DIR_METALLB = os.path.join(TEMPLATE_DIR, "metallb-deployment") +TEMPLATE_DEPLOYMENT_DIR_NMSTATE = os.path.join(TEMPLATE_DIR, "nmstate-deployment") TEMPLATE_CEPH_DIR = os.path.join(TEMPLATE_DIR, "ceph") TEMPLATE_CSI_DIR = os.path.join(TEMPLATE_DIR, "CSI") TEMPLATE_CSI_LVM_DIR = os.path.join(TEMPLATE_CSI_DIR, "lvm") @@ -887,11 +888,30 @@ TEMPLATE_DEPLOYMENT_DIR_OCP, "qe-app-registry-catalog-source.yaml" ) +# NMState deployment +NMSTATE_NAMESPACE_YAML = os.path.join(TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "namespace.yaml") +NMSTATE_OPERATORGROUP_YAML = os.path.join( + TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "operatorgroup.yaml" +) +NMSTATE_SUBSCRIPTION_YAML = os.path.join( + TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "subscription.yaml" +) +NMSTATE_INSTANCE_YAML = os.path.join( + TEMPLATE_DEPLOYMENT_DIR_NMSTATE, "nmstate_instance.yaml" +) +NMSTATE_NAMESPACE = "openshift-nmstate" +NMSTATE_CSV_NAME = "kubernetes-nmstate-operator" + # Multus Networks MULTUS_PUBLIC_NET_YAML = os.path.join(TEMPLATE_DEPLOYMENT_DIR, "multus-public-net.yaml") MULTUS_CLUSTER_NET_YAML = os.path.join( TEMPLATE_DEPLOYMENT_DIR, "multus-cluster-net.yaml" ) +NODE_NETWORK_CONFIGURATION_POLICY = os.path.join( + TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy.yaml" +) +NETWORK_ATTACHEMENT_DEFINITION = "network-attachment-definitions.k8s.cni.cncf.io" + OPERATOR_SOURCE_NAME = "ocs-operatorsource" diff --git a/ocs_ci/templates/nmstate-deployment/namespace.yaml b/ocs_ci/templates/nmstate-deployment/namespace.yaml new file mode 100644 index 00000000000..61d6bb95cca --- /dev/null +++ b/ocs_ci/templates/nmstate-deployment/namespace.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: openshift-nmstate + name: openshift-nmstate + name: openshift-nmstate +spec: + finalizers: + - kubernetes diff --git a/ocs_ci/templates/nmstate-deployment/nmstate_instance.yaml b/ocs_ci/templates/nmstate-deployment/nmstate_instance.yaml new file mode 100644 index 00000000000..cbba0c6c1b8 --- /dev/null +++ b/ocs_ci/templates/nmstate-deployment/nmstate_instance.yaml @@ -0,0 +1,4 @@ +apiVersion: nmstate.io/v1 +kind: NMState +metadata: + name: nmstate diff --git a/ocs_ci/templates/nmstate-deployment/operatorgroup.yaml b/ocs_ci/templates/nmstate-deployment/operatorgroup.yaml new file mode 100644 index 00000000000..6b25af98678 --- /dev/null +++ b/ocs_ci/templates/nmstate-deployment/operatorgroup.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + annotations: + olm.providedAPIs: NMState.v1.nmstate.io + name: openshift-nmstate + namespace: openshift-nmstate +spec: + targetNamespaces: + - openshift-nmstate diff --git a/ocs_ci/templates/nmstate-deployment/subscription.yaml b/ocs_ci/templates/nmstate-deployment/subscription.yaml new file mode 100644 index 00000000000..bb884069d90 --- /dev/null +++ b/ocs_ci/templates/nmstate-deployment/subscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + labels: + operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: "" + name: kubernetes-nmstate-operator + namespace: openshift-nmstate +spec: + channel: stable + installPlanApproval: Automatic + name: kubernetes-nmstate-operator + source: redhat-operators + sourceNamespace: openshift-marketplace diff --git a/ocs_ci/templates/ocs-deployment/multus-public-net.yaml b/ocs_ci/templates/ocs-deployment/multus-public-net.yaml index 5d2647077b6..49249f50802 100644 --- a/ocs_ci/templates/ocs-deployment/multus-public-net.yaml +++ b/ocs_ci/templates/ocs-deployment/multus-public-net.yaml @@ -13,6 +13,7 @@ spec: "mode": "bridge", "ipam": { "type": "whereabouts", - "range": "192.168.20.0/24" + "range": "192.168.20.0/24", + "routes": [{"dst": "192.168.252.0/24"}] } }' diff --git a/ocs_ci/templates/ocs-deployment/node_network_configuration_policy.yaml b/ocs_ci/templates/ocs-deployment/node_network_configuration_policy.yaml new file mode 100644 index 00000000000..d5b804799ef --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/node_network_configuration_policy.yaml @@ -0,0 +1,29 @@ +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: ceph-public-net-shim-worker-node + namespace: openshift-storage +spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + kubernetes.io/hostname: worker-node + desiredState: + interfaces: + - name: odf-pub-shim + description: Shim interface used to connect host to OpenShift Data Foundation public Multus network + type: mac-vlan + state: up + mac-vlan: + base-iface: enp1s0f1 + mode: bridge + promiscuous: true + ipv4: + enabled: true + dhcp: false + address: + - ip: 192.168.252.1 # STATIC IP FOR worker node + prefix-length: 24 + routes: + config: + - destination: 192.168.20.0/24 + next-hop-interface: odf-pub-shim