Test ODF must-gather in modular mode (#9343)

* OCS must gather with flags

Signed-off-by: oviner <[email protected]>

ocs_ci/ocs/must_gather/const_must_gather.py

@@ -1008,3 +1008,26 @@
         ),
     },
 }
+
+CEPH_ONLY = [
+    "/ceph/logs/",
+    "/ceph/must_gather_commands/",
+    "/ceph/must_gather_commands_json_output/",
+    "/ceph/namespaces/",
+]
+CEPH_LOGS_ONLY = [
+    "/ceph_logs/journal_",
+    "/ceph_logs/ceph_daemon_log",
+    "/ceph_logs/kernel",
+]
+NAMESPACED_ONLY = ["/namespaces/all/"]
+CLUSTERSCOPED_ONLY = [
+    "/cluster-scoped-resources/oc_output/",
+    "/cluster-scoped-resources/core/",
+]
+NOOBAA_ONLY = [
+    "/noobaa/raw_output",
+    "/noobaa/namespaces",
+    "/noobaa/logs/openshift-storage",
+]
+DR_ONLY = ["/namespaces/openshift-dr-system/"]

ocs_ci/ocs/must_gather/must_gather.py

@@ -35,6 +35,7 @@ def __init__(self):
         self.files_not_exist = list()
         self.files_content_issue = list()
         self.ocs_version = version.get_semantic_ocs_version_from_config()
+        self.full_paths = list()
 
     @property
     def log_type(self):
@@ -46,13 +47,16 @@ def log_type(self, type_log):
             raise ValueError("log type arg must be a string")
         self.type_log = type_log
 
-    def collect_must_gather(self):
+    def collect_must_gather(self, ocs_flags=None):
         """
         Collect ocs_must_gather and copy the logs to a temporary folder.
 
+        Args:
+            ocs_flags (str): flags to must gather command for example ["-- /usr/bin/gather -cs"]
+
         """
         temp_folder = tempfile.mkdtemp()
-        collect_ocs_logs(dir_name=temp_folder, ocp=False)
+        collect_ocs_logs(dir_name=temp_folder, ocp=False, ocs_flags=ocs_flags)
         self.root = temp_folder + "_ocs_logs"
 
     def search_file_path(self):
@@ -245,6 +249,60 @@ def verify_noobaa_diagnostics(self):
                 logger.error("noobaa_diagnostics.tar.gz does not exist")
                 self.files_not_exist.append("noobaa_diagnostics.tar.gz")
 
+    def get_all_paths(self):
+        """
+        Get all paths in must gather dir
+
+        """
+        for root, dirs, files in os.walk(self.root):
+            for name in files + dirs:
+                full_path = os.path.join(root, name)
+                self.full_paths.append(full_path)
+
+    def verify_paths_in_dir(self, paths):
+        """
+        Verify paths exist in must gather directory
+
+        Args:
+            paths (list): list of paths in mg directory
+                for example ``/ceph_logs/journal_`` exist in ``/mg_dir/a/b/ceph/ceph_logs/journal_compute-1/log.log``
+
+        Returns:
+            list: the paths do not exist in mg dir
+
+        """
+        paths_exist = []
+        for path in paths:
+            status = False
+            for full_path in self.full_paths:
+                if path in full_path:
+                    status = True
+            if not status:
+                paths_exist.append(path)
+        return paths_exist
+
+    def verify_paths_not_in_dir(self, paths):
+        """
+        Verify paths do not exist in must gather directory
+
+        Args:
+            paths (list): list of paths in mg directory
+              for example ``/ceph_logs/journal_`` exist in ``/mg_dir/a/b/ceph/ceph_logs/journal_compute-1/log.log``
+
+        Returns:
+            list: the paths exist in mg dir
+
+        """
+        paths_not_exist = []
+        for path in paths:
+            status = True
+            for full_path in self.full_paths:
+                if path in full_path:
+                    status = False
+            if not status:
+                paths_not_exist.append(path)
+        return paths_not_exist
+
     def validate_must_gather(self):
         """
         Validate must-gather

ocs_ci/ocs/utils.py

@@ -1119,7 +1119,13 @@ def collect_noobaa_db_dump(log_dir_path, cluster_config=None):
 
 
 def _collect_ocs_logs(
-    cluster_config, dir_name, ocp=True, ocs=True, mcg=False, status_failure=True
+    cluster_config,
+    dir_name,
+    ocp=True,
+    ocs=True,
+    mcg=False,
+    status_failure=True,
+    ocs_flags=None,
 ):
     """
     This function runs in thread
@@ -1174,6 +1180,7 @@ def _collect_ocs_logs(
             ocs_log_dir_path,
             ocs_must_gather_image_and_tag,
             cluster_config=cluster_config,
+            command=ocs_flags,
         )
         if (
             ocsci_config.DEPLOYMENT.get("disconnected")
@@ -1250,7 +1257,9 @@ def _collect_ocs_logs(
         log.info(out)
 
 
-def collect_ocs_logs(dir_name, ocp=True, ocs=True, mcg=False, status_failure=True):
+def collect_ocs_logs(
+    dir_name, ocp=True, ocs=True, mcg=False, status_failure=True, ocs_flags=None
+):
     """
     Collects OCS logs
 
@@ -1262,6 +1271,7 @@ def collect_ocs_logs(dir_name, ocp=True, ocs=True, mcg=False, status_failure=Tru
         mcg (bool): True for collecting MCG logs (noobaa db dump)
         status_failure (bool): Whether the collection is after success or failure,
             allows better naming for folders under logs directory
+        ocs_flags (str): flags to ocs must gather command for example ["-- /usr/bin/gather -cs"]
 
     """
     results = None
@@ -1275,6 +1285,7 @@ def collect_ocs_logs(dir_name, ocp=True, ocs=True, mcg=False, status_failure=Tru
                 ocs=ocs,
                 mcg=mcg,
                 status_failure=status_failure,
+                ocs_flags=ocs_flags,
             )
             for cluster in ocsci_config.clusters
         ]

tests/functional/z_cluster/test_must_gather_modular.py

@@ -0,0 +1,97 @@
+import logging
+import pytest
+
+from ocs_ci.framework.pytest_customization.marks import brown_squad, bugzilla
+from ocs_ci.framework.testlib import (
+    ManageTest,
+    tier2,
+    skipif_external_mode,
+    skipif_ms_consumer,
+    skipif_hci_client,
+)
+from ocs_ci.ocs.must_gather.must_gather import MustGather
+from ocs_ci.ocs.must_gather import const_must_gather
+
+logger = logging.getLogger(__name__)
+
+
+@brown_squad
+@bugzilla("2162812")
+class TestMustGather(ManageTest):
+    @tier2
+    @pytest.mark.parametrize(
+        argnames=[
+            "ceph",
+            "ceph_logs",
+            "namespaced",
+            "clusterscoped",
+            "noobaa",
+            "dr",
+        ],
+        argvalues=[
+            pytest.param(
+                *[True, True, False, True, False, False],
+                marks=[
+                    pytest.mark.polarion_id("OCS-5797"),
+                    skipif_external_mode,
+                    skipif_ms_consumer,
+                    skipif_hci_client,
+                ],
+            ),
+            pytest.param(
+                *[True, True, True, False, True, False],
+                marks=[
+                    pytest.mark.polarion_id("OCS-5797"),
+                    skipif_external_mode,
+                    skipif_ms_consumer,
+                    skipif_hci_client,
+                ],
+            ),
+        ],
+    )
+    def test_must_gather_modular(
+        self, ceph, ceph_logs, namespaced, clusterscoped, noobaa, dr
+    ):
+        """
+        Tests OCS must gather with flags
+
+        Test Process:
+        1.Collect mg with relevant flags for example
+            oc adm must-gather --image=quay.io/rhceph-dev/ocs-must-gather:latest-4.15 -- /usr/bin/gather -c -cl -n
+        2.Calculate the paths that should be included in the ocs mg dir
+        3.Calculate the paths that should not be included in the ocs mg dir
+        4.Verify paths exist in must gather directory
+        5.Verify paths do not exist in must gather directory
+        """
+
+        flags_cmd = "/usr/bin/gather "
+        paths_exist = list()
+        paths_not_exist = list()
+
+        options = [
+            (ceph, const_must_gather.CEPH_ONLY, "-c "),
+            (ceph_logs, const_must_gather.CEPH_LOGS_ONLY, "-cl "),
+            (namespaced, const_must_gather.NAMESPACED_ONLY, "-ns "),
+            (clusterscoped, const_must_gather.CLUSTERSCOPED_ONLY, "-cs "),
+            (noobaa, const_must_gather.NOOBAA_ONLY, "-n "),
+            (dr, const_must_gather.DR_ONLY, "-d "),
+        ]
+
+        for flag, paths, param_value in options:
+            if flag:
+                for path in paths:
+                    paths_exist.append(path)
+                flags_cmd += param_value
+            else:
+                for path in paths:
+                    paths_not_exist.append(path)
+        mustgather_obj = MustGather()
+        mustgather_obj.collect_must_gather(ocs_flags=flags_cmd)
+        mustgather_obj.get_all_paths()
+        folders_exist = mustgather_obj.verify_paths_in_dir(paths_exist)
+        folders_not_exist = mustgather_obj.verify_paths_not_in_dir(paths_not_exist)
+        assert len(folders_not_exist) + len(folders_exist) == 0, (
+            f"\nMode: {flags_cmd}"
+            f"\nThe folders don't exist [should exist]: {folders_exist} "
+            f"\nThe folders exist [should not exist]: {folders_not_exist}"
+        )