Merge pull request #8483 from clacroix12/test-ownership

Test Squad Ownership Decorators

ocs_ci/framework/pytest_customization/marks.py

@@ -484,6 +484,10 @@
 purple_squad = pytest.mark.purple_squad
 red_squad = pytest.mark.red_squad
 turquoise_squad = pytest.mark.turquoise_squad
+yellow_squad = pytest.mark.yellow_squad
+
+# Ignore test during squad decorator check in pytest collection
+ignore_owner = pytest.mark.ignore_owner
 
 # Marks to identify the cluster type in which the test case should run
 runs_on_provider = pytest.mark.runs_on_provider

ocs_ci/helpers/pvc_ops.py

@@ -2,6 +2,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from time import sleep
 
+from ocs_ci.framework.pytest_customization.marks import brown_squad
 from ocs_ci.framework.testlib import ignore_leftovers
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.resources.pvc import delete_pvcs
@@ -48,6 +49,7 @@ def delete_pods(pod_objs):
         pod_obj.delete()
 
 
+@brown_squad
 @ignore_leftovers
 def test_create_delete_pvcs(multi_pvc_factory, pod_factory, project=None):
     # create the pods for deleting

ocs_ci/ocs/constants.py

@@ -1830,6 +1830,7 @@
     "Yellow": ["/managed-service/"],
     "Turquoise": ["/disaster-recovery/"],
 }
+SQUAD_CHECK_IGNORED_MARKERS = ["ignore_owner", "libtest"]
 
 PRODUCTION_JOBS_PREFIX = ["jnk"]
 

ocs_ci/ocs/exceptions.py

@@ -631,6 +631,10 @@ class ArchitectureNotSupported(Exception):
     pass
 
 
+class MissingSquadDecoratorError(Exception):
+    pass
+
+
 class PDBNotCreatedException(Exception):
     pass
 

pytest.ini

@@ -57,6 +57,7 @@ markers =
     black_squad: marker for black squad
     yellow_squad: marker for yellow squad
     turquoise_squad: marker for turquoise squad
+    ignore_owner: marker to ignore test during squad decorator check in pytest collection
 
 # Clusterctx used without hyphen, to keep the original format if it's None
 log_format = %(asctime)s - %(threadName)s - %(name)s - %(levelname)s %(clusterctx)s - %(message)s

tests/conftest.py

@@ -2,6 +2,7 @@
 import copy
 import logging
 import os
+import pandas as pd
 import random
 import time
 import tempfile
@@ -44,6 +45,7 @@
     PoolNotDeletedFromUI,
     StorageClassNotDeletedFromUI,
     ResourceNotDeleted,
+    MissingSquadDecoratorError,
 )
 from ocs_ci.ocs.mcg_workload import mcg_job_factory as mcg_job_factory_implementation
 from ocs_ci.ocs.node import get_node_objs, schedule_nodes
@@ -181,6 +183,97 @@ def pytest_logger_config(logger_config):
     logger_config.set_formatter_class(OCSLogFormatter)
 
 
+def verify_squad_owners(items):
+    """
+    Verify that all tests collected are decorated with a squad marker
+
+    Args:
+        items: list of collected tests
+
+    """
+    items_without_squad_marker = {}
+    for item in items:
+        base_dir = os.path.join(constants.TOP_DIR, "tests")
+        ignored_markers = constants.SQUAD_CHECK_IGNORED_MARKERS
+        if item.fspath.strpath.startswith(base_dir):
+            item_markers = [marker.name for marker in item.iter_markers()]
+            if any(marker in item_markers for marker in ignored_markers):
+                log.debug(
+                    "Ignoring test case %s as it has a marker in the ignore list",
+                    item.name,
+                )
+            elif not any(["_squad" in marker for marker in item_markers]):
+                log.debug("%s is missing a squad owner marker", item.name)
+                items_without_squad_marker.update({item.name: item.fspath.strpath})
+
+    if items_without_squad_marker:
+        msg = f"""
+Missing squad decorator for the following test items: {json.dumps(items_without_squad_marker, indent=4)}
+
+Tests are required to be decorated with their squad owner. Please add the tests respective owner.
+
+For example:
+
+    @magenta_squad
+    def test_name():
+
+Test owner marks can be imported from `ocs_ci.framework.pytest_customization.marks`
+            """
+        raise MissingSquadDecoratorError(msg)
+
+
+def export_squad_marker_to_csv(items, filename=None):
+    """
+    Export data regarding tests that are missing squad markers to a CSV
+
+    Args:
+        items: list of collected tests
+        filename: name of the file to export the data to
+
+    """
+    _filename = filename or "squad_decorator_data.csv"
+    test_data = {"File": [], "Name": [], "Suggestions": []}
+    ignored_markers = constants.SQUAD_CHECK_IGNORED_MARKERS
+    for item in items:
+        item_markers = [marker.name for marker in item.iter_markers()]
+        if any(marker in item_markers for marker in ignored_markers):
+            log.debug(
+                "Ignoring test case %s as it has a marker in the ignore list", item.name
+            )
+        else:
+            item_squad = None
+            for marker in item_markers:
+                if "_squad" in marker:
+                    item_squad = marker.split("_")[0]
+                    item_squad = item_squad.capitalize()
+                    log.info("Test item %s has squad marker: %s", item.name, marker)
+            if not item_squad:
+                suggested_squads = []
+                for squad, paths in constants.SQUADS.items():
+                    for _path in paths:
+                        test_path = os.path.relpath(
+                            item.fspath.strpath, constants.TOP_DIR
+                        )
+                        if _path in test_path:
+                            suggested_squads.append(squad)
+                test_data["File"].append(item.fspath.strpath)
+                test_data["Name"].append(item.name)
+                test_data["Suggestions"].append(",".join(suggested_squads))
+
+    df = pd.DataFrame(data=test_data)
+    df.to_csv(
+        _filename,
+        header=["File ", "Test Name", "Squad Suggestions"],
+        index=False,
+        sep=",",
+        mode="a",
+    )
+    num_tests = len(test_data["Name"])
+    num_files = len(set(test_data["File"]))
+    log.info("Exported squad marker info to %s", _filename)
+    log.info("%s tests require action across %s files", num_tests, num_files)
+
+
 def pytest_collection_modifyitems(session, items):
     """
     A pytest hook to filter out skipped tests satisfying
@@ -196,24 +289,16 @@ def pytest_collection_modifyitems(session, items):
     deploy = config.RUN["cli_params"].get("deploy")
     skip_ocs_deployment = config.ENV_DATA["skip_ocs_deployment"]
 
+    # Verify tests are decorated with the correct squad owner
+    verify_squad_owners(items)
+
     # Add squad markers to each test item based on filepath
     for item in items:
         # check, if test already have squad marker manually assigned
-        skip_path_squad_marker = False
         for marker in item.iter_markers():
             if "_squad" in marker.name:
                 squad = marker.name.split("_")[0]
                 item.user_properties.append(("squad", squad.capitalize()))
-                skip_path_squad_marker = True
-        if not skip_path_squad_marker:
-            for squad, paths in constants.SQUADS.items():
-                for _path in paths:
-                    # Limit the test_path to the tests directory
-                    test_path = os.path.relpath(item.fspath.strpath, constants.TOP_DIR)
-                    if _path in test_path:
-                        item.add_marker(f"{squad.lower()}_squad")
-                        item.user_properties.append(("squad", squad))
-                        break
 
     if not (teardown or deploy or (deploy and skip_ocs_deployment)):
         for item in items[:]:

tests/disaster-recovery/sc_arbiter/test_netsplit.py

@@ -3,6 +3,7 @@
 import time
 import ocpnetsplit
 
+from ocs_ci.framework.pytest_customization.marks import turquoise_squad
 from ocs_ci.utility.retry import retry
 from ocs_ci.ocs.exceptions import CommandFailed, CephHealthException
 from ocs_ci.ocs import constants
@@ -76,6 +77,7 @@ def get_logfile_map_from_logwriter_pods(logwriter_pods, is_rbd=False):
     return log_file_map
 
 
+@turquoise_squad
 class TestNetSplit:
     @pytest.fixture()
     def init_sanity(self, request):

tests/e2e/flowtest/mcg/test_mcg_namespace_disruptions_crd.py

@@ -8,6 +8,7 @@
     skipif_aws_creds_are_missing,
     flowtests,
     skipif_managed_service,
+    red_squad,
 )
 from ocs_ci.framework.testlib import E2ETest, skipif_ocs_version
 from ocs_ci.ocs.bucket_utils import (
@@ -32,6 +33,7 @@
 logger = logging.getLogger(__name__)
 
 
+@red_squad
 @skipif_managed_service
 @skipif_aws_creds_are_missing
 @skipif_ocs_version("<4.7")

tests/e2e/flowtest/mcg/test_mcg_namespace_disruptions_rpc.py

@@ -9,6 +9,7 @@
     skipif_aws_creds_are_missing,
     flowtests,
     skipif_managed_service,
+    red_squad,
 )
 from ocs_ci.framework.testlib import E2ETest, skipif_ocs_version
 from ocs_ci.ocs.bucket_utils import (
@@ -38,6 +39,7 @@
 logger = logging.getLogger(__name__)
 
 
+@red_squad
 @skipif_managed_service
 @skipif_aws_creds_are_missing
 @skipif_ocs_version("!=4.6")

tests/e2e/flowtest/pvc_snapshot_and_clone/test_pgsql_pvc_snapshot_and_clone.py

@@ -1,6 +1,7 @@
 import logging
 import pytest
 
+from ocs_ci.framework.pytest_customization.marks import magenta_squad
 from ocs_ci.framework.testlib import (
     skipif_ocs_version,
     skipif_ocp_version,
@@ -14,6 +15,7 @@
 log = logging.getLogger(__name__)
 
 
+@magenta_squad
 @flowtests
 class TestPvcSnapshotAndCloneWithBaseOperation(E2ETest):
     """

tests/e2e/flowtest/pvc_snapshot_and_clone/test_pgsql_pvc_snapshot_and_clone_with_base_operation.py

@@ -2,7 +2,7 @@
 import pytest
 
 from concurrent.futures import ThreadPoolExecutor
-
+from ocs_ci.framework.pytest_customization.marks import magenta_squad
 from ocs_ci.framework.testlib import (
     skipif_ocs_version,
     skipif_ocp_version,
@@ -19,6 +19,7 @@
 log = logging.getLogger(__name__)
 
 
+@magenta_squad
 @flowtests
 @ignore_leftovers
 class TestPvcSnapshotAndCloneWithBaseOperation(E2ETest):

tests/e2e/flowtest/test_base_operation_node_drain.py

@@ -5,6 +5,7 @@
 from concurrent.futures import ThreadPoolExecutor
 
 from ocs_ci.framework.pytest_customization.marks import (
+    magenta_squad,
     skipif_bm,
     skipif_aws_i3,
     skipif_vsphere_ipi,
@@ -28,6 +29,7 @@ class TestBaseOperationNodeDrain(E2ETest):
 
     """
 
+    @magenta_squad
     @skipif_aws_i3
     @skipif_bm
     @skipif_vsphere_ipi

tests/e2e/kcs/test_disable_mcg_external_service.py

@@ -9,11 +9,13 @@
     polarion_id,
     bugzilla,
     skipif_external_mode,
+    magenta_squad,
 )
 
 logger = logging.getLogger(__name__)
 
 
+@magenta_squad
 class TestDisableMCGExternalService:
     @pytest.fixture()
     def patch_noobaa_object(self, request):

tests/e2e/kcs/test_maintenance_pod.py

@@ -8,6 +8,7 @@
     bugzilla,
     tier2,
     skipif_external_mode,
+    magenta_squad,
 )
 from ocs_ci.framework.testlib import E2ETest
 from ocs_ci.ocs.resources.deployment import (
@@ -25,6 +26,7 @@
 logger = logging.getLogger(__name__)
 
 
+@magenta_squad
 @tier2
 @skipif_external_mode
 @bugzilla("2103256")

tests/e2e/kcs/test_mon_crash_recovery_scenario.py

@@ -12,14 +12,19 @@
 from ocs_ci.ocs.resources.pvc import get_pvc_objs
 from ocs_ci.ocs.resources.pod import get_ceph_tools_pod, run_io_in_bg
 from ocs_ci.ocs.resources.storage_cluster import ceph_mon_dump
-from ocs_ci.framework.pytest_customization.marks import tier3, skipif_external_mode
+from ocs_ci.framework.pytest_customization.marks import (
+    tier3,
+    skipif_external_mode,
+    magenta_squad,
+)
 from ocs_ci.ocs.defaults import OCS_OPERATOR_NAME
 from ocs_ci.helpers.helpers import wait_for_resource_state
 
 
 log = logging.getLogger(__name__)
 
 
+@magenta_squad
 @tier3
 @pytest.mark.polarion_id("OCS-4942")
 @pytest.mark.bugzilla("2151591")

tests/e2e/kcs/test_monitor_recovery.py

@@ -16,6 +16,7 @@
     skipif_external_mode,
     system_test,
     skipif_ocp_version,
+    magenta_squad,
 )
 from ocs_ci.helpers.sanity_helpers import Sanity
 from ocs_ci.ocs.ocp import OCP, switch_to_project
@@ -49,6 +50,7 @@
 logger = logging.getLogger(__name__)
 
 
+@magenta_squad
 @system_test
 @ignore_leftovers
 @pytest.mark.last

tests/e2e/kcs/test_noobaa_db_backup_and_recovery.py

@@ -2,6 +2,7 @@
 import pytest
 
 from ocs_ci.ocs import warp
+from ocs_ci.framework.pytest_customization.marks import magenta_squad
 from ocs_ci.framework.testlib import (
     ignore_leftovers,
     E2ETest,
@@ -20,6 +21,7 @@
 log = logging.getLogger(__name__)
 
 
+@magenta_squad
 @tier3
 @ignore_leftovers
 @skipif_managed_service

tests/e2e/kcs/test_noobaa_rebuild.py

@@ -3,6 +3,7 @@
 import pytest
 
 from ocs_ci.framework import config
+from ocs_ci.framework.pytest_customization.marks import magenta_squad
 from ocs_ci.framework.testlib import (
     ignore_leftovers,
     E2ETest,
@@ -21,6 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
+@magenta_squad
 @tier3
 @ignore_leftovers
 @pytest.mark.polarion_id("OCS-2653")