Merge branch 'mr/pmderodat/xunit-importer' into 'master'

e3-convert-xunit: warn about dangling XFAILs annotations

See merge request it/e3-testsuite!21

NEWS.md

@@ -1,6 +1,7 @@
 27.0 (Not released yet)
 =======================
 
+* `e3-convert-xunit`: warn about dangling XFAILs annotations.
 * `DiffTestDriver`: tolerate missing baseline files when rewriting baselines.
 
 26.0 (2023-01-19)

src/e3/testsuite/report/xunit.py

@@ -153,6 +153,33 @@ def __init__(
         self.index = index
         self.xfails = xfails or {}
 
+        self.dangling_xfails: set[str] = set()
+        """
+        Set of tests for which a failure is expected, but that are not present
+        in the testsuite report. Computed in the "run" method.
+        """
+
+    def warn_dangling_xfails(self) -> None:
+        """
+        Print warnings for dangling entries in the "xfails" dict.
+
+        This prints warnings on the standard output to mention all tests for
+        which a failure is expected, but that are not present in the testsuite
+        report.
+        """
+        if not self.dangling_xfails:
+            return
+        print(
+            "warning: the following tests are expected to fail but are not"
+            " present in the testsuite results:"
+        )
+        for test_name in sorted(self.dangling_xfails):
+            reason = self.xfails.get(test_name)
+            if reason:
+                print(f"  {test_name} ({reason})")
+            else:
+                print(f"  {test_name}")
+
     def run(self, filename: str) -> None:
         """Read a xUnit report and import its results in the report index.
 
@@ -247,6 +274,9 @@ def run(self, filename: str) -> None:
                 result.set_status(status, message)
                 self.index.save_and_add_result(result)
 
+        # Now that all tests are known, compute the set of dangling XFAILs
+        self.dangling_xfails = set(self.xfails) - set(self.index.entries)
+
     SLUG_RE = re.compile("[a-zA-Z0-9_.]+")
 
     def slugify(self, name: str) -> str:
@@ -350,6 +380,7 @@ def convert_main(argv: list[str] | None = None) -> None:
                     importer.run(os.path.join(root, f))
         else:
             importer.run(path)
+    importer.warn_dangling_xfails()
     index.write()
 
     if args.gaia_output:

tests/tests/test_xunit.py

@@ -14,6 +14,12 @@
 from .utils import create_testsuite, run_testsuite
 
 
+def write_xfails_yaml(filename, xfails):
+    """Write a XFAILs YAML file."""
+    with open(filename, "w") as f:
+        yaml.dump(xfails, f)
+
+
 class TestBasic:
     """Check that requesting a XUnit testsuite report works."""
 
@@ -149,7 +155,7 @@ def test(self, tmp_path):
         assert failure.text == "Control character: \\x01\nDone.\n"
 
 
-def test_import(tmp_path):
+def test_import(tmp_path, capsys):
     """Test that the xUnit importer works as expected."""
     xml_filename = str(tmp_path / "tmp.xml")
     with open(xml_filename, "w") as f:
@@ -234,15 +240,16 @@ def test_import(tmp_path):
         )
 
     xfails_filename = str(tmp_path / "xfails.yaml")
-    xfails = {
-        "XFails.test-ok": "",
-        "XFails.test-failure": "",
-        "XFails.test-error": "",
-        "XFails.test-skipped": "",
-        "XFails.test-failure-message": "Expected failure message",
-    }
-    with open(xfails_filename, "w") as f:
-        yaml.dump(xfails, f)
+    write_xfails_yaml(
+        xfails_filename,
+        {
+            "XFails.test-ok": "",
+            "XFails.test-failure": "",
+            "XFails.test-error": "",
+            "XFails.test-skipped": "",
+            "XFails.test-failure-message": "Expected failure message",
+        },
+    )
 
     results_dir = str(tmp_path / "results")
     convert_main(
@@ -326,6 +333,10 @@ def check_log(test_name, log):
     check("XFails.pytest-xfail", Status.XFAIL, "Known bug")
     check_log("XFails.pytest-xfail", "Some error logging")
 
+    captured = capsys.readouterr()
+    assert captured.out == ""
+    assert captured.err == ""
+
 
 def test_import_dirs(tmp_path):
     """Test XML reports search in the xUnit conversion script."""
@@ -380,3 +391,40 @@ def test_gaia(tmp_path):
 
     with open(str(results_dir / "results"), "r") as f:
         assert f.read() == "MyTestsuite.MyTestcase:OK:\n"
+
+
+def test_dangling_xfails(tmp_path, capsys):
+    """Test emission of warnings for dangling XFAILs."""
+    xml_report = str(tmp_path / "test.xml")
+    with open(xml_report, "w") as f:
+        f.write(
+            """<?xml version="1.0" encoding="utf-8"?>
+            <testsuites name="MyTestsuites">
+              <testsuite name="MyTestsuite">
+                <testcase name="MyTestcase"></testcase>
+              </testsuite>
+            </testsuites>
+            """
+        )
+
+    xfails_filename = str(tmp_path / "xfails.yaml")
+    write_xfails_yaml(xfails_filename, {"foo": "reason", "bar": ""})
+
+    convert_main(
+        [
+            "-o",
+            str(tmp_path / "results"),
+            "--xfails",
+            xfails_filename,
+            xml_report,
+        ]
+    )
+
+    captured = capsys.readouterr()
+    assert captured.out == (
+        "warning: the following tests are expected to fail but are not present"
+        " in the testsuite results:\n"
+        "  bar\n"
+        "  foo (reason)\n"
+    )
+    assert captured.err == ""