Merge pull request #35 from xenserver/add-coveragerc-mark-not-covered…

…-branches

CP-41819: Py3: Collect, check and report code coverage from Python3 unit test runs

.coveragerc

@@ -0,0 +1,34 @@
+[run]
+source =
+       xen-bugtool
+       tests/
+
+[report]
+# Regular expressions for lines to exclude from consideration
+exclude_lines =
+    # Don't complain if tests don't hit catch-all exception handlers:
+    except:
+    except Exception
+    except IOError
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Enable these selectively if you want to allow these raises without cover:
+    # (If you want no complaint when tests don't hit raising these Assertions)
+    # raise AssertionError
+    # raise NotImplementedError
+    # raise RuntimeError
+    # raise ValueError
+    # \<assert\>.*
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+    if TYPE_CHECKING:
+    # skip any line with a `pass` (such as may be used for @abstractmethod or @suffixed_method)
+    pass
+
+precision = 1
+include =
+    xen-bugtool
+    tests/*

.github/workflows/main.yml

@@ -69,7 +69,10 @@ jobs:
     name: "Python3: Pre-Commit Suite"
     runs-on: ubuntu-22.04
     steps:
+
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # For diff-cover to get the changed lines: origin/master..HEAD
 
       # https://www.python4data.science/en/latest/productive/git/advanced/hooks/ci.html
       - uses: actions/setup-python@v4
@@ -91,3 +94,23 @@ jobs:
         env:
           # Skip the no-commit-to-branch check inside of GitHub CI (for CI on merge to master)
           SKIP: no-commit-to-branch
+
+      - name: Pytest coverage comment
+        if: ${{ github.actor != 'nektos/act' }}
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          junitxml-path: .git/pytest.xml
+          pytest-xml-coverage-path: .git/coverage.xml
+          unique-id-for-comment: pre-commit-coverage
+          title: https://github.com/marketplace/actions/pytest-coverage-comment
+
+      - name: Upload coverage reports to Codecov
+        if: ${{ github.actor != 'nektos/act' }}
+        uses: codecov/codecov-action@v3
+        with:
+          directory: .git
+          env_vars: OS,PYTHON
+          fail_ci_if_error: true
+          flags: unittest
+          name: coverage
+          verbose: true

.pre-commit-config.yaml

@@ -64,10 +64,39 @@ repos:
     -   id: pytest
         name: check that the Xen-Bugtool Test Environment passes
         entry: env PYTHONDEVMODE=yes python3 -m pytest tests/unit
+            --cov xen-bugtool
+            --cov tests/unit
+            --junitxml=.git/pytest.xml
+            --cov-report term-missing
+            --cov-report html:.git/coverage.html
+            --cov-report xml:.git/coverage.xml
+        require_serial: true
         pass_filenames: false
         language: python
         types: [python]
-        additional_dependencies: [defusedxml, pytest, lxml, XenAPI]
+        additional_dependencies: [coverage, defusedxml, pytest-coverage, lxml, XenAPI]
+
+
+    -   id: diff-cover
+        name: check that that the changed lines are covered by tests
+        entry: diff-cover --ignore-whitespace --compare-branch=origin/master
+            --show-uncovered --html-report .git/coverage-diff.html
+            --fail-under 100 .git/coverage.xml
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [diff-cover]
+
+
+    -   id: coverage-report
+        name: check coverage report for minimum overall coverage
+        entry: coverage report --fail-under 55 #| tee .git/coverage.txt
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [coverage]
 -   repo: https://github.com/PyCQA/autoflake
     rev: v2.2.1
     hooks:

tests/integration/conftest.py

@@ -36,10 +36,10 @@ def run_test_functions_with_private_tmpfs_output_directory():
     os.chdir(BUGTOOL_OUTPUT_DIR)
     # Assert that the test case did not leave any unchecked output file as in the output directory:
     remaining_files = []
-    for currentpath, _, files in os.walk("."):
+    for current_path, _, files in os.walk("."):  # pragma: no cover
         for file in files:
-            remaining_files.append(os.path.join(currentpath, file))
-    if remaining_files:
+            remaining_files.append(os.path.join(current_path, file))  # pragma: no cover
+    if remaining_files:  # pragma: no cover
         print("Remaining (possibly unchecked) files found:")
         print(remaining_files)
         os.chdir(BUGTOOL_OUTPUT_DIR)

tests/integration/namespace_container.py

@@ -16,7 +16,7 @@ def unshare(flags):
     libc = ctypes.CDLL(None, use_errno=True)
     libc.unshare.argtypes = [ctypes.c_int]
     rc = libc.unshare(flags)
-    if rc != 0:
+    if rc != 0:  # pragma: no cover
         errno = ctypes.get_errno()
         raise OSError(errno, os.strerror(errno), flags)
 
@@ -33,7 +33,7 @@ def mount(source="none", target="", fs="", flags=0, options=""):
     )
     print("mount -t " + fs + " -o '" + options + "' " + source + "\t" + target)
     result = libc.mount(source.encode(), target.encode(), fs.encode(), flags, options.encode())
-    if result < 0:
+    if result < 0:  # pragma: no cover
         errno = ctypes.get_errno()
         raise OSError(errno, "mount " + target + " (options=" + options + "): " + os.strerror(errno))
 
@@ -42,7 +42,7 @@ def umount(target):
     """Wrapper for the Linux umount system call, supports Python2.7 and Python3.x"""
     libc = ctypes.CDLL(None, use_errno=True)
     result = libc.umount(ctypes.c_char_p(target.encode()))
-    if result < 0:
+    if result < 0:  # pragma: no cover
         errno = ctypes.get_errno()
         raise OSError(errno, "umount " + target + ": " + os.strerror(errno))
 

tests/integration/utils.py

@@ -13,7 +13,7 @@
 
 # pyright: ignore[reportMissingImports]
 if sys.version_info.major == 2:
-    from commands import getstatusoutput  # type:ignore[import-not-found]
+    from commands import getstatusoutput  # type:ignore[import-not-found] # pragma: no cover
 else:
     from subprocess import getstatusoutput
 
@@ -57,12 +57,12 @@ def assert_content_from_dom0_template(path, control_path=None):
     print(control)
     if os.path.isdir(path):
         result = filecmp.dircmp(path, control)
-        if result.diff_files or result.right_only:
+        if result.diff_files or result.right_only:  # pragma: no cover
             print(result.report)
             raise RuntimeError("Missing or Differing files found in " + path)
     else:
         if not filecmp.cmp(path, control):
-            os.system("cat " + path)
+            os.system("cat " + path)  # pragma: no cover
             raise RuntimeError(control)
     # Remove verified output files/directories. Untested files will remain and cause the testcase to FAIL:
     try:
@@ -71,7 +71,7 @@ def assert_content_from_dom0_template(path, control_path=None):
         shutil.rmtree(path)
 
 
-def extract(zip_or_tar_archive, archive_type):
+def extract(zip_or_tar_archive, archive_type):  # pragma: no cover
     """Extract a passed zip, tar or tar.bz2 archive into the current working directory"""
     if sys.version_info > (3, 0):
         if archive_type == "zip" and os.environ.get("GITHUB_ACTION"):

tests/unit/conftest.py

@@ -26,7 +26,7 @@ def imported_bugtool(testdir):
     def import_from_file(module_name, file_path):
         import sys
 
-        if sys.version_info.major == 2:
+        if sys.version_info.major == 2:  # pragma: no cover
             import imp  # pylint: disable=deprecated-module  # pyright: ignore[reportMissingImports]
 
             return imp.load_source(module_name, file_path)

tests/unit/test_xapidb_filter.py

@@ -80,5 +80,5 @@ def test_xapi_database_filter(bugtool):
 
     # Double-check with parseString(): Its output will differ between Py2/Py3
     # though, so we will use it for one language version at a time:
-    if sys.version_info < (3, 0):
+    if sys.version_info < (3, 0):  # pragma: no cover
         assert xml.dom.minidom.parseString(filtered).toprettyxml(indent="    ") == expected

xen-bugtool

@@ -68,7 +68,7 @@ from xml.etree.ElementTree import Element
 
 import defusedxml.sax
 
-if sys.version_info.major == 2:
+if sys.version_info.major == 2:  # pragma: no cover
     from commands import getoutput  # pyright: ignore[reportMissingImports]
     from urllib import urlopen  # type:ignore[attr-defined]