From 0ac65043c0f3499e48b12893f1034e496923f85d Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 8 Jan 2024 12:00:00 +0100 Subject: [PATCH] Add getting code coverage from unit tests, mark not covered branches --- .coveragerc | 34 ++++++++++++++++++++++++ .github/workflows/main.yml | 23 ++++++++++++++++ .pre-commit-config.yaml | 31 ++++++++++++++++++++- tests/integration/conftest.py | 6 ++--- tests/integration/namespace_container.py | 6 ++--- tests/integration/utils.py | 8 +++--- tests/unit/conftest.py | 2 +- tests/unit/test_xapidb_filter.py | 2 +- xen-bugtool | 2 +- 9 files changed, 100 insertions(+), 14 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..640b241b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,34 @@ +[run] +source = + xen-bugtool + tests/ + +[report] +# Regular expressions for lines to exclude from consideration +exclude_lines = + # Don't complain if tests don't hit catch-all exception handlers: + except: + except Exception + except IOError + # Have to re-enable the standard pragma + pragma: no cover + + # Enable these selectively if you want to allow these raises without cover: + # (If you want no complaint when tests don't hit raising these Assertions) + # raise AssertionError + # raise NotImplementedError + # raise RuntimeError + # raise ValueError + # \.* + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + if TYPE_CHECKING: + # skip any line with a `pass` (such as may be used for @abstractmethod or @suffixed_method) + pass + +precision = 1 +include = + xen-bugtool + tests/* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4d8a027..39315b3a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,10 @@ jobs: name: "Python3: Pre-Commit Suite" runs-on: ubuntu-22.04 steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # For diff-cover to get the changed lines: origin/master..HEAD # https://www.python4data.science/en/latest/productive/git/advanced/hooks/ci.html - uses: actions/setup-python@v4 @@ -91,3 +94,23 @@ jobs: env: # Skip the no-commit-to-branch check inside of GitHub CI (for CI on merge to master) SKIP: no-commit-to-branch + + - name: Pytest coverage comment + if: ${{ github.actor != 'nektos/act' }} + uses: MishaKav/pytest-coverage-comment@main + with: + junitxml-path: .git/pytest.xml + pytest-xml-coverage-path: .git/coverage.xml + unique-id-for-comment: pre-commit-coverage + title: https://github.com/marketplace/actions/pytest-coverage-comment + + - name: Upload coverage reports to Codecov + if: ${{ github.actor != 'nektos/act' }} + uses: codecov/codecov-action@v3 + with: + directory: .git + env_vars: OS,PYTHON + fail_ci_if_error: true + flags: unittest + name: coverage + verbose: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afb1561f..22314396 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,10 +64,39 @@ repos: - id: pytest name: check that the Xen-Bugtool Test Environment passes entry: env PYTHONDEVMODE=yes python3 -m pytest tests/unit + --cov xen-bugtool + --cov tests/unit + --junitxml=.git/pytest.xml + --cov-report term-missing + --cov-report html:.git/coverage.html + --cov-report xml:.git/coverage.xml + require_serial: true pass_filenames: false language: python types: [python] - additional_dependencies: [defusedxml, pytest, lxml, XenAPI] + additional_dependencies: [coverage, defusedxml, pytest-coverage, lxml, XenAPI] + + + - id: diff-cover + name: check that that the changed lines are covered by tests + entry: diff-cover --ignore-whitespace --compare-branch=origin/master + --show-uncovered --html-report .git/coverage-diff.html + --fail-under 100 .git/coverage.xml + require_serial: true + pass_filenames: false + language: python + types: [python] + additional_dependencies: [diff-cover] + + + - id: coverage-report + name: check coverage report for minimum overall coverage + entry: coverage report --fail-under 55 #| tee .git/coverage.txt + require_serial: true + pass_filenames: false + language: python + types: [python] + additional_dependencies: [coverage] - repo: https://github.com/PyCQA/autoflake rev: v2.2.1 hooks: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 36951e78..9bc417b5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -36,10 +36,10 @@ def run_test_functions_with_private_tmpfs_output_directory(): os.chdir(BUGTOOL_OUTPUT_DIR) # Assert that the test case did not leave any unchecked output file as in the output directory: remaining_files = [] - for currentpath, _, files in os.walk("."): + for current_path, _, files in os.walk("."): # pragma: no cover for file in files: - remaining_files.append(os.path.join(currentpath, file)) - if remaining_files: + remaining_files.append(os.path.join(current_path, file)) # pragma: no cover + if remaining_files: # pragma: no cover print("Remaining (possibly unchecked) files found:") print(remaining_files) os.chdir(BUGTOOL_OUTPUT_DIR) diff --git a/tests/integration/namespace_container.py b/tests/integration/namespace_container.py index ea07fb48..529269ff 100644 --- a/tests/integration/namespace_container.py +++ b/tests/integration/namespace_container.py @@ -16,7 +16,7 @@ def unshare(flags): libc = ctypes.CDLL(None, use_errno=True) libc.unshare.argtypes = [ctypes.c_int] rc = libc.unshare(flags) - if rc != 0: + if rc != 0: # pragma: no cover errno = ctypes.get_errno() raise OSError(errno, os.strerror(errno), flags) @@ -33,7 +33,7 @@ def mount(source="none", target="", fs="", flags=0, options=""): ) print("mount -t " + fs + " -o '" + options + "' " + source + "\t" + target) result = libc.mount(source.encode(), target.encode(), fs.encode(), flags, options.encode()) - if result < 0: + if result < 0: # pragma: no cover errno = ctypes.get_errno() raise OSError(errno, "mount " + target + " (options=" + options + "): " + os.strerror(errno)) @@ -42,7 +42,7 @@ def umount(target): """Wrapper for the Linux umount system call, supports Python2.7 and Python3.x""" libc = ctypes.CDLL(None, use_errno=True) result = libc.umount(ctypes.c_char_p(target.encode())) - if result < 0: + if result < 0: # pragma: no cover errno = ctypes.get_errno() raise OSError(errno, "umount " + target + ": " + os.strerror(errno)) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 688af940..88c44fa4 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -13,7 +13,7 @@ # pyright: ignore[reportMissingImports] if sys.version_info.major == 2: - from commands import getstatusoutput # type:ignore[import-not-found] + from commands import getstatusoutput # type:ignore[import-not-found] # pragma: no cover else: from subprocess import getstatusoutput @@ -57,12 +57,12 @@ def assert_content_from_dom0_template(path, control_path=None): print(control) if os.path.isdir(path): result = filecmp.dircmp(path, control) - if result.diff_files or result.right_only: + if result.diff_files or result.right_only: # pragma: no cover print(result.report) raise RuntimeError("Missing or Differing files found in " + path) else: if not filecmp.cmp(path, control): - os.system("cat " + path) + os.system("cat " + path) # pragma: no cover raise RuntimeError(control) # Remove verified output files/directories. Untested files will remain and cause the testcase to FAIL: try: @@ -71,7 +71,7 @@ def assert_content_from_dom0_template(path, control_path=None): shutil.rmtree(path) -def extract(zip_or_tar_archive, archive_type): +def extract(zip_or_tar_archive, archive_type): # pragma: no cover """Extract a passed zip, tar or tar.bz2 archive into the current working directory""" if sys.version_info > (3, 0): if archive_type == "zip" and os.environ.get("GITHUB_ACTION"): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 213b8727..032a68f3 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -26,7 +26,7 @@ def imported_bugtool(testdir): def import_from_file(module_name, file_path): import sys - if sys.version_info.major == 2: + if sys.version_info.major == 2: # pragma: no cover import imp # pylint: disable=deprecated-module # pyright: ignore[reportMissingImports] return imp.load_source(module_name, file_path) diff --git a/tests/unit/test_xapidb_filter.py b/tests/unit/test_xapidb_filter.py index 25da665c..15ca2a37 100644 --- a/tests/unit/test_xapidb_filter.py +++ b/tests/unit/test_xapidb_filter.py @@ -80,5 +80,5 @@ def test_xapi_database_filter(bugtool): # Double-check with parseString(): Its output will differ between Py2/Py3 # though, so we will use it for one language version at a time: - if sys.version_info < (3, 0): + if sys.version_info < (3, 0): # pragma: no cover assert xml.dom.minidom.parseString(filtered).toprettyxml(indent=" ") == expected diff --git a/xen-bugtool b/xen-bugtool index b193813b..33720ec6 100755 --- a/xen-bugtool +++ b/xen-bugtool @@ -68,7 +68,7 @@ from xml.etree.ElementTree import Element import defusedxml.sax -if sys.version_info.major == 2: +if sys.version_info.major == 2: # pragma: no cover from commands import getoutput # pyright: ignore[reportMissingImports] from urllib import urlopen # type:ignore[attr-defined]