CI: Improve unit tests to test the bugtool's main() as well

Add tests/unit/test_main.py which also uses test functions which were added for earlier tests to test the main function of bugtool to produce the bugtool output archives for all types: zip, tar, and tar.bz2. The main test checks the output of the bugtool application to ensure that it matches the expected output. It compares the captured output with the expected output and performs various assertions to validate the output. It extracts the output files from the archive and checks that the xAPI db and the inventory.xml. Tested output: - The start and end of the collected bugtool messages from captured.out - The inventory.xml file is checked to be validated using its XML schema. - The xapi-db.xml is checked to have secrets filtered using a dummy xapi db Signed-off-by: Bernhard Kaindl <[email protected]>
xenserver · Jan 11, 2024 · 46e8b1f · 46e8b1f
1 parent 1aae840
commit 46e8b1f
Show file tree

Hide file tree

Showing 7 changed files with 337 additions and 53 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
         name: 'check and fix files to have no trailing whitespace'
     -   id: end-of-file-fixer
         name: 'check and fix that files have a trailing newline'
-        exclude: tests/integration/sar-file-collection.test.sh
+        exclude: tests/integration/dom0-template
     -   id: mixed-line-ending
         args: ['--fix=lf']
         name: 'check and fix that line endings are line feeds'
@@ -58,45 +58,10 @@ repos:
     -   id: check-merge-conflict
     -   id: check-yaml
         name: 'check the syntax of yaml files'
-# Run "python3 -m pip install -r requirements-dev.txt" to run pytest or use "git commit --no-verify":
--   repo: local
-    hooks:
-    -   id: pytest
-        name: check that the Xen-Bugtool Test Environment passes
-        entry: env PYTHONDEVMODE=yes python3 -m pytest tests/unit
-            --cov xen-bugtool
-            --cov tests/unit
-            --junitxml=.git/pytest.xml
-            --cov-report term-missing
-            --cov-report html:.git/coverage.html
-            --cov-report xml:.git/coverage.xml
-        require_serial: true
-        pass_filenames: false
-        language: python
-        types: [python]
-        additional_dependencies: [coverage, defusedxml, pytest-coverage, lxml, XenAPI]
-
-
-    -   id: diff-cover
-        name: check that that the changed lines are covered by tests
-        entry: diff-cover --ignore-whitespace --compare-branch=origin/master
-            --show-uncovered --html-report .git/coverage-diff.html
-            --fail-under 100 .git/coverage.xml
-        require_serial: true
-        pass_filenames: false
-        language: python
-        types: [python]
-        additional_dependencies: [diff-cover]
 
+# The fast checks are first, to fix them first,
+# pytest with code coverage at the end.
 
-    -   id: coverage-report
-        name: check coverage report for minimum overall coverage
-        entry: coverage report --fail-under 55 #| tee .git/coverage.txt
-        require_serial: true
-        pass_filenames: false
-        language: python
-        types: [python]
-        additional_dependencies: [coverage]
 -   repo: https://github.com/PyCQA/autoflake
     rev: v2.2.1
     hooks:
@@ -105,13 +70,17 @@ repos:
         args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"]
         language: python
         files: \.py$
+
+
 -   repo: https://github.com/psf/black
     rev: 23.11.0
     hooks:
     -   id: black
         name: Ensure that all files (excluding xen-bugtool itself) are black-formatted
         args: [--safe, --quiet]
         exclude: xen-bugtool
+
+
 -   repo: https://github.com/akaihola/darker
     rev: 1.7.2
     hooks:
@@ -123,20 +92,73 @@ repos:
         args: [darker-xen-bugtool]
         additional_dependencies:
           - isort
+
+
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
     -   id: mypy
         name: Run mypy to check e.g. that all expected arguments are passed to functions etc
         additional_dependencies: [defusedxml, pytest, types-lxml]
+
+
 -   repo: https://github.com/pycqa/pylint
-    rev: v3.0.2
+    rev: v3.0.3
     hooks:
     -   id: pylint
         name: Run pylint to check that docstrings are added (and all other enabled checks)
         log_file: ".git/pre-commit-pylint.log"
+
+
+# Run "python3 -m pip install -r requirements-dev.txt" to run pytest or use "git commit --no-verify":
+-   repo: local
+    hooks:
+    -   id: pytest
+        name: check that the Xen-Bugtool Test Environment passes
+        entry: env PYTHONDEVMODE=yes python3 -m pytest tests/unit
+            --cov xen-bugtool
+            --cov tests/unit
+            --junitxml=.git/pytest.xml
+            --cov-report term-missing
+            --cov-report html:.git/coverage.html
+            --cov-report xml:.git/coverage.xml
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies:
+        - coverage
+        - defusedxml
+        - pytest-coverage
+        - pytest-mock
+        - lxml
+        - XenAPI
+
+
+    -   id: diff-cover
+        name: check that that the changed lines are covered by tests
+        entry: diff-cover --ignore-whitespace --compare-branch=origin/master
+            --show-uncovered --html-report .git/coverage-diff.html
+            --fail-under 100 .git/coverage.xml
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [diff-cover]
+
+
+    -   id: coverage-report
+        name: check coverage report for minimum overall coverage
+        entry: coverage report --fail-under 55 #| tee .git/coverage.txt
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [coverage]
+
+
 -   repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.339
+    rev: v1.1.345
     hooks:
     -   id: pyright
         name: Run pyright to check the unit tests for any typing warnings (use for bugtool later)

diff --git a/.pylintrc b/.pylintrc
@@ -161,6 +161,7 @@ disable=anomalous-backslash-in-string,
         invalid-name,
         invalid-name,
         len-as-condition,
+        missing-type-doc,  # better do typing than type in the param docstring
         no-else-break,
         no-else-return,
         no-name-in-module,

diff --git a/tests/integration/sar-file-collection.test.sh b/tests/integration/sar-file-collection.test.sh
@@ -74,4 +74,4 @@ popd
 # Check that the tar and zip outputs are identical (except for date and uptime):
 sed -i "s/\\<$tar_basename\\>/$zip_basename/" tar/inventory.xml
 sed -i 's/date="[^"]*"//;s/uptime="[^"]*"//' {tar,zip}/inventory.xml
-diff -rNu tar zip
+diff -rNu tar zip
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
@@ -1,5 +1,6 @@
 """tests/unit/conftest.py: pytest fixtures for unit-testing functions in the xen-bugtool script"""
 import os
+import sys
 
 import pytest
 
@@ -24,8 +25,6 @@ def imported_bugtool(testdir):
 
     # This uses the deprecated imp module because it needs also to run with Python2.7 for now:
     def import_from_file(module_name, file_path):
-        import sys
-
         if sys.version_info.major == 2:  # pragma: no cover
             import imp  # pylint: disable=deprecated-module  # pyright: ignore[reportMissingImports]
 
@@ -47,6 +46,8 @@ def import_from_file(module_name, file_path):
 
     bugtool = import_from_file("xen-bugtool", testdir + "/../../xen-bugtool")
     bugtool.ProcOutput.debug = True
+    # Prepend tests/mocks to force the use of our mock xen.lowlevel.xc module:
+    sys.path.insert(0, testdir + "/../mocks")
     return bugtool
 
 
@@ -55,4 +56,5 @@ def bugtool(imported_bugtool):
     """Test fixture for unit tests, initializes the bugtool data dict for each test"""
     # Init import_bugtool.data, so each unit test function gets it pristine:
     imported_bugtool.data = {}
+    sys.argv = ["xen-bugtool"]
     return imported_bugtool