Add pytest-based integration test framework

Signed-off-by: Bernhard Kaindl <[email protected]>

.pylintrc

@@ -25,6 +25,7 @@ analyse-fallback-blocks=yes
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
 extension-pkg-allow-list=xen.lowlevel
+extension-pkg-whitelist=lxml.etree
 
 # Return non-zero exit code if any of these messages/categories are detected,
 # even if score is above --fail-under value. Syntax same as enable. Messages

README-pytest.md

@@ -0,0 +1,47 @@
+## Test suite based on `pytest` and Linux namespaces
+
+The initial shell-based tests were quick to create, but only meant as an early prototype:
+They need root capabilities through GitHub CI or `act` using a local `docker` or `podman` as backend container runtime to run.
+Such container runtime is not installed on some servers, so it was not easy to check them.
+
+Also, care needed to be taken to check their logs:
+There can be errors in such shell scripts which go unnoticed.
+And these can make the test pass even when the `bugtool` output was not properly verified.
+
+This `pytest` test framework which runs `xen-bugtool` with simulated `root` capabilities, can be run by any regular user and does not need special system configuration.
+
+### Installing packages to run the `pytest`-based test suite
+
+Verify to have `pip` installed in your `python2.7` environment.
+Install the libraries required by `xen-bugtool`:
+```sh
+python2 -m pip install --user -r requirements.txt
+```
+
+The test framework itself can use Python2 or Python3, depending on the installed Python builds, you can install one or both:
+
+```sh
+[ -e ~/.local/bin/pytest ] && mv ~/.local/bin/pytest ~/.local/bin/pytest-previous
+python2 -m pip install --user -r requirements-dev.txt
+mv ~/.local/bin/pytest ~/.local/bin/pytest-2
+python3 -m pip install --user -r requirements-dev.txt
+mv ~/.local/bin/pytest ~/.local/bin/pytest-3
+```
+
+### Running the `pytest`-based test suite
+
+On some older CentOS `~/.local/bin/pytest-2` may not work, use `pytest-3` on those.
+
+Until more of `xen-bugtool` supports Python3, you have to limit `pytest-3` execution to `tests/integration`:
+```py
+python3 -m pytest tests/integration -v -rF
+===================================== test session starts =====================================
+platform linux -- Python 3.6.8, pytest-7.0.1, pluggy-1.0.0 -- /usr/bin/python3
+cachedir: .pytest_cache
+rootdir: /your-workdir/status-report
+collected 1 item
+
+tests/integration/test_xenserver_config.py::test_bugtool_entries_xenserver_config PASSED [100%]
+
+====================================== 1 passed in 0.19s ======================================
+```

requirements-dev.txt

@@ -1 +1,2 @@
+lxml
 pytest

requirements.txt

@@ -1 +1,5 @@
-defusedxml==0.5.0
+# Initially, xen-bugtool was deployed in xs8 using defusedxml==0.5.0.
+# The common python-defusedxml.srpm has since been updated to the current
+# defusedxml-0.7.1.
+# Tests are now free to use the current version of defusedxml:
+defusedxml

tests/integration/conftest.py

@@ -0,0 +1,51 @@
+"""conftest.py: Test fixtures to test xen-bugtool using namespaces, supported on any Linux and GitHub CI"""
+from __future__ import print_function
+
+import os
+import shutil
+
+import pytest
+from distutils.dir_util import copy_tree
+from namespace_container import activate_private_test_namespace, mount, umount
+from utils import BUGTOOL_DOM0_TEMPL, run
+
+
+@pytest.fixture(autouse=True, scope="session")
+def check_test_tools_usability():
+    """At the start of the pytest session, check that the required tools are provided by the host OS"""
+    try:
+        run(["unzip", "-v"])
+    except OSError:
+        raise RuntimeError("Please install a working unzip tool on the host")
+
+
+@pytest.fixture(autouse=True, scope="session")
+def create_and_enter_test_environment():
+    """At the start of the pytest session, activate a namespace with bindmounts for testing xen-bugtool"""
+    activate_private_test_namespace(BUGTOOL_DOM0_TEMPL, ["/etc", "/opt", "/usr/sbin", "/usr/lib/systemd"])
+    os.environ["PYTHONPATH"] = "tests/mocks"
+
+
+@pytest.fixture(autouse=True, scope="function")
+def run_test_functions_with_private_tmpfs_output_directory():
+    """Run each test function with a private bugtool output directory using tmpfs"""
+    # Works in conjunction of having entered a private test namespace for the entire pytest session before:
+    mount(target="/var", fs="tmpfs", options="size=128M")
+    # To provide test files below /var, subdirectores can be bind-mounted/created here
+    # (or the tmpfs mount above could be done on BUGTOOL_OUTPUT_DIR)
+    # run_bugtool_entry() will chdir to the output directory, so change back afterwards:
+    srcdir = os.getcwd()
+    yield
+    # Assert that the test case did not leave any unchecked output fileas in the output directory:
+    remaining_files = []
+    for currentpath, _, files in os.walk("."):
+        for file in files:
+            remaining_files.append(os.path.join(currentpath, file))
+    if remaining_files:
+        print("Remaining (possibly unchecked) files found:")
+        print(remaining_files)
+        run(["find", "-type", "f"])
+        print("Ensure that these files are checked, remove them when checked.")
+        raise RuntimeError("Remaining (possibly unchecked) files found. Run 'pytest -rF' for logs")
+    os.chdir(srcdir)
+    umount("/var")

tests/integration/dom0-template/opt/xensource/bin/static-vdis

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo -n "$@"

tests/integration/dom0-template/usr/sbin/mdadm

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo -n "$@"

tests/integration/namespace_container.py

@@ -0,0 +1,71 @@
+"""namespace_container.py: Functions for creating a test environment container on any Linux and GitHub CI"""
+import ctypes
+import os
+from utils import run
+
+CLONE_NEWUSER = 0x10000000
+CLONE_NEWNET = 0x40000000
+CLONE_NEWNS = 0x00020000
+MS_BIND = 4096
+MS_REC = 16384
+MS_PRIVATE = 1 << 18
+
+
+def unshare(flags):
+    """Wrapper for the Linux libc/system call to unshare Linux kernel namespaces"""
+    libc = ctypes.CDLL(None, use_errno=True)
+    libc.unshare.argtypes = [ctypes.c_int]
+    rc = libc.unshare(flags)
+    if rc != 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno), flags)
+
+
+def mount(source="none", target="", fs="", flags=0, options=""):
+    """Wrapper for the Linux libc/system call to mount an fs, supports Python2.7 and Python3.x"""
+    libc = ctypes.CDLL(None, use_errno=True)
+    libc.mount.argtypes = (
+        ctypes.c_char_p,
+        ctypes.c_char_p,
+        ctypes.c_char_p,
+        ctypes.c_ulong,
+        ctypes.c_char_p,
+    )
+    print("mount -t " + fs + " -o '" + options + "' " + source + "\t" + target)
+    result = libc.mount(source.encode(), target.encode(), fs.encode(), flags, options.encode())
+    if result < 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, "mount " + target + " (options=" + options + "): " + os.strerror(errno))
+
+
+def umount(target):
+    """Wrapper for the Linux umount system call, supports Python2.7 and Python3.x"""
+    libc = ctypes.CDLL(None, use_errno=True)
+    result = libc.umount(ctypes.c_char_p(target.encode()))
+    if result < 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, "umount " + target + ": " + os.strerror(errno))
+
+
+def activate_private_test_namespace(bindmount_root, bindmount_mountpoints):
+    """Activate a new private mount, network and user namespace with the user behaving like uid 0 (root)"""
+    # Implements the sequence that `unshare -rmn <command>` uses. Namespace is active even without fork():
+    real_uid = os.getuid()
+    real_gid = os.getgid()
+    unshare(CLONE_NEWUSER | CLONE_NEWNET | CLONE_NEWNS)
+    # Setup uidmap for the user's uid to behave like uid 0 would (eg for bugtool's root user check)
+    with open("/proc/self/uid_map", "wb") as proc_self_uidmap:
+        proc_self_uidmap.write(b"0 %d 1" % real_uid)
+    # Setup setgroups behave like gid 0 would (needed for new tmpfs mounts for test output files):
+    with open("/proc/self/setgroups", "wb") as proc_self_setgroups:
+        proc_self_setgroups.write(b"deny")
+    # Setup gidmap for the user's gid to behave like gid 0 would (needed for tmpfs mounts):
+    with open("/proc/self/gid_map", "wb") as proc_self_gidmap:
+        proc_self_gidmap.write(b"0 %d 1" % real_gid)
+    # Prepare a private root mount in the new mount namespace, needed for mounting a private tmpfs on /var:
+    mount(target="/", flags=MS_REC | MS_PRIVATE)
+    # Bind-mount the Dom0 template directories in the private mount namespace to provide the test files:
+    for mountpoint in bindmount_mountpoints:
+        if not os.path.exists(mountpoint):
+            raise RuntimeError("Mountpoint missing on host! Please run: sudo mkdir " + mountpoint)
+        mount(source=bindmount_root + mountpoint, target=mountpoint, flags=MS_BIND)

tests/integration/test_xenserver_config.py

@@ -0,0 +1,17 @@
+"""tests/integration/test_xenserver_config.py: Test xen-bugtool --entries=xenserver-config"""
+import os
+
+from utils import check_cmd, check_file, run_bugtool_entry, verify_content_from_dom0_template
+
+
+def test_bugtool_entries_xenserver_config():
+    """Test xen-bugtool --entries=xenserver-config in test jail created by auto-fixtures in conftest.py"""
+    run_bugtool_entry("xenserver-config")
+    assert check_file("ls-lR-%opt%xensource.out").splitlines()[0] == "/opt/xensource:"
+    assert check_file("ls-lR-%etc%xensource%static-vdis.out") == ""
+    assert check_file("static-vdis-list.out") == "list"
+    os.chdir("..")
+    check_cmd(["tar", "xvf", "xenserver-config/etc/systemd.tar"], "xenserver-config/etc/systemd.tar")
+    os.chdir("xenserver-config")
+    verify_content_from_dom0_template("etc/xensource-inventory")
+    verify_content_from_dom0_template("etc/systemd")

tests/integration/utils.py

@@ -0,0 +1,74 @@
+"""tests/integration/utils.py: utility functions to test xen-bugtool by invoking it"""
+from __future__ import print_function
+
+import filecmp
+import os
+import shutil
+import sys
+from subprocess import PIPE, Popen
+
+from lxml import etree
+
+if sys.version_info.major == 2:
+    from commands import getoutput  # pyright: ignore[reportMissingImports]
+else:
+    from subprocess import getoutput
+
+BUGTOOL_OUTPUT_DIR = "/var/opt/xen/bug-report/"
+BUGTOOL_DOM0_TEMPL = "tests/integration/dom0-template"
+
+
+def run(command):
+    """Run the given shell command, print it's stdout and stderr and return them"""
+    process = Popen(command, stdout=PIPE, stderr=PIPE, universal_newlines=True)
+    stdout, stderr = process.communicate()
+    returncode = process.wait()
+    print("# " + " ".join(command) + ":")
+    if returncode:
+        raise Exception(returncode, stderr)
+    print(stdout, stderr)
+    return stdout, stderr
+
+
+def check_cmd(cmd, remove_file):
+    """Run the given command, print its stdout and stderr and return them. Remove the given remove_file"""
+    stdout, stderr = run(cmd)
+    # After successfuly verficiation of the files, remove the checked output file(missed files remain):
+    os.unlink(remove_file)
+    return stdout, stderr
+
+
+def check_file(path):
+    """Return the contents of the passed bugtool output file for verification"""
+    with open(path) as handle:
+        contents = handle.read()
+    # After successfuly verficiation of the files, remove the checked output file (missed files remain):
+    os.unlink(path)
+    return contents
+
+
+def verify_content_from_dom0_template(path):
+    """Compare the contents of output directories or files with the test's Dom0 template directories"""
+    assert path[0] != "/"
+    assert filecmp.dircmp(path, BUGTOOL_DOM0_TEMPL + path)
+    # After successfuly verficiation of the files, remove the checked output files (missed files remain):
+    try:
+        os.unlink(path)
+    except OSError:
+        shutil.rmtree(path)
+
+
+def run_bugtool_entry(test_entry):
+    """Run bugtool for the given entry or entries, extract the output, and chdir to it"""
+    os.environ["XENRT_BUGTOOL_BASENAME"] = test_entry
+    # For case the default python interpreter of the user is python3, we must use python2(for now):
+    print(getoutput("python2 ./xen-bugtool -y --entries=xenserver-config --output=zip"))
+    srcdir = os.getcwd()
+    os.chdir(BUGTOOL_OUTPUT_DIR)
+    check_cmd(["unzip", test_entry + ".zip"], test_entry + ".zip")
+    os.chdir(test_entry)
+    # Validate the extracted inventory.xml using the XML schema from the test framework:
+    with open(srcdir + "/tests/integration/inventory.xsd") as xmlschema:
+        etree.XMLSchema(etree.parse(xmlschema)).assertValid(etree.parse("inventory.xml"))
+        # After successfuly validation of the inventory.xml, remove it (not removed files make the test fail):
+        os.unlink("inventory.xml")