diff --git a/.pylintrc b/.pylintrc index 06096bbc..fb86860f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,6 +25,7 @@ analyse-fallback-blocks=yes # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. extension-pkg-allow-list=xen.lowlevel +extension-pkg-whitelist=lxml.etree # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages diff --git a/README-pytest.md b/README-pytest.md new file mode 100644 index 00000000..17f84491 --- /dev/null +++ b/README-pytest.md @@ -0,0 +1,47 @@ +## Test suite based on `pytest` and Linux namespaces + +The initial shell-based tests were quick to create, but only meant as an early prototype: +They need root capabilities through GitHub CI or `act` using a local `docker` or `podman` as backend container runtime to run. +Such container runtime is not installed on some servers, so it was not easy to check them. + +Also, care needed to be taken to check their logs: +There can be errors in such shell scripts which go unnoticed. +And these can make the test pass even when the `bugtool` output was not properly verified. + +This `pytest` test framework which runs `xen-bugtool` with simulated `root` capabilities, can be run by any regular user and does not need special system configuration. + +### Installing packages to run the `pytest`-based test suite + +Verify to have `pip` installed in your `python2.7` environment. +Install the libraries required by `xen-bugtool`: +```sh +python2 -m pip install --user -r requirements.txt +``` + +The test framework itself can use Python2 or Python3, depending on the installed Python builds, you can install one or both: + +```sh +[ -e ~/.local/bin/pytest ] && mv ~/.local/bin/pytest ~/.local/bin/pytest-previous +python2 -m pip install --user -r requirements-dev.txt +mv ~/.local/bin/pytest ~/.local/bin/pytest-2 +python3 -m pip install --user -r requirements-dev.txt +mv ~/.local/bin/pytest ~/.local/bin/pytest-3 +``` + +### Running the `pytest`-based test suite + +On some older CentOS `~/.local/bin/pytest-2` may not work, use `pytest-3` on those. + +Until more of `xen-bugtool` supports Python3, you have to limit `pytest-3` execution to `tests/integration`: +```py +python3 -m pytest tests/integration -v -rF +===================================== test session starts ===================================== +platform linux -- Python 3.6.8, pytest-7.0.1, pluggy-1.0.0 -- /usr/bin/python3 +cachedir: .pytest_cache +rootdir: /your-workdir/status-report +collected 1 item + +tests/integration/test_xenserver_config.py::test_bugtool_entries_xenserver_config PASSED [100%] + +====================================== 1 passed in 0.19s ====================================== +``` diff --git a/requirements-dev.txt b/requirements-dev.txt index e079f8a6..cb14c82a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ +lxml pytest diff --git a/requirements.txt b/requirements.txt index 8dafddee..aecf408d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ -defusedxml==0.5.0 +# Initially, xen-bugtool was deployed in xs8 using defusedxml==0.5.0. +# The common python-defusedxml.srpm has since been updated to the current +# defusedxml-0.7.1. +# Tests are now free to use the current version of defusedxml: +defusedxml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..4d3af5b2 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,51 @@ +"""conftest.py: Test fixtures to test xen-bugtool using namespaces, supported on any Linux and GitHub CI""" +from __future__ import print_function + +import os +import shutil + +import pytest +from distutils.dir_util import copy_tree +from namespace_container import activate_private_test_namespace, mount, umount +from utils import BUGTOOL_DOM0_TEMPL, run + + +@pytest.fixture(autouse=True, scope="session") +def check_test_tools_usability(): + """At the start of the pytest session, check that the required tools are provided by the host OS""" + try: + run(["unzip", "-v"]) + except OSError: + raise RuntimeError("Please install a working unzip tool on the host") + + +@pytest.fixture(autouse=True, scope="session") +def create_and_enter_test_environment(): + """At the start of the pytest session, activate a namespace with bindmounts for testing xen-bugtool""" + activate_private_test_namespace(BUGTOOL_DOM0_TEMPL, ["/etc", "/opt", "/usr/sbin", "/usr/lib/systemd"]) + os.environ["PYTHONPATH"] = "tests/mocks" + + +@pytest.fixture(autouse=True, scope="function") +def run_test_functions_with_private_tmpfs_output_directory(): + """Run each test function with a private bugtool output directory using tmpfs""" + # Works in conjunction of having entered a private test namespace for the entire pytest session before: + mount(target="/var", fs="tmpfs", options="size=128M") + # To provide test files below /var, subdirectores can be bind-mounted/created here + # (or the tmpfs mount above could be done on BUGTOOL_OUTPUT_DIR) + # run_bugtool_entry() will chdir to the output directory, so change back afterwards: + srcdir = os.getcwd() + yield + # Assert that the test case did not leave any unchecked output fileas in the output directory: + remaining_files = [] + for currentpath, _, files in os.walk("."): + for file in files: + remaining_files.append(os.path.join(currentpath, file)) + if remaining_files: + print("Remaining (possibly unchecked) files found:") + print(remaining_files) + run(["find", "-type", "f"]) + print("Ensure that these files are checked, remove them when checked.") + raise RuntimeError("Remaining (possibly unchecked) files found. Run 'pytest -rF' for logs") + os.chdir(srcdir) + umount("/var") diff --git a/tests/integration/dom0-template/opt/xensource/bin/static-vdis b/tests/integration/dom0-template/opt/xensource/bin/static-vdis new file mode 100755 index 00000000..c86cdc29 --- /dev/null +++ b/tests/integration/dom0-template/opt/xensource/bin/static-vdis @@ -0,0 +1,2 @@ +#!/bin/sh +echo -n "$@" diff --git a/tests/integration/dom0-template/usr/sbin/mdadm b/tests/integration/dom0-template/usr/sbin/mdadm new file mode 100755 index 00000000..c86cdc29 --- /dev/null +++ b/tests/integration/dom0-template/usr/sbin/mdadm @@ -0,0 +1,2 @@ +#!/bin/sh +echo -n "$@" diff --git a/tests/integration/namespace_container.py b/tests/integration/namespace_container.py new file mode 100644 index 00000000..4500f879 --- /dev/null +++ b/tests/integration/namespace_container.py @@ -0,0 +1,71 @@ +"""namespace_container.py: Functions for creating a test environment container on any Linux and GitHub CI""" +import ctypes +import os +from utils import run + +CLONE_NEWUSER = 0x10000000 +CLONE_NEWNET = 0x40000000 +CLONE_NEWNS = 0x00020000 +MS_BIND = 4096 +MS_REC = 16384 +MS_PRIVATE = 1 << 18 + + +def unshare(flags): + """Wrapper for the Linux libc/system call to unshare Linux kernel namespaces""" + libc = ctypes.CDLL(None, use_errno=True) + libc.unshare.argtypes = [ctypes.c_int] + rc = libc.unshare(flags) + if rc != 0: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno), flags) + + +def mount(source="none", target="", fs="", flags=0, options=""): + """Wrapper for the Linux libc/system call to mount an fs, supports Python2.7 and Python3.x""" + libc = ctypes.CDLL(None, use_errno=True) + libc.mount.argtypes = ( + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_ulong, + ctypes.c_char_p, + ) + print("mount -t " + fs + " -o '" + options + "' " + source + "\t" + target) + result = libc.mount(source.encode(), target.encode(), fs.encode(), flags, options.encode()) + if result < 0: + errno = ctypes.get_errno() + raise OSError(errno, "mount " + target + " (options=" + options + "): " + os.strerror(errno)) + + +def umount(target): + """Wrapper for the Linux umount system call, supports Python2.7 and Python3.x""" + libc = ctypes.CDLL(None, use_errno=True) + result = libc.umount(ctypes.c_char_p(target.encode())) + if result < 0: + errno = ctypes.get_errno() + raise OSError(errno, "umount " + target + ": " + os.strerror(errno)) + + +def activate_private_test_namespace(bindmount_root, bindmount_mountpoints): + """Activate a new private mount, network and user namespace with the user behaving like uid 0 (root)""" + # Implements the sequence that `unshare -rmn ` uses. Namespace is active even without fork(): + real_uid = os.getuid() + real_gid = os.getgid() + unshare(CLONE_NEWUSER | CLONE_NEWNET | CLONE_NEWNS) + # Setup uidmap for the user's uid to behave like uid 0 would (eg for bugtool's root user check) + with open("/proc/self/uid_map", "wb") as proc_self_uidmap: + proc_self_uidmap.write(b"0 %d 1" % real_uid) + # Setup setgroups behave like gid 0 would (needed for new tmpfs mounts for test output files): + with open("/proc/self/setgroups", "wb") as proc_self_setgroups: + proc_self_setgroups.write(b"deny") + # Setup gidmap for the user's gid to behave like gid 0 would (needed for tmpfs mounts): + with open("/proc/self/gid_map", "wb") as proc_self_gidmap: + proc_self_gidmap.write(b"0 %d 1" % real_gid) + # Prepare a private root mount in the new mount namespace, needed for mounting a private tmpfs on /var: + mount(target="/", flags=MS_REC | MS_PRIVATE) + # Bind-mount the Dom0 template directories in the private mount namespace to provide the test files: + for mountpoint in bindmount_mountpoints: + if not os.path.exists(mountpoint): + raise RuntimeError("Mountpoint missing on host! Please run: sudo mkdir " + mountpoint) + mount(source=bindmount_root + mountpoint, target=mountpoint, flags=MS_BIND) diff --git a/tests/integration/test_xenserver_config.py b/tests/integration/test_xenserver_config.py new file mode 100644 index 00000000..fcf0cdfc --- /dev/null +++ b/tests/integration/test_xenserver_config.py @@ -0,0 +1,17 @@ +"""tests/integration/test_xenserver_config.py: Test xen-bugtool --entries=xenserver-config""" +import os + +from utils import check_cmd, check_file, run_bugtool_entry, verify_content_from_dom0_template + + +def test_bugtool_entries_xenserver_config(): + """Test xen-bugtool --entries=xenserver-config in test jail created by auto-fixtures in conftest.py""" + run_bugtool_entry("xenserver-config") + assert check_file("ls-lR-%opt%xensource.out").splitlines()[0] == "/opt/xensource:" + assert check_file("ls-lR-%etc%xensource%static-vdis.out") == "" + assert check_file("static-vdis-list.out") == "list" + os.chdir("..") + check_cmd(["tar", "xvf", "xenserver-config/etc/systemd.tar"], "xenserver-config/etc/systemd.tar") + os.chdir("xenserver-config") + verify_content_from_dom0_template("etc/xensource-inventory") + verify_content_from_dom0_template("etc/systemd") diff --git a/tests/integration/utils.py b/tests/integration/utils.py new file mode 100644 index 00000000..b949872e --- /dev/null +++ b/tests/integration/utils.py @@ -0,0 +1,74 @@ +"""tests/integration/utils.py: utility functions to test xen-bugtool by invoking it""" +from __future__ import print_function + +import filecmp +import os +import shutil +import sys +from subprocess import PIPE, Popen + +from lxml import etree + +if sys.version_info.major == 2: + from commands import getoutput # pyright: ignore[reportMissingImports] +else: + from subprocess import getoutput + +BUGTOOL_OUTPUT_DIR = "/var/opt/xen/bug-report/" +BUGTOOL_DOM0_TEMPL = "tests/integration/dom0-template" + + +def run(command): + """Run the given shell command, print it's stdout and stderr and return them""" + process = Popen(command, stdout=PIPE, stderr=PIPE, universal_newlines=True) + stdout, stderr = process.communicate() + returncode = process.wait() + print("# " + " ".join(command) + ":") + if returncode: + raise Exception(returncode, stderr) + print(stdout, stderr) + return stdout, stderr + + +def check_cmd(cmd, remove_file): + """Run the given command, print its stdout and stderr and return them. Remove the given remove_file""" + stdout, stderr = run(cmd) + # After successfuly verficiation of the files, remove the checked output file(missed files remain): + os.unlink(remove_file) + return stdout, stderr + + +def check_file(path): + """Return the contents of the passed bugtool output file for verification""" + with open(path) as handle: + contents = handle.read() + # After successfuly verficiation of the files, remove the checked output file (missed files remain): + os.unlink(path) + return contents + + +def verify_content_from_dom0_template(path): + """Compare the contents of output directories or files with the test's Dom0 template directories""" + assert path[0] != "/" + assert filecmp.dircmp(path, BUGTOOL_DOM0_TEMPL + path) + # After successfuly verficiation of the files, remove the checked output files (missed files remain): + try: + os.unlink(path) + except OSError: + shutil.rmtree(path) + + +def run_bugtool_entry(test_entry): + """Run bugtool for the given entry or entries, extract the output, and chdir to it""" + os.environ["XENRT_BUGTOOL_BASENAME"] = test_entry + # For case the default python interpreter of the user is python3, we must use python2(for now): + print(getoutput("python2 ./xen-bugtool -y --entries=xenserver-config --output=zip")) + srcdir = os.getcwd() + os.chdir(BUGTOOL_OUTPUT_DIR) + check_cmd(["unzip", test_entry + ".zip"], test_entry + ".zip") + os.chdir(test_entry) + # Validate the extracted inventory.xml using the XML schema from the test framework: + with open(srcdir + "/tests/integration/inventory.xsd") as xmlschema: + etree.XMLSchema(etree.parse(xmlschema)).assertValid(etree.parse("inventory.xml")) + # After successfuly validation of the inventory.xml, remove it (not removed files make the test fail): + os.unlink("inventory.xml")