diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29f7b738..553b4d9c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ env: PYTHONDEVMODE: yes jobs: python-checks: - name: Python checks + name: Run the Xen-Bugtool Test Environment runs-on: ubuntu-20.04 steps: - name: Checkout code @@ -32,5 +32,5 @@ jobs: - name: Run pylint-1.9.4 for pylint --py3k linting (configured in .pylintrc) run: python2 -m pylint xen-bugtool - - name: Run python2 -m pytest to execute unit tests - run: python2 -m pytest + - name: Run python2 -m pytest to execute all unit and integration tests + run: python2 -m pytest -v -rA diff --git a/.pylintrc b/.pylintrc index 06096bbc..fb86860f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,6 +25,7 @@ analyse-fallback-blocks=yes # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. extension-pkg-allow-list=xen.lowlevel +extension-pkg-whitelist=lxml.etree # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages diff --git a/README-pytest.md b/README-pytest.md new file mode 100644 index 00000000..436feb54 --- /dev/null +++ b/README-pytest.md @@ -0,0 +1,84 @@ +# Guide to Setting Up and Running the Xen-Bugtool Test Environment + +This document provides detailed information about the test environment for `xen-bugtool`, +a tool designed to assist with debugging XenServer issues. +The test environment runs `xen-bugtool` in a container with simulated `root` capabilities, +allowing it to be run by any regular user without requiring special system configuration. + +This document guides you through the setup of the container and the execution of each test case. + +## Lessons Learned from the Initial Prototype + +The initial shell-based tests were quick to create, but were only meant as an early prototype. +These tests required an external environment with root privileges (preferably a container) to run. +Additionally, failed checks because of mismatched file paths could go unnoticed, +leading to tests passing even when the `bugtool` output was not properly verified. + +## Installing Packages to Run the Xen-Bugtool Test Environment + +Ensure that `pip` is installed in your `python2.7` environment. Same for Python3, but it's optional. + +Install the libraries required by `xen-bugtool` with the following command: + +```sh +# Install the libraries required to run `xen-bugtool` itself into your environment: +python2 -m pip install --user -r requirements.txt +``` + +The test framework itself can use Python2 or Python3. +Depending on the installed builds, you can install one or both: + +```sh +# Install pytest and its depdendencies into your environment: +python2 -m pip install --user -r requirements-dev.txt # and/or: +python3 -m pip install --user -r requirements-dev.txt +``` + +## Running the Xen-Bugtool Test Environment + +```py +# Run the Xen-Bugtool Test Environment +python3 -m pytest tests/integration +``` +Hint: `python2 -m pytest` could be used too, but it may run into problems on CentOS 8.0. + +## Implementation of the Xen-Bugtool Test Environment in Pytest + +This section provides detailed information about the test environment for `xen-bugtool`. +It explains how the container is set up and how each test case is executed. + +The test environment is implemented by automatic `pytest` fixtures creating a container for the `pytest` process session and its sub-processes. +A fresh instance of a clean `bugtool` output directory (located at `/var/opt/xen/bug-report`) +is provided for invocation of the test functions. + +## Container Setup + +The container is implemented in [tests/integration/namespace_container.py](tests/integration/namespace_container.py). + +`xen-bugtool` requires to be run as root and checks it and writes its output to `/var/opt/xen/bug-report`. +The container setup provides these root privileges and a new mount namespace to bind-mount the test files. +It allows to bind-mount the directory trees with test files at the places where `xen-bugtool` collects them. +The mount namespace also allows to create a `tmpfs` mount at `/var/opt/xen/bug-report` for the output data. +It avoids the need to create a temporary directory and to bind-mount it into the container. +Additionally, it ensures that no temporary data can accumulate on the host. + +It implements the equivalent of `/usr/bin/unshare --map-root-user --mount --net` and mounts test directories: + +- `--map-root-user`: Creates a Linux user namespace where the effective user and group IDs of the current user are mapped to the superuser UID and GID. +- `--mount`: Creates a Linux mount namespace for mounting temporary file systems and bind-mounting test data. +- `--net`: Creates a Linux network namespace to ensure that `bugtool` works without outside connectivity. + +The container then bind-mounts the directory trees from [tests/integration/dom0-template](tests/integration/dom0-template) into the container. +This means the object under test (`xen-bugtool`) runs in an isolated test environment. + +## Test Case Execution + +For each test case and each output format, the test fixture and the test case code perform the following steps: + +1. The fixture mounts a `tmpfs` directory for `bugtool` to use for its output at the static path it uses. +2. The fixture yields to the test case implementation for running the individual test. +3. The test case code implementation runs `xen-bugtool` with the arguments required by the test. +4. The function to run it checks that the `inventory.xml` in the output archive conforms to the XML schema. +5. The test case checks all output files and removes all files that it successfully verified. +6. The fixture then asserts that the test case has not left any remaining unexpected files in the output tree. +7. The fixture then unmounts the temporary output directory, preparing for the next test case to start afresh. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..6197705e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,10 @@ +[pytest] +# By default, show reports for failed tests: +addopts=-rF +# imp is only used in a unit test, will be updated to python3 later: +filterwarnings=ignore:the imp module is deprecated +# Enable live logging of the python log output, starting with log level INFO by default: +log_cli=True +log_cli_level=INFO +# By default, run the tests in the tests directory: +testpaths=tests/ diff --git a/requirements-dev.txt b/requirements-dev.txt index e079f8a6..cb14c82a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ +lxml pytest diff --git a/requirements.txt b/requirements.txt index 8dafddee..aecf408d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ -defusedxml==0.5.0 +# Initially, xen-bugtool was deployed in xs8 using defusedxml==0.5.0. +# The common python-defusedxml.srpm has since been updated to the current +# defusedxml-0.7.1. +# Tests are now free to use the current version of defusedxml: +defusedxml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..2dc37464 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,48 @@ +"""conftest.py: Test fixtures to test xen-bugtool using namespaces, supported on any Linux and GitHub CI""" +from __future__ import print_function + +import os + +import pytest + +from namespace_container import activate_private_test_namespace, mount, umount +from utils import BUGTOOL_DOM0_TEMPL, run + + +@pytest.fixture(autouse=True, scope="session") +def create_and_enter_test_environment(): + """At the start of the pytest session, activate a namespace with bindmounts for testing xen-bugtool""" + activate_private_test_namespace(BUGTOOL_DOM0_TEMPL, ["/etc", "/opt", "/usr/sbin", "/usr/lib/systemd"]) + os.environ["PYTHONPATH"] = "tests/mocks" + + +# zip, tar, tar.bz2 are the three output formats suppored by xen_bugtool: +@pytest.fixture(scope="function", params=("zip", "tar", "tar.bz2")) +def output_archive_type(request): + """Parameterized fixture which causes the tests to run for each of the three output_archive_types""" + return request.param + + +@pytest.fixture(autouse=True, scope="function") +def run_test_functions_with_private_tmpfs_output_directory(): + """Run each test function with a private bugtool output directory using tmpfs""" + # Works in conjunction of having entered a private test namespace for the entire pytest session before: + mount(target="/var", fs="tmpfs", options="size=128M") + # To provide test files below /var, subdirectores can be bind-mounted/created here + # (or the tmpfs mount above could be done on BUGTOOL_OUTPUT_DIR) + # run_bugtool_entry() will chdir to the output directory, so change back afterwards: + srcdir = os.getcwd() + yield + # Assert that the test case did not leave any unchecked output fileas in the output directory: + remaining_files = [] + for currentpath, _, files in os.walk("."): + for file in files: + remaining_files.append(os.path.join(currentpath, file)) + if remaining_files: + print("Remaining (possibly unchecked) files found:") + print(remaining_files) + run(["find", "-type", "f"]) + print("Ensure that these files are checked, remove them when checked.") + raise RuntimeError("Remaining (possibly unchecked) files found. Run 'pytest -rF' for logs") + os.chdir(srcdir) + umount("/var") diff --git a/tests/integration/dom0-template/opt/xensource/bin/static-vdis b/tests/integration/dom0-template/opt/xensource/bin/static-vdis new file mode 100755 index 00000000..c86cdc29 --- /dev/null +++ b/tests/integration/dom0-template/opt/xensource/bin/static-vdis @@ -0,0 +1,2 @@ +#!/bin/sh +echo -n "$@" diff --git a/tests/integration/dom0-template/usr/sbin/mdadm b/tests/integration/dom0-template/usr/sbin/mdadm new file mode 100755 index 00000000..c86cdc29 --- /dev/null +++ b/tests/integration/dom0-template/usr/sbin/mdadm @@ -0,0 +1,2 @@ +#!/bin/sh +echo -n "$@" diff --git a/tests/integration/namespace_container.py b/tests/integration/namespace_container.py new file mode 100644 index 00000000..162a8aab --- /dev/null +++ b/tests/integration/namespace_container.py @@ -0,0 +1,87 @@ +"""namespace_container.py: Functions for creating a test environment container on any Linux and GitHub CI""" +import ctypes +import os + +from utils import run + +CLONE_NEWUSER = 0x10000000 +CLONE_NEWNET = 0x40000000 +CLONE_NEWNS = 0x00020000 +MS_BIND = 4096 +MS_REC = 16384 +MS_PRIVATE = 1 << 18 + + +def unshare(flags): + """Wrapper for the Linux libc/system call to unshare Linux kernel namespaces""" + libc = ctypes.CDLL(None, use_errno=True) + libc.unshare.argtypes = [ctypes.c_int] + rc = libc.unshare(flags) + if rc != 0: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno), flags) + + +def mount(source="none", target="", fs="", flags=0, options=""): + """Wrapper for the Linux libc/system call to mount an fs, supports Python2.7 and Python3.x""" + libc = ctypes.CDLL(None, use_errno=True) + libc.mount.argtypes = ( + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_ulong, + ctypes.c_char_p, + ) + print("mount -t " + fs + " -o '" + options + "' " + source + "\t" + target) + result = libc.mount(source.encode(), target.encode(), fs.encode(), flags, options.encode()) + if result < 0: + errno = ctypes.get_errno() + raise OSError(errno, "mount " + target + " (options=" + options + "): " + os.strerror(errno)) + + +def umount(target): + """Wrapper for the Linux umount system call, supports Python2.7 and Python3.x""" + libc = ctypes.CDLL(None, use_errno=True) + result = libc.umount(ctypes.c_char_p(target.encode())) + if result < 0: + errno = ctypes.get_errno() + raise OSError(errno, "umount " + target + ": " + os.strerror(errno)) + + +def activate_private_test_namespace(bindmount_root, bindmount_mountpoints): + """Activate a new private mount, network and user namespace with the user behaving like uid 0 (root) + + xen-bugtool requires to be run as root and checks it and writes its output to /var/opt/xen/bug-report. + The container setup provides these root privileges and a new mount namespace to bind-mount the test files. + It allows to bind-mount the directory trees with test files at the places where xen-bugtool collects them. + The mount namespace also allows to create a tmpfs mount at /var/opt/xen/bug-report for the output data. + It avoids the need to create a temporary directory and to bind-mount it into the container. + Additionally, it ensures that no temporary data can accumulate on the host. + + This function implements the equivalent of `/usr/bin/unshare --map-root-user --mount --net` + and mounts the passed bindmount_mountpoints below bindmount_root: + + - --map-root-user: Create a user namespace where euid/egid are mapped to the superuser UID and GID. + - --mount: Create a Linux mount namespace for mounting temporary file systems and bind-mounting test data. + - --net: Create a Linux network namespace to ensure that `bugtool` works without outside connectivity. + """ + # Implements the sequence that `unshare -rmn ` uses. Namespace is active even without fork(): + real_uid = os.getuid() + real_gid = os.getgid() + unshare(CLONE_NEWUSER | CLONE_NEWNET | CLONE_NEWNS) + # Setup uidmap for the user's uid to behave like uid 0 would (eg for bugtool's root user check) + with open("/proc/self/uid_map", "wb") as proc_self_uidmap: + proc_self_uidmap.write(b"0 %d 1" % real_uid) + # Setup setgroups behave like gid 0 would (needed for new tmpfs mounts for test output files): + with open("/proc/self/setgroups", "wb") as proc_self_setgroups: + proc_self_setgroups.write(b"deny") + # Setup gidmap for the user's gid to behave like gid 0 would (needed for tmpfs mounts): + with open("/proc/self/gid_map", "wb") as proc_self_gidmap: + proc_self_gidmap.write(b"0 %d 1" % real_gid) + # Prepare a private root mount in the new mount namespace, needed for mounting a private tmpfs on /var: + mount(target="/", flags=MS_REC | MS_PRIVATE) + # Bind-mount the Dom0 template directories in the private mount namespace to provide the test files: + for mountpoint in bindmount_mountpoints: + if not os.path.exists(mountpoint): + raise RuntimeError("Mountpoint missing on host! Please run: sudo mkdir " + mountpoint) + mount(source=bindmount_root + mountpoint, target=mountpoint, flags=MS_BIND) diff --git a/tests/integration/test_xenserver_config.py b/tests/integration/test_xenserver_config.py new file mode 100644 index 00000000..1e396f28 --- /dev/null +++ b/tests/integration/test_xenserver_config.py @@ -0,0 +1,18 @@ +"""tests/integration/test_xenserver_config.py: Test xen-bugtool --entries=xenserver-config""" +import os + +from utils import assert_cmd, check_file, run_bugtool_entry, verify_content_from_dom0_template + + +def test_xenserver_config(output_archive_type): + """Test xen-bugtool --entries=xenserver-config in test jail created by auto-fixtures in conftest.py""" + entry = "xenserver-config" + run_bugtool_entry(output_archive_type, entry) + assert check_file("ls-lR-%opt%xensource.out").splitlines()[0] == "/opt/xensource:" + assert check_file("ls-lR-%etc%xensource%static-vdis.out") == "" + assert check_file("static-vdis-list.out") == "list" + os.chdir("..") + assert_cmd(["tar", "xvf", entry + "/etc/systemd.tar"], entry + "/etc/systemd.tar") + os.chdir(entry) + verify_content_from_dom0_template("etc/xensource-inventory") + verify_content_from_dom0_template("etc/systemd") diff --git a/tests/integration/utils.py b/tests/integration/utils.py new file mode 100644 index 00000000..c89702cb --- /dev/null +++ b/tests/integration/utils.py @@ -0,0 +1,90 @@ +"""tests/integration/utils.py: utility functions to test xen-bugtool by invoking it""" +from __future__ import print_function + +import filecmp +import os +import shutil +import sys +import tarfile +import zipfile +from subprocess import PIPE, Popen + +from lxml import etree + +if sys.version_info.major == 2: + from commands import getoutput # pyright: ignore[reportMissingImports] +else: + from subprocess import getoutput + +BUGTOOL_OUTPUT_DIR = "/var/opt/xen/bug-report/" +BUGTOOL_DOM0_TEMPL = "tests/integration/dom0-template" + + +def run(command): + """Run the given shell command, print it's stdout and stderr and return them""" + process = Popen(command, stdout=PIPE, stderr=PIPE, universal_newlines=True) + stdout, stderr = process.communicate() + returncode = process.wait() + print("# " + " ".join(command) + ":") + if returncode: + raise Exception(returncode, stderr) + print(stdout, stderr) + return stdout, stderr + + +def assert_cmd(cmd, remove_file): + """Run the given command, print its stdout and stderr and return them. Remove the given remove_file""" + stdout, stderr = run(cmd) + # After successfuly verficiation of the files, remove the checked output file(missed files remain): + os.unlink(remove_file) + return stdout, stderr + + +def check_file(path): + """Return the contents of the passed bugtool output file for verification""" + with open(path) as handle: + contents = handle.read() + # After successfuly verficiation of the files, remove the checked output file (missed files remain): + os.unlink(path) + return contents + + +def verify_content_from_dom0_template(path): + """Compare the contents of output directories or files with the test's Dom0 template directories""" + assert path[0] != "/" + assert filecmp.dircmp(path, BUGTOOL_DOM0_TEMPL + path) + # After successfuly verficiation of the files, remove the checked output files (missed files remain): + try: + os.unlink(path) + except OSError: + shutil.rmtree(path) + + +def run_bugtool_entry(archive_type, test_entries): + """Run bugtool for the given entry or entries, extract the output, and chdir to it""" + os.environ["XENRT_BUGTOOL_BASENAME"] = test_entries + # For case the default python interpreter of the user is python3, we must use python2(for now): + command = "python2 ./xen-bugtool -y --output=%s --entries=%s" % (archive_type, test_entries) + print("# " + command) + print(getoutput(command)) + srcdir = os.getcwd() + os.chdir(BUGTOOL_OUTPUT_DIR) + output_file = test_entries + "." + archive_type + print("# Unpacking " + BUGTOOL_OUTPUT_DIR + output_file + " and verifying inventory.xml") + # Python2.7 does not have shutil.unpack_archive(): + # shutil.unpack_archive(output_file): + if archive_type == "zip": + archive = zipfile.ZipFile(output_file) + elif archive_type == "tar": + archive = tarfile.open(output_file) + elif archive_type == "tar.bz2": + archive = tarfile.open(output_file, "r:bz2") + else: + raise RuntimeError("Unsupported output archive type: %s" % archive_type) + archive.extractall() + os.chdir(test_entries) + # Validate the extracted inventory.xml using the XML schema from the test framework: + with open(srcdir + "/tests/integration/inventory.xsd") as xmlschema: + etree.XMLSchema(etree.parse(xmlschema)).assertValid(etree.parse("inventory.xml")) + # After successfuly validation of the inventory.xml, remove it (not removed files make the test fail): + os.unlink("inventory.xml") diff --git a/tests/unit/test_xapidb_filter.py b/tests/unit/test_xapidb_filter.py index ec42f25b..da2b6567 100644 --- a/tests/unit/test_xapidb_filter.py +++ b/tests/unit/test_xapidb_filter.py @@ -1,11 +1,12 @@ # This uses the deprecated imp module because it has to run with Python2.7 for now: -import imp # pylint: disable=deprecated-module import os +import sys import xml.dom.minidom +import pytest + testdir = os.path.dirname(__file__) -bugtool = imp.load_source("bugtool", testdir + "/../../xen-bugtool") original = r""" @@ -53,7 +54,11 @@ """ +@pytest.mark.skipif(sys.version_info >= (3,0), reason="requires python2") def test_xapi_database_filter(): """Assert that bugtool.DBFilter().output() filters the xAPI database as expected""" + import imp # pylint: disable=deprecated-module # pyright: ignore[reportMissingImports] + + bugtool = imp.load_source("bugtool", testdir + "/../../xen-bugtool") filtered = bugtool.DBFilter(original).output() assert xml.dom.minidom.parseString(filtered).toprettyxml(indent=" ") == expected