bugtool,test_output.py: Fix collecting output files defined by plugin

tests/integration/dom0-template/usr/sbin/cat

@@ -0,0 +1 @@
+/bin/cat

tests/integration/dom0-template/usr/sbin/ls

@@ -0,0 +1 @@
+/bin/ls

tests/unit/test_output.py

@@ -0,0 +1,74 @@
+"""Unit tests for bugtool core functions creating minimal output archives"""
+import os
+import tarfile
+import zipfile
+
+from lxml.etree import XMLSchema, parse  # pytype: disable=import-error
+
+
+def assert_valid_inventory_schema(inventory_tree):
+    """Assert that the passed inventory validates against the inventory schema"""
+
+    with open(os.getcwd() + "/tests/integration/inventory.xsd") as xmlschema:
+        XMLSchema(parse(xmlschema)).assertValid(inventory_tree)
+
+
+def assert_mock_bugtool_plugin_output(extracted):
+    """Assertion check of the output files from the mock bugtool plugin"""
+    assert_valid_inventory_schema(parse(extracted + "inventory.xml"))
+    with open(extracted + "proc_version.out") as proc_version:
+        assert proc_version.read()[:14] == "Linux version "
+    with open(extracted + "ls-l-%etc.out") as etc:
+        assert etc.read()[:6] == "total "
+    with open(extracted + "proc/self/status") as status:
+        assert status.read()[:5] == "Name:"
+    with open(extracted + "proc/sys/fs/epoll/max_user_watches") as max_user_watches:
+        assert int(max_user_watches.read()) > 0
+
+
+def minimal_bugtool(bugtool, dom0_template, archive, subdir):
+    """Load the plugins from the template and include the generated inventory"""
+
+    # Load the mock plugin from dom0_template and process the plugin's caps:
+    bugtool.PLUGIN_DIR = dom0_template + "/etc/xensource/bugtool"
+    bugtool.entries = ["mock"]
+    bugtool.load_plugins(just_capabilities=False)
+    bugtool.data["ls -l /etc"]["cmd_args"][2] = dom0_template + "/etc"
+    bugtool.collect_data(subdir, archive)
+    bugtool.include_inventory(archive, subdir)
+    archive.close()
+    return True
+
+
+def test_tar_output(bugtool, tmp_path, dom0_template):
+    """Assert that a bugtool unit test creates a valid minimal tar archive"""
+
+    bugtool.BUG_DIR = tmp_path
+    archive = bugtool.TarOutput("tarball", "tar", -1)
+    subdir = "tardir"
+
+    # Create a minimal bugtool output archive to test core functions:
+    if not minimal_bugtool(bugtool, dom0_template, archive, subdir):
+        return  # To be fixed with the conversion to use io.BytesIO
+
+    # Check the TarFile contents
+    tmp = str(tmp_path)
+    tarfile.TarFile(tmp + "/tarball.tar").extractall(tmp)
+    assert_mock_bugtool_plugin_output(tmp + "/" + subdir + "/")
+
+
+def test_zip_output(bugtool, tmp_path, dom0_template):
+    """Assert that a bugtool unit test creates a valid minimal zip archive"""
+
+    bugtool.BUG_DIR = tmp_path
+    archive = bugtool.ZipOutput("zipfile")
+    subdir = "zipdir"
+
+    # Create a minimal bugtool output archive to test core functions:
+    if not minimal_bugtool(bugtool, dom0_template, archive, subdir):
+        return  # To be fixed with the conversion to use io.BytesIO
+
+    # Check the ZipFile contents
+    tmp = str(tmp_path)
+    zipfile.ZipFile(tmp + "/zipfile.zip").extractall(tmp)
+    assert_mock_bugtool_plugin_output(tmp + "/" + subdir + "/")

xen-bugtool

@@ -557,6 +557,13 @@ def get_recent_logs(logs, verbosity):
     return [x[1] for x in (logs[-verbosity:] if verbosity < 9 else logs)]
 
 
+def include_inventory(archive, dir):
+    """Add the inventory.xml to the archive, filled from the current data"""
+
+    info = StringIOmtime(make_inventory(data, dir))
+    archive.add_path_with_data(construct_filename(dir, "inventory.xml", {}), info)
+
+
 def collect_data(subdir, archive):
     process_lists = {}
 
@@ -573,7 +580,7 @@ def collect_data(subdir, archive):
                            filename.startswith('/sys/')):
             # proc files must be read into memory
             try:
-                f = open(v['filename'], 'r')
+                f = open(v["filename"], "rb")
                 s = f.read(unlimited_data and -1 or caps[cap][MAX_SIZE])
                 f.close()
                 if unlimited_data or caps[cap][MAX_SIZE] == -1 or \
@@ -585,8 +592,9 @@ def collect_data(subdir, archive):
                     cap_sizes[cap] += len(s)
                 else:
                     log("Omitting %s, size constraint of %s exceeded" % (v['filename'], cap))
-            except:
-                pass
+            except IOError as e:
+                if e.errno != 2:
+                    log("IOError reading %s: %s" % (filename, e))
         elif "func" in v:
             try:
                 s = no_unicode(v["func"](cap))
@@ -1180,9 +1188,7 @@ exclude those logs from the archive.
     collect_data(subdir, archive)
 
     # include inventory
-    inventory = {'cap': None, 'output': StringIOmtime(no_unicode(make_inventory(data, subdir)))}
-    archive.add_path_with_data(construct_filename(subdir, 'inventory.xml', inventory),
-                StringIOmtime(no_unicode(make_inventory(data, subdir))))
+    include_inventory(archive, subdir)
 
     if archive.close():
         res = 0
@@ -1830,7 +1836,7 @@ def make_inventory(inventory, subdir):
 
     for inventory_key, inventory_item in inventory.items():
         inventory_entry(document, subdir, inventory_key, inventory_item)
-    return document.toprettyxml()
+    return document.toprettyxml(encoding="utf-8")
 
 def inventory_entry(document, subdir, k, v):
     try:
@@ -1879,7 +1885,7 @@ def construct_filename(subdir, k, v):
         if s.find('.') == -1:
             s += '.out'
 
-    return no_unicode(os.path.join(subdir, s))
+    return os.path.join(subdir, s)
 
 
 def update_capabilities():