Fix getting output from sar -A, moving sar to "system-load"

When the human-readable output of `sar -A` was added in April
to xenserver-logs, the timeout for this command was not defined.

The senserver-logs capability uses the default timeout of -1
which is racey for commands as they are immediately stopped
due to the timout running out unless the are very very brief.

To fix this, it was suggested that sar (a system load report)
could be moved to a new capability. Also, it should be noted that
sar records are binary system load records, not xenserver-specific
log files.

Therefore, create a new capability (system_load), repsonsible for
collecting the the sar data files (and the humand-readable of output
of `sar -A`) with a timeout of 30 seconds and a max_size of 70 MB.

Signed-off-by: Bernhard Kaindl <[email protected]>

tests/integration/sar-file-collection.test.sh

@@ -9,14 +9,22 @@ set -o errexit
 set -o pipefail
 if [[ -n "$TRACE" ]]; then set -o xtrace; fi
 set -o nounset
+SRC=$PWD
 : ${PYTHON:=python2}
 
+# The bugtool capabilitiy this test is testing:
+CAP=system-load
+
+# Test object: A dummy sar files (Unix system activity reporter from the sysstat pkg)
+# and the output of a fake `sar -A` command. URL: https://github.com/sysstat/sysstat
+
 # Prepare test container: Mock the files used by this test
 cp -a tests/integration/dom0-template/* /
 mkdir -p  /var/log/sa
 echo sa  >/var/log/sa/sa01
 echo sr  >/var/log/sa/sar31
-echo -e '#!/bin/sh\necho $*' >/bin/sar;chmod +x /bin/sar
+echo -e '#!/bin/sh\nsleep 2;find xen-bugtool tests/ -type f|xargs cat;echo $*' >/bin/sar
+chmod +x /bin/sar
 
 # Enter a clean test environment
 rm -rf   .tmp/tests/sar-file-collection
@@ -27,10 +35,20 @@ export PYTHONPATH=~-/tests/mocks
 # Check that mocking xen.lowlevel.xc works for this test
 $PYTHON -c "from xen.lowlevel.xc import Error, xc;xc().domain_getinfo()"
 
-# Run xen-bugtool --entries=xenserver-logs to capture the dummy SAR files
-# and run the mocked sar command:
-$PYTHON ~-/xen-bugtool -y --entries=xenserver-logs --output=tar --outfd=1 -s |
-    tar xvf - --strip-components=1
+# Run xen-bugtool --entries=$CAP to test tar output (to a file descriptor in this case)
+tar_basename=tar
+export XENRT_BUGTOOL_BASENAME=$tar_basename
+# Test creating a tar archive on a file descriptor using --output=tar --outfd=fd
+$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=tar --outfd=2 2>tar.bz2
+tar xvf tar.bz2
+
+# Run xen-bugtool --entries=$CAP --output=zip to test zip output (output to file only)
+zip_basename=zip
+export XENRT_BUGTOOL_BASENAME=$zip_basename
+$PYTHON ~-/xen-bugtool -y --entries=$CAP --output=zip
+unzip -o -d. /var/opt/xen/bug-report/zip.zip
+
+pushd $zip_basename
 
 # Show a detailed file list in the outlog log for human analysis in case of errors
 find * -type f -print0 | xargs -0 ls -l
@@ -39,15 +57,21 @@ find * -type f -print0 | xargs -0 ls -l
 grep -q '^sa$' var/log/sa/sa01
 grep -q '^sr$' var/log/sa/sar31
 
-# xen-bugtool is expected to call sar -A, and the symlink to echo captures it:
-grep -q '^-A$' sar-A.out
+# bugtool call sar -A, printing dummy data and the arguments it was given. Check that:
+tail -1 sar-A.out | grep -q '^-A$'
 
 # There is likely a xml tool to check the file names, in inventory.xml,
 # but after verfiying the the files were included it should be sufficent
 # check that they are also mentioned int the inventory.xml:
-grep -q 'filename="bug-report-[0-9]*/var/log/sa/sa01"'  inventory.xml
-grep -q 'filename="bug-report-[0-9]*/var/log/sa/sar31"' inventory.xml
-grep -q 'filename="bug-report-[0-9]*/sar-A.out"'        inventory.xml
+grep -q 'filename=".*/var/log/sa/sa01"'  inventory.xml
+grep -q 'filename=".*/var/log/sa/sar31"' inventory.xml
+grep -q 'filename=".*/sar-A.out"'        inventory.xml
 
 # Validate the extracted inventory.xml using the XML Schema
-xmllint --schema ~-/tests/integration/inventory.xsd inventory.xml
+xmllint --schema $SRC/tests/integration/inventory.xsd inventory.xml
+popd
+
+# Check that the tar and zip outputs are identical (except for date and uptime):
+sed -i "s/\\<$tar_basename\\>/$zip_basename/" tar/inventory.xml
+sed -i 's/date="[^"]*"//;s/uptime="[^"]*"//' {tar,zip}/inventory.xml
+diff -rNu tar zip

xen-bugtool

@@ -352,6 +352,7 @@ CAP_PAM                  = 'pam'
 CAP_PROCESS_LIST         = 'process-list'
 CAP_PERSISTENT_STATS     = 'persistent-stats'
 CAP_BLOCK_SCHEDULER      = 'block-scheduler'
+CAP_SYSTEM_LOAD          = 'system-load'
 CAP_SYSTEM_LOGS          = 'system-logs'
 CAP_SYSTEM_SERVICES      = 'system-services'
 CAP_TAPDISK_LOGS         = 'tapdisk-logs'
@@ -426,6 +427,7 @@ cap(CAP_PERSISTENT_STATS,    PII_NO,                    max_size=50*MB,
     max_time=60, checked=False, hidden=True)
 cap(CAP_PROCESS_LIST,        PII_YES,                   max_size=30*KB,
     max_time=60)
+cap(CAP_SYSTEM_LOAD,         PII_NO,                    max_size=70*MB, max_time=30)
 cap(CAP_SYSTEM_LOGS,         PII_MAYBE,                 max_size=50*MB,
     max_time=10)
 cap(CAP_SYSTEM_SERVICES,     PII_NO,                    max_size=128*KB,
@@ -1096,13 +1098,14 @@ exclude those logs from the archive.
                                   'telemetry/telemetry.log.%d', 'telemetry/telemetry.log.%d.gz']]]
 
     # Collect SAR data (binary and text, and add today's report with sar -A)
-    sa_logs = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_XENSERVER_LOGS][VERBOSITY])
-    cmd_output(CAP_XENSERVER_LOGS, ['sar', '-A'])
+    cmd_output(CAP_SYSTEM_LOAD, ['sar', '-A'])
+    sar_data = get_recent_logs(glob.glob("/var/log/sa/sa*[0-9][0-9]"), caps[CAP_SYSTEM_LOAD][VERBOSITY])
+    update_cap_size(CAP_SYSTEM_LOAD, size_of_all(sar_data))
+    file_output(CAP_SYSTEM_LOAD, sar_data)
 
     qemu_logs = get_recent_logs(glob.glob('/tmp/qemu.[0-9]*'), caps[CAP_XENSERVER_LOGS][VERBOSITY])
-    update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + sa_logs + qemu_logs))
+    update_cap_size(CAP_XENSERVER_LOGS, size_of_all(xenserver_logs + qemu_logs))
     file_output(CAP_XENSERVER_LOGS, xenserver_logs)
-    file_output(CAP_XENSERVER_LOGS, sa_logs)
     file_output(CAP_XENSERVER_LOGS, qemu_logs)
     tree_output(CAP_XENSERVER_LOGS, OEM_CONFIG_DIR, OEM_XENSERVER_LOGS_RE)