Update p4_branch

.appveyor.yml

@@ -7,19 +7,18 @@
 
 environment:
   matrix:
-    - PYTHON: "C:\\Python35"
     - PYTHON: "C:\\Python36"
     - PYTHON: "C:\\Python37"
     - PYTHON: "C:\\Python38"
+    - PYTHON: "C:\\Python39"
 
 build: off
 
 install:
   - set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
   # The package installation via our setup.py file uses easy_install, which fails to correctly install lxml.
   # As some Python environments don't have lxml preinstalled, install it here to avoid errors during the execution in those cases.
-  # Furthermore, since the move to setup.cfg on Python 3.5 it fails to automatically install nose before running nosetests.
-  - python -m pip install lxml nose
+  - python -m pip install lxml
   - python -m pip install --user .
 
 test_script:

.gitlab-ci.yml

@@ -59,11 +59,6 @@ stages:
     paths:
       - coverage.xml
 
-unit-tests:python-3.5:
-  <<: *unit-tests
-  variables:
-    PYTHON_VERSION: '3.5'
-
 unit-tests:python-3.6:
   <<: *unit-tests
   variables:
@@ -180,12 +175,6 @@ reuse:
     - schedules
     - web
 
-build-docker:test:python-3.5:
-  extends: .build-docker
-  variables:
-    DOCKERFILE: test/Dockerfile.python-3.5
-    IMAGE: test:python-3.5
-
 build-docker:test:python-3.6:
   extends: .build-docker
   variables:

.travis.yml

@@ -7,11 +7,12 @@
 
 language: python
 sudo: required
-dist: xenial
+dist: bionic
 python:
-  - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
+  - "3.9"
 install:
   - pip install .
   # Install code formatter black, but only on versions where it is available

CHANGELOG.md

@@ -9,6 +9,44 @@ SPDX-License-Identifier: Apache-2.0
 
 # BenchExec Changelog
 
+## BenchExec 3.7
+
+This is expected to be the last BenchExec release that supports Python 3.5,
+newer releases will require Python 3.6 or newer.
+Please [cf. issue #717](https://github.com/sosy-lab/benchexec/issues/717)
+for our plan on dropping support for further Python versions.
+
+We would like to note that Linux kernel 5.11
+brings a major improvement for BenchExec users not on Ubuntu:
+Now it should be possible to use the overlayfs feature as a regular user,
+no need to pass `--read-only-dir /` and similar parameters.
+We updated our [installation instructions](https://github.com/sosy-lab/benchexec/blob/master/doc/INSTALL.md)
+accordingly and also clarified that BenchExec requires x86 or ARM machines
+and recommend Linux kernel 4.14 or newer due to reduced cgroups overhead.
+
+Changes in this release:
+
+- In HTML tables, the following settings are now stored in the hash part of the URL:
+  - Column sorting
+  - Page size of the table, i.e., how many rows are shown
+  - Filters for task names that are defined by entering text
+    into the left-most input field of the filter row of the table.
+    Previously this would only work for task-name filters defined in the filter sidebar.
+  This means that using the back/forward navigation of the browser
+  will change these settings and that they can be present in shared links.
+- Fix a few cases of printing of statistics information in HTML tables.
+  This affects corner cases like the number of visible decimal digits for `0`
+  and trailing zeroes for the standard deviation in the tooltip.
+- When a user requests rounding to a certain number of decimal digits,
+  the filtering functionality of the HTML tables will now use the raw values,
+  not the rounded values.
+  This is consistent with the behavior when rounding is not explicitly requested
+  and BenchExec applies the default rounding rules.
+- Fix harmless stack trace printed at end of `benchexec` execution
+  in cases like of early termination, e.g., if the tool could not be found.
+- Some improvements to tool-info modules.
+- Several updates of JS libraries, but this should not bring user-visible changes.
+
 ## BenchExec 3.6
 
 - One tool-info module improved.

benchexec/__init__.py

@@ -34,7 +34,7 @@
 Variables ending with "tag" contain references to XML tag objects created by the XML parser.
 """
 
-__version__ = "3.7-dev"
+__version__ = "3.8-dev"
 
 
 class BenchExecException(Exception):

benchexec/benchexec.py

@@ -56,7 +56,7 @@ def start(self, argv):
 
         for arg in self.config.files:
             if not os.path.exists(arg) or not os.path.isfile(arg):
-                parser.error("File {0} does not exist.".format(repr(arg)))
+                parser.error(f"File {arg!r} does not exist.")
 
         if os.path.isdir(self.config.output_path):
             self.config.output_path = os.path.normpath(self.config.output_path) + os.sep
@@ -85,11 +85,14 @@ def create_argument_parser(self):
         """
         parser = argparse.ArgumentParser(
             fromfile_prefix_chars="@",
-            description="""Execute benchmarks for a given tool with a set of input files.
-               Benchmarks are defined in an XML file given as input.
-               Command-line parameters can additionally be read from a file if file name prefixed with '@' is given as argument.
-               The tool table-generator can be used to create tables for the results.
-               Part of BenchExec: https://github.com/sosy-lab/benchexec/""",
+            description="""
+                Execute benchmarks for a given tool with a set of input files.
+                Benchmarks are defined in an XML file given as input.
+                Command-line parameters can additionally be read from a file
+                if file name prefixed with '@' is given as argument.
+                The tool table-generator can be used to create tables for the results.
+                Part of BenchExec: https://github.com/sosy-lab/benchexec/
+            """,
         )
 
         parser.add_argument(
@@ -110,8 +113,11 @@ def create_argument_parser(self):
             "--rundefinition",
             dest="selected_run_definitions",
             action="append",
-            help="Run only the specified RUN_DEFINITION from the benchmark definition file. "
-            + "This option can be specified several times and can contain wildcards.",
+            help="""
+                Execute only the specified RUN_DEFINITION from the benchmark-definition
+                file.
+                This option can be specified several times and can contain wildcards.
+            """,
             metavar="RUN_DEFINITION",
         )
 
@@ -120,15 +126,19 @@ def create_argument_parser(self):
             "--tasks",
             dest="selected_sourcefile_sets",
             action="append",
-            help="Run only the tasks from the tasks tag with TASKS as name. "
-            + "This option can be specified several times and can contain wildcards.",
+            help="""
+                Run only the tasks from the tasks tag with TASKS as name.
+                This option can be specified several times and can contain wildcards.
+            """,
             metavar="TASKS",
         )
 
         parser.add_argument(
             "--tool-directory",
-            help="Use benchmarked tool from given directory "
-            "instead of looking in PATH and the current directory",
+            help="""
+                Use benchmarked tool from given directory
+                instead of looking in PATH and the current directory.
+            """,
             metavar="DIR",
             type=util.non_empty_str,
         )
@@ -148,25 +158,31 @@ def create_argument_parser(self):
             dest="output_path",
             type=str,
             default=self.DEFAULT_OUTPUT_PATH,
-            help="Output prefix for the generated results. "
-            + "If the path is a folder files are put into it,"
-            + "otherwise it is used as a prefix for the resulting files.",
+            help="""
+                Output prefix for the generated results.
+                If the path is a folder files are put into it,
+                otherwise it is used as a prefix for the resulting files.
+            """,
         )
 
         parser.add_argument(
             "--description-file",
-            help="Path to a text file whose contents will be included in the results "
-            "as their description",
+            help="""
+                Path to a text file whose contents will be included in the results
+                as their description
+            """,
         )
 
         parser.add_argument(
             "-T",
             "--timelimit",
             dest="timelimit",
             default=None,
-            help='Time limit for each run, e.g. "90s" '
-            "(overwrites time limit and hard time limit from XML file, "
-            'use "-1" to disable time limits completely)',
+            help="""
+                Time limit for each run, e.g. "90s"
+                (overwrites time limit and hard time limit from XML file,
+                use "-1" to disable time limits completely)
+            """,
             metavar="SECONDS",
         )
 
@@ -175,9 +191,12 @@ def create_argument_parser(self):
             "--walltimelimit",
             dest="walltimelimit",
             default=None,
-            help='Wall time limit for each run, e.g. "90s" '
-            "(overwrites wall time limit from XML file, "
-            'use "-1" to use CPU time limit plus a few seconds, such value is also used by default)',
+            help="""
+                Wall time limit for each run, e.g. "90s"
+                (overwrites wall time limit from XML file,
+                use "-1" to use CPU time limit plus a few seconds,
+                such value is also used by default)
+            """,
             metavar="SECONDS",
         )
 
@@ -214,16 +233,21 @@ def create_argument_parser(self):
             dest="coreset",
             default=None,
             type=util.parse_int_list,
-            help="Limit the set of cores BenchExec will use for all runs "
-            "(Applied only if the number of CPU cores is limited).",
+            help="""
+                Limit the set of cores BenchExec will use for all runs "
+                (Applied only if the number of CPU cores is limited).
+            """,
             metavar="N,M-K",
         )
         parser.add_argument(
             "--no-hyperthreading",
             dest="use_hyperthreading",
             action="store_false",
             default=True,
-            help="Disable assignment of more than one sibling virtual core to a single run",
+            help="""
+                Disable assignment of more than one sibling virtual core
+                to a single run
+            """,
         )
 
         parser.add_argument(
@@ -239,8 +263,8 @@ def parse_filesize_value(value):
                 if value == -1:
                     return None
                 logging.warning(
-                    'Value "%s" for logfile size interpreted as MB for backwards compatibility, '
-                    "specify a unit to make this unambiguous.",
+                    'Value "%s" for logfile size interpreted as MB for '
+                    "backwards compatibility, specify a unit to make this unambiguous.",
                     value,
                 )
                 value = value * _BYTE_FACTOR * _BYTE_FACTOR
@@ -262,21 +286,31 @@ def parse_filesize_value(value):
             "--filesCountLimit",
             type=int,
             metavar="COUNT",
-            help="maximum number of files the tool may write to (checked periodically, counts only files written in container mode or to temporary directories)",
+            help="""
+                maximum number of files the tool may write to
+                (checked periodically, counts only files written in container mode
+                or to temporary directories)
+            """,
         )
         parser.add_argument(
             "--filesSizeLimit",
             type=util.parse_memory_value,
             metavar="BYTES",
-            help="maximum size of files the tool may write (checked periodically, counts only files written in container mode or to temporary directories)",
+            help="""
+                Maximum size of files the tool may write
+                (checked periodically, counts only files written in container mode
+                or to temporary directories)
+            """,
         )
 
         parser.add_argument(
             "--commit",
             dest="commit",
             action="store_true",
-            help="If the output path is a git repository without local changes, "
-            + "add and commit the result files.",
+            help="""
+                If the output path is a git repository without local changes, "
+                add and commit the result files.
+            """,
         )
 
         parser.add_argument(
@@ -337,31 +371,27 @@ def execute_benchmark(self, benchmark_file):
             self.config,
             self.config.start_time or util.read_local_time(),
         )
-        self.check_existing_results(benchmark)
-
-        self.executor.init(self.config, benchmark)
-        output_handler = OutputHandler(
-            benchmark, self.executor.get_system_info(), self.config.compress_results
-        )
+        try:
+            self.check_existing_results(benchmark)
 
-        logging.debug(
-            "I'm benchmarking %r consisting of %s run sets using %s %s.",
-            benchmark_file,
-            len(benchmark.run_sets),
-            benchmark.tool_name,
-            benchmark.tool_version or "(unknown version)",
-        )
+            self.executor.init(self.config, benchmark)
+            output_handler = OutputHandler(
+                benchmark, self.executor.get_system_info(), self.config.compress_results
+            )
+            try:
+                logging.debug(
+                    "I'm benchmarking %r consisting of %s run sets using %s %s.",
+                    benchmark_file,
+                    len(benchmark.run_sets),
+                    benchmark.tool_name,
+                    benchmark.tool_version or "(unknown version)",
+                )
 
-        try:
-            result = self.executor.execute_benchmark(benchmark, output_handler)
+                result = self.executor.execute_benchmark(benchmark, output_handler)
+            finally:
+                output_handler.close()
         finally:
             benchmark.tool.close()
-            output_handler.close()
-            # remove useless log folder if it is empty
-            try:
-                os.rmdir(benchmark.log_folder)
-            except OSError:
-                pass
 
         if self.config.commit and not self.stopped_by_interrupt:
             try:
@@ -385,15 +415,13 @@ def check_existing_results(self, benchmark):
         """
         if os.path.exists(benchmark.log_folder):
             sys.exit(
-                "Output directory {0} already exists, will not overwrite existing results.".format(
-                    benchmark.log_folder
-                )
+                f"Output directory {benchmark.log_folder} already exists, "
+                f"will not overwrite existing results."
             )
         if os.path.exists(benchmark.log_zip):
             sys.exit(
-                "Output archive {0} already exists, will not overwrite existing results.".format(
-                    benchmark.log_zip
-                )
+                f"Output archive {benchmark.log_zip} already exists, "
+                f"will not overwrite existing results."
             )
 
     def stop(self):