Overlay handling with fuse-overlayfs

benchexec/containerized_tool.py

@@ -206,7 +206,6 @@ def _init_container(
 
     # Container config
     container.setup_user_mapping(os.getpid(), uid, gid)
-    _setup_container_filesystem(temp_dir, dir_modes, container_system_config)
     if container_system_config:
         socket.sethostname(container.CONTAINER_HOSTNAME)
     if not network_access:
@@ -225,6 +224,10 @@ def _init_container(
         os.waitpid(pid, 0)
         os._exit(0)
 
+    # We setup the container's filesystem in the child process.
+    # Delaying this until after the fork can avoid "Transport endpoint not connected" issue.
+    _setup_container_filesystem(temp_dir, dir_modes, container_system_config)
+
     # Finalize container setup in child
     container.mount_proc(container_system_config)  # only possible in child
     container.drop_capabilities()

benchexec/libc.py

@@ -184,14 +184,27 @@ class CapData(_ctypes.Structure):
     _ctypes.POINTER(CapData * 2),
 ]
 
+capget = _libc.capget
+"""Get the capabilities of the current thread."""
+capget.errcheck = _check_errno
+capget.argtypes = [
+    _ctypes.POINTER(CapHeader),
+    _ctypes.POINTER(CapData * 2),
+]
+
 LINUX_CAPABILITY_VERSION_3 = 0x20080522  # /usr/include/linux/capability.h
+LINUX_CAPABILITY_U32S_3 = 2  # /usr/include/linux/capability.h
 CAP_SYS_ADMIN = 21  # /usr/include/linux/capability.h
+PR_CAP_AMBIENT = 47  # /usr/include/linux/prctl.h
+PR_CAP_AMBIENT_RAISE = 2  # /usr/include/linux/prctl.h
+PR_CAP_AMBIENT_CLEAR_ALL = 4  # /usr/include/linux/prctl.h
 
 prctl = _libc.prctl
 """Modify options of processes: http://man7.org/linux/man-pages/man2/prctl.2.html"""
 prctl.errcheck = _check_errno
 prctl.argtypes = [c_int, c_ulong, c_ulong, c_ulong, c_ulong]
 
+
 # /usr/include/linux/prctl.h
 PR_SET_DUMPABLE = 4
 PR_GET_SECCOMP = 21

benchexec/test_runexecutor.py

@@ -1199,6 +1199,130 @@ def test_uptime_without_lxcfs(self):
             uptime, 10, f"Uptime {uptime}s unexpectedly low in container"
         )
 
+    def test_fuse_overlay(self):
+        if not container.get_fuse_overlayfs_executable():
+            self.skipTest("fuse-overlayfs not available")
+        with tempfile.TemporaryDirectory(prefix="BenchExec_test_") as temp_dir:
+            test_file_path = os.path.join(temp_dir, "test_file")
+            with open(test_file_path, "wb") as test_file:
+                test_file.write(b"TEST_TOKEN")
+
+            self.setUp(
+                dir_modes={
+                    "/": containerexecutor.DIR_READ_ONLY,
+                    "/home": containerexecutor.DIR_HIDDEN,
+                    "/tmp": containerexecutor.DIR_HIDDEN,
+                    temp_dir: containerexecutor.DIR_OVERLAY,
+                },
+            )
+            result, output = self.execute_run(
+                "/bin/sh",
+                "-c",
+                f"if [ $({self.cat} {test_file_path}) != TEST_TOKEN ]; then exit 1; fi; \
+                {self.echo} TOKEN_CHANGED >{test_file_path}",
+            )
+            self.check_result_keys(result, "returnvalue")
+            self.check_exitcode(result, 0, "exit code of inner runexec is not zero")
+            self.assertTrue(
+                os.path.exists(test_file_path),
+                f"File '{test_file_path}' removed, output was:\n" + "\n".join(output),
+            )
+            with open(test_file_path, "rb") as test_file:
+                test_token = test_file.read()
+            self.assertEqual(
+                test_token.strip(),
+                b"TEST_TOKEN",
+                f"File '{test_file_path}' content is incorrect. Expected 'TEST_TOKEN', but got:\n{test_token}",
+            )
+
+    def test_triple_nested_runexec(self):
+        if not container.get_fuse_overlayfs_executable():
+            self.skipTest("missing fuse-overlayfs")
+
+        # Check if COV_CORE_SOURCE environment variable is set and remove it.
+        # This is necessary because the coverage tool will not work in the nested runexec.
+        coverage_env_var = os.environ.pop("COV_CORE_SOURCE", None)
+
+        with tempfile.TemporaryDirectory(prefix="BenchExec_test_") as temp_dir:
+            overlay_dir = os.path.join(temp_dir, "overlay")
+            os.makedirs(overlay_dir)
+            test_file = os.path.join(overlay_dir, "TEST_FILE")
+            output_dir = os.path.join(temp_dir, "output")
+            os.makedirs(output_dir)
+            mid_output_file = os.path.join(output_dir, "mid_output.log")
+            inner_output_file = os.path.join(output_dir, "inner_output.log")
+            with open(test_file, "w") as f:
+                f.write("TEST_TOKEN")
+                f.seek(0)
+
+            outer_cmd = [
+                "python3",
+                runexec,
+                "--full-access-dir",
+                "/",
+                "--overlay-dir",
+                overlay_dir,
+                "--full-access-dir",
+                output_dir,
+                "--hidden-dir",
+                "/tmp",
+                "--output",
+                mid_output_file,
+                "--",
+            ]
+            mid_cmd = [
+                "python3",
+                runexec,
+                "--full-access-dir",
+                "/",
+                "--overlay-dir",
+                overlay_dir,
+                "--full-access-dir",
+                output_dir,
+                "--hidden-dir",
+                "/tmp",
+                "--output",
+                inner_output_file,
+                "--",
+            ]
+            inner_cmd = [
+                "/bin/sh",
+                "-c",
+                f"if [ $({self.cat} {test_file}) != TEST_TOKEN ]; then exit 1; fi; {self.echo} TOKEN_CHANGED >{test_file}",
+            ]
+            combined_cmd = outer_cmd + mid_cmd + inner_cmd
+
+            self.setUp(
+                dir_modes={
+                    "/": containerexecutor.DIR_FULL_ACCESS,
+                    "/tmp": containerexecutor.DIR_HIDDEN,
+                    overlay_dir: containerexecutor.DIR_OVERLAY,
+                    output_dir: containerexecutor.DIR_FULL_ACCESS,
+                },
+            )
+            outer_result, outer_output = self.execute_run(*combined_cmd)
+            self.check_result_keys(outer_result, "returnvalue")
+            self.check_exitcode(
+                outer_result, 0, "exit code of outer runexec is not zero"
+            )
+            with open(mid_output_file, "r") as f:
+                self.assertIn("returnvalue=0", f.read().strip().splitlines())
+            self.assertTrue(
+                os.path.exists(test_file),
+                f"File '{test_file}' removed, output was:\n" + "\n".join(outer_output),
+            )
+            with open(test_file, "r") as f:
+                test_token = f.read()
+                self.assertEqual(
+                    test_token.strip(),
+                    "TEST_TOKEN",
+                    f"File '{test_file}' content is incorrect. Expected 'TEST_TOKEN', but got:\n{test_token}",
+                )
+
+        # Restore COV_CORE_SOURCE environment variable
+        if coverage_env_var is not None:
+            os.environ["COV_CORE_SOURCE"] = coverage_env_var
+
 
 class _StopRunThread(threading.Thread):
     def __init__(self, delay, runexecutor):

debian/control

@@ -20,7 +20,7 @@ Package: benchexec
 Architecture: all
 Pre-Depends: ${misc:Pre-Depends}
 Depends: ${python3:Depends}, python3-pkg-resources, ${misc:Depends}, ucf
-Recommends: cpu-energy-meter, libseccomp2, lxcfs, python3-coloredlogs, python3-pystemd
+Recommends: cpu-energy-meter, fuse-overlayfs (>= 1.10), libseccomp2, lxcfs, python3-coloredlogs, python3-pystemd
 Description: Framework for Reliable Benchmarking and Resource Measurement
  BenchExec allows benchmarking non-interactive tools on Linux systems.
  It measures CPU time, wall time, and memory usage of a tool,

doc/INSTALL.md

@@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0
 
 The following packages are optional but recommended dependencies:
 - [cpu-energy-meter] will let BenchExec measure energy consumption on Intel CPUs.
+- [fuse-overlayfs] (version 1.10 or newer) allows to use the overlay directory mode for containers in cases where the kernel-based overlayfs does not work.
 - [libseccomp2] provides better container isolation.
 - [LXCFS] provides better container isolation.
 - [coloredlogs] provides nicer log output.
@@ -115,7 +116,7 @@ Of course you can also install BenchExec in a virtualenv if you are familiar wit
 On systems without systemd you can omit the `[systemd]` part.
 
 Please make sure to configure cgroups as [described below](#setting-up-cgroups)
-and install [cpu-energy-meter], [libseccomp2], [LXCFS], and [pqos_wrapper] if desired.
+and install [cpu-energy-meter], [fuse-overlayfs], [libseccomp2], [LXCFS], and [pqos_wrapper] if desired.
 
 ### Containerized Environments
 
@@ -137,7 +138,7 @@ otherwise pip will try to download and build this module,
 which needs a compiler and several development header packages.
 
 Please make sure to configure cgroups as [described below](#setting-up-cgroups)
-and install [cpu-energy-meter], [libseccomp2], [LXCFS], and [pqos_wrapper] if desired.
+and install [cpu-energy-meter], [fuse-overlayfs], [libseccomp2], [LXCFS], and [pqos_wrapper] if desired.
 
 
 ## Kernel Requirements
@@ -155,7 +156,7 @@ on **Linux 5.11 or newer**, so we suggest at least this kernel version.
 And if your system is using cgroups v2 (cf. below),
 the full feature set requires **Linux 5.19 or newer**.
 
-On kernels than 5.11, you need to avoid using the overlay filesystem (cf. below),
+On kernels older than 5.11, you need to avoid using the kernel-based overlay filesystem (cf. below),
 all other features are supported.
 However, we strongly recommend to use at least **Linux 4.14 or newer**
 because it reduces the overhead of BenchExec's memory measurements and limits.
@@ -188,8 +189,13 @@ that are not usable on all distributions by default:
 - **Unprivileged Overlay Filesystem**: This is only available since Linux 5.11
   (kernel option `CONFIG_OVERLAY_FS`),
   but also present in all Ubuntu kernels, even older ones.
-  Users of older kernels on other distributions can still use container mode, but have to choose a different mode
-  of mounting the file systems in the container, e.g., with `--read-only-dir /` (see below).
+  Users of older kernels on other distributions can still use container mode,
+  but have to install [fuse-overlayfs] or choose a different mode
+  of mounting the file systems in the container, e.g., with `--read-only-dir /`
+  (cf. [container configuration](container.md#directory-access-modes)).
+  Note that the kernel-based overlayfs does not support some specific configurations
+  (such as the default mode of overlay for `/`),
+  so [fuse-overlayfs] is often useful or required anyway.
 
 If container mode does not work, please check the [common problems](container.md#common-problems).
 
@@ -382,6 +388,7 @@ Please refer to the [development instructions](DEVELOPMENT.md).
 
 [coloredlogs]: https://pypi.org/project/coloredlogs/
 [cpu-energy-meter]: https://github.com/sosy-lab/cpu-energy-meter
+[fuse-overlayfs]: https://github.com/containers/fuse-overlayfs
 [libseccomp2]: https://github.com/seccomp/libseccomp
 [LXCFS]: https://github.com/lxc/lxcfs
 [pqos]: https://github.com/intel/intel-cmt-cat/tree/master/pqos

doc/benchexec-in-container.md

@@ -47,7 +47,8 @@ or
 ```
 docker run --privileged --cap-drop=all -t my-container benchexec <arguments>
 ```
-
+If you want BenchExec to use `fuse-overlayfs` in the container,
+also specify `--device /dev/fuse`.
 
 ## BenchExec in Interactive Containers
 

doc/container.md

@@ -69,7 +69,9 @@ For each directory in the container one of the following four access modes can b
   Writes to this directory will not be visible on the host.
 - **read-only**: This directory is visible in the container, but read-only.
 - **overlay**: This directory is visible in the container and
-  an [overlay filesystem](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt)
+  an overlay filesystem (either from the
+  [kernel](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt)
+  or [fuse-overlayfs])
   is layered on top of it that redirects all write accesses.
   This means that write accesses are possible in the container, but the effect of any write
   is not visible on the host, only inside the container, and not written to disk.
@@ -205,13 +207,13 @@ You can still use BenchExec if you completely disable the container mode with `-
 #### `Failed to configure container: [Errno 19] Creating overlay mount for '...' failed: No such device`
 Your kernel does not support the overlay filesystem,
 please check the [system requirements](INSTALL.md#kernel-requirements).
-You can use a different access mode for directories, e.g., with `--read-only-dir /`.
+You can use [fuse-overlayfs] or a different access mode for directories, e.g., with `--read-only-dir /`.
 If some directories need to be writable, specify other directory modes for these directories as described above.
 
 #### `Failed to configure container: [Errno 1] Creating overlay mount for '...' failed: Operation not permitted`
 Your kernel does not allow mounting the overlay filesystem inside a container.
-For this you need either Ubuntu or kernel version 5.11 or newer.
-Alternatively, if you cannot use either,
+For this you need either Ubuntu, [fuse-overlayfs], or kernel version 5.11 or newer.
+Alternatively, if you cannot use any of these,
 you can use a different access mode for directories, e.g., with `--read-only-dir /`.
 If some directories need to be writable, specify other directory modes for these directories as described above.
 
@@ -226,6 +228,9 @@ Another limitation of the kernel is that one can only nest overlays twice,
 so if you want to run a container inside a container inside a container,
 at least one of these needs to use a non-overlay mode for this path.
 
+We recommend the installation of [fuse-overlayfs] in version 1.10 or newer,
+which supports all of these use cases.
+
 #### `Cannot change into working directory inside container: [Errno 2] No such file or directory`
 Either you have specified an invalid directory as working directory with `--dir`,
 or your current directory on the host is hidden inside the container
@@ -253,3 +258,5 @@ If it still occurs, please attach to all child process of BenchExec
 with `sudo gdb -p <PID>`, get a stack trace with `bt`,
 and [report an issue](https://github.com/sosy-lab/benchexec/issues/new) with as much information as possible.
 BenchExec will usually be able to continue if the hanging child process is killed.
+
+[fuse-overlayfs]: https://github.com/containers/fuse-overlayfs