mock: new --scrub-all-chroots option

Fixes: #521

behave/features/scrub-all-chroots.feature

@@ -0,0 +1,8 @@
+Feature: Clean all chroots
+
+    @clan_all_chroots
+    Scenario: The --scrub-all-chroots works as expected
+        When mock is run with "--shell true" options
+        And mock is run with "--scrub-all-chroots" options
+        Then the directory /var/lib/mock is empty
+        And the directory /var/cache/mock is empty

behave/features/steps/other.py

@@ -238,3 +238,9 @@ def step_impl(context):
 @given('next mock call uses {option} option')
 def step_impl(context, option):
     context.next_mock_options.append(option)
+
+
+@then("the directory {directory} is empty")
+def step_impl(_, directory):
+    assert_that(os.path.exists(directory), equal_to(True))
+    assert_that(not os.listdir(directory), equal_to(True))

mock/docs/mock.1

@@ -51,6 +51,8 @@ mock  [options] \fB\-\-dnf\-cmd\fR [\fIarguments ...\fR]
 mock  [options] \fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR
 .LP
 mock  [options] \fB\-\-isolated\-build \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR
+.LP
+mock  [options] {\fB\-\-scrub\fR=\fITYPE\fP,\fB\-\-scrub\-all\-chroots\fR}
 
 
 .SH "DESCRIPTION"
@@ -220,6 +222,15 @@ see site\-defaults.cfg for more information.
 \fB\-\-scrub\fR=\fITYPE\fP
 Completely remove the specified chroot or cache dir or all of the chroot and cache.  \fITYPE\fR is one of all, chroot, bootstrap, cache, root\-cache, c\-cache, yum\-cache or dnf\-cache. In fact, dnf\-cache is just alias for yum\-cache, and both remove Dnf and Yum cache.
 .TP
+\fB\-\-scrub\-all\-chroots\fP
+Run \fBmock \-\-scrub=all \-r <\fIchroot\fB>\fR for all chroots that appear to
+have been used previously (some leftovers in \fB/var/lib/mock\fR or
+\fB/var/cache/mock\fR were detected by the heuristic).  This option cannot clean
+leftovers for chroots with configurations in non-standard locations, or if the
+configuration is no longer available.  It also attempts to detect previous use
+of \fB\-\-uniqueext\fR and adjusts the corresponding \fB\-\-scrub=all\fR call
+accordingly.
+.TP
 \fB\-\-shell\fP [\fI\-\-\fR] [\fICOMMAND\fR [\fIARGS...\fR]]
 Shell mode.  Run the specified command interactively within the chroot (no
 \fB\-\-clean\fR is performed).  If no command specified, \fB/bin/sh\fR is run

mock/py/mock.py

@@ -87,6 +87,7 @@
 from mockbuild.state import State
 from mockbuild.trace_decorator import traceLog
 import mockbuild.uid
+from mockbuild.scrub_all import scrub_all_chroots
 
 signal_names = {1: "SIGHUP",
                 13: "SIGPIPE",
@@ -153,6 +154,14 @@ def command_parse():
                       metavar=scrub_metavar,
                       help="completely remove the specified chroot "
                            "or cache dir or all of the chroot and cache")
+    parser.add_option(
+        "--scrub-all-chroots", action="store_const", dest="mode",
+        const="scrub-all-chroots", help=(
+            "Run mock --scrub=all for all chroots that appear to have been "
+            "used previously (see manual page for more info)."
+        ),
+
+    )
     parser.add_option("--init", action="store_const", const="init", dest="mode",
                       help="initialize the chroot, do not build anything")
     parser.add_option("--installdeps", action="store_const", const="installdeps",
@@ -606,9 +615,9 @@ def do_debugconfig(config_opts, expand=False):
 
 
 @traceLog()
-def do_listchroots(config_opts, uidManager):
+def do_listchroots(config_path, uidManager):
     uidManager.run_in_subprocess_without_privileges(
-        config.list_configs, config_opts,
+        config.list_configs, config_path,
     )
 
 
@@ -688,6 +697,9 @@ def main():
     if options.printrootpath or options.list_snapshots:
         options.verbose = 0
 
+    if options.mode == "scrub-all-chroots":
+        return scrub_all_chroots()
+
     # config path -- can be overridden on cmdline
     config_path = MOCKCONFDIR
     if options.configdir:
@@ -1008,7 +1020,7 @@ def run_command(options, args, config_opts, commands, buildroot):
         do_debugconfig(config_opts, True)
 
     elif options.mode == 'listchroots':
-        do_listchroots(config_opts, buildroot.uid_manager)
+        do_listchroots(config_opts["config_path"], buildroot.uid_manager)
 
     elif options.mode == 'orphanskill':
         util.orphansKill(buildroot.make_chroot_path())

mock/py/mockbuild/config.py

@@ -869,40 +869,66 @@ def parse_config_filename(config_filename):
     basename_without_ext = os.path.splitext(basename)[0]
     return (basename, dirname, basename_without_ext)
 
-def get_global_configs(config_opts):
+
+def get_global_configs(config_path):
     """ Return filenames of global configs of chroots"""
     result = []
-    for config_filename in glob("{}/*.cfg".format(config_opts['config_path'])):
+    for config_filename in glob(f"{config_path}/*.cfg"):
         if config_filename in ["/etc/mock/chroot-aliases.cfg", "/etc/mock/site-defaults.cfg"]:
             continue
         result.append(config_filename)
     return result
 
 
-def get_user_config_files(config_opts):
+def get_user_config_files():
     """ Return filenames of user configs of chroots """
     uid = os.getuid()
     custom_path = os.path.join(os.path.expanduser('~' + pwd.getpwuid(uid)[0]), '.config/mock/*.cfg')
     result = glob(custom_path)
     return result
 
 
+def traverse_chroot_configs(config_path=MOCKCONFDIR,
+                            system_configs_traversed_cb=None,
+                            include_eol=False):
+    """
+    Traverse the configuration directories, starting from the system-wide
+    directories, then going through $HOME/.mock/, and yield the (config_path,
+    config_filename, eol=True|False) 3-aries.  Config_path is just passed to
+    callee unchanged!
+    """
+    for config_filename in sorted(get_global_configs(config_path)):
+        yield config_path, config_filename, False
+
+    if include_eol:
+        for config_filename in sorted(get_global_configs(
+                os.path.join(config_path, "eol"))):
+            yield config_path, config_filename, True
+
+    if system_configs_traversed_cb:
+        system_configs_traversed_cb()
+
+    user_config_files = get_user_config_files()
+    if user_config_files:
+        # ~/.config/mock/CHROOTNAME.cfg
+        for config_filename in sorted(user_config_files):
+            yield config_path, config_filename, False
+
+
 @traceLog()
-def list_configs(config_opts):
+def list_configs(config_path):
     log = logging.getLogger()
     log.disabled = True
     # array to save config paths
     print("{} {}".format("config name".ljust(34), "description"))
     print("Global configs:")
-    for config_filename in sorted(get_global_configs(config_opts)):
-        print_description(config_opts['config_path'], config_filename)
-    user_config_files = get_user_config_files(config_opts)
-    if user_config_files:
+
+    def _print_custom():
         print("Custom configs:")
-        # ~/.config/mock/CHROOTNAME.cfg
-        for config_filename in sorted(user_config_files):
-            print_description(config_opts['config_path'], config_filename)
-        log.disabled = False
+
+    for cp, fn, _ in traverse_chroot_configs(config_path, _print_custom):
+        print_description(cp, fn)
+    log.disabled = False
 
 
 @traceLog()

mock/py/mockbuild/scrub_all.py

@@ -0,0 +1,107 @@
+"""
+Logic for observing /var/lib/mock and /var/cache/mock, and try to cleanup as
+much as possible sub-directories there.
+"""
+
+
+import os
+from glob import glob
+import subprocess
+from mockbuild.constants import MOCKCONFDIR
+from mockbuild.config import traverse_chroot_configs
+
+
+def _do_scrub(configs, weird, chroot, suffix=None):
+    if suffix and chroot + "-" + suffix in weird:
+        print(f"skipping weird scrub: {chroot} {suffix}")
+        return
+
+    # FIXME: use mockbuild.backend.Commands().scrub("all" instead
+    base_cmd = ["mock", "--scrub=all", "-r"]
+    cmd = base_cmd + ["eol/" + chroot if configs[chroot] == "eol" else chroot]
+    if suffix is not None:
+        cmd += ["--uniqueext", suffix]
+
+    print("## Calling:", ' '.join(cmd), "##")
+    subprocess.call(cmd)
+
+
+def scrub_all_chroots():
+    """
+    Traverse the important directories, and try to clean them up via
+    `--scrub=all` logic.
+    """
+
+    configs = {}
+    scrub = set()
+    scrub_bootstrap = set()
+    scrub_uniqueext = set()
+    scrub_uniqueext_bootstrap = set()
+    scrub_weird = set()
+    guessing_suffix = {}
+
+    configs = {os.path.basename(f)[:-4]: ("eol" if eol else "normal")
+               for _, f, eol in traverse_chroot_configs(MOCKCONFDIR,
+                                                        include_eol=True)}
+    for directory in glob("/var/lib/mock/*") + glob("/var/cache/mock/*"):
+        if not os.path.isdir(directory):
+            continue
+
+        directory = os.path.basename(directory)
+
+        if directory in configs:
+            scrub.add(directory)
+            continue
+
+        if directory.endswith("-bootstrap"):
+            directory_no_bootstrap = directory[:-10]
+            if directory_no_bootstrap in configs:
+                scrub_bootstrap.add(directory_no_bootstrap)
+                continue
+
+        guessing_suffix[directory] = None
+
+    for config, _ in configs.items():
+        for directory in list(guessing_suffix.keys()):
+            if guessing_suffix[directory]:
+                # already found the cleaning thing
+                continue
+
+            if directory.startswith(config):
+
+                suffix = directory[len(config) + 1:]
+                if suffix.endswith("-bootstrap"):
+                    # See this:
+                    # 1. alma+epel-8-x86_64-php-bootstrap
+                    # 2. alma+epel-8-x86_64-bootstrap-php
+                    # The 1. is weird, and we miss the corresponding
+                    # configuration.  The second could be a "php" uniqueext.
+                    weird_chroot = directory[:-10]
+                    scrub_weird.add(weird_chroot)
+                    continue
+
+                start = "bootstrap-"
+                if suffix.startswith(start):
+                    suffix = suffix[len(start):]
+                    scrub_uniqueext_bootstrap.add((config, suffix))
+                else:
+                    scrub_uniqueext.add((config, suffix))
+
+                guessing_suffix[directory] = "uniqueext"
+
+    for sc, suffix in scrub_uniqueext_bootstrap - scrub_uniqueext:
+        _do_scrub(configs, scrub_weird, sc, suffix)
+
+    for sc, suffix in scrub_uniqueext:
+        _do_scrub(configs, scrub_weird, sc, suffix)
+
+    for only_bootstrap in scrub_bootstrap - scrub:
+        _do_scrub(configs, scrub_weird, only_bootstrap)
+
+    for sc in scrub:
+        _do_scrub(configs, scrub_weird, sc)
+
+    for directory, found in guessing_suffix.items():
+        if found:
+            continue
+        print(f"Unknown directory: {directory}")

releng/release-notes-next/scrub-all-chroots.feature

@@ -0,0 +1,4 @@
+This version addresses [issue#521][], which requested a cleanup option for
+all chroots.  A [new][PR#1337] option, `--scrub-all-chroots`, has been
+added.  It can detect leftovers in `/var/lib/mock` or `/var/cache/mock`
+and make multiple `mock --scrub=all` calls accordingly.