[dvsim] Cleanup FlowCfg::write_results() / utils::clean_odirs()

Signed-off-by: Harry Callahan <[email protected]>

util/dvsim/Deploy.py

@@ -673,7 +673,7 @@ def __init__(self, run_items, sim_cfg):
                                                 sim_cfg.__dict__)
 
         # Prune previous merged cov directories, keeping only the lastest 7.
-        prev_cov_db_dirs = clean_odirs(odir=self.cov_merge_db_dir, max_odirs=7)
+        prev_cov_db_dirs = clean_odirs(new_odir=self.cov_merge_db_dir, max_odirs=7)
 
         # If the --cov-merge-previous command line switch is passed, then
         # merge coverage with the previous runs.

util/dvsim/FlowCfg.py

@@ -488,18 +488,20 @@ def gen_results_summary(self):
     def write_results(self) -> None:
         """Write results to files.
 
-        This function converts text_md to HTML and writes the result to a file
-        in self.results_dir with the file name given by html_filename.  If
-        json_str is not None, this function additionally writes json_str to a
-        file with the same path and base name as the HTML file but with '.json'
-        as suffix.
+        This function:
+        - Writes self.results_md to a file
+        - Converts self.results_md to HTML, and write it to a file
+
+        If the intended new results directory already exists, rename it based on
+        a timestamp of when it was created. Then create the new directory again
+        and write the new results into it.
         """
 
-        # Prepare reports directory, keeping 90 day history.
-        clean_odirs(odir=self.results_dir, max_odirs=89)
-        mk_path(self.results_dir)
+        # Prepare reports directory, keeping the 90 newest results.
+        clean_odirs(new_odir=self.results_dir, max_odirs=89)
 
-        # Write results to the report area.
+        # Now write the result files for this run into its result directory.
+        mk_path(self.results_dir)
         self.md_report_path.write_text(self.results_md)
         self.html_report_path.write_text(self.results_html)
 

util/dvsim/Launcher.py

@@ -172,7 +172,7 @@ def _make_odir(self):
 
         # If renew_odir flag is True - then move it.
         if self.renew_odir:
-            clean_odirs(odir=self.deploy.odir, max_odirs=self.max_odirs)
+            clean_odirs(new_odir=self.deploy.odir, max_odirs=self.max_odirs)
         os.makedirs(self.deploy.odir, exist_ok=True)
 
     def _link_odir(self, status):

util/dvsim/LsfLauncher.py

@@ -63,7 +63,7 @@ def prepare_workspace_for_cfg(cfg):
         # Create the job dir.
         LsfLauncher.jobs_dir[cfg] = Path(cfg.scratch_path, "lsf",
                                          cfg.timestamp)
-        clean_odirs(odir=LsfLauncher.jobs_dir[cfg], max_odirs=2)
+        clean_odirs(new_odir=LsfLauncher.jobs_dir[cfg], max_odirs=2)
         os.makedirs(Path(LsfLauncher.jobs_dir[cfg]), exist_ok=True)
 
     @staticmethod

util/dvsim/utils.py

@@ -612,40 +612,57 @@ def mk_symlink(path, link):
             rm_path(link)
 
 
-def clean_odirs(odir, max_odirs, ts_format=TS_FORMAT):
-    """Clean previous output directories.
-
-    When running jobs, we may want to maintain a limited history of
-    previous invocations. This method finds and deletes the output
-    directories at the base of input arg 'odir' with the oldest timestamps,
-    if that limit is reached. It returns a list of directories that
-    remain after deletion.
+def clean_odirs(new_odir, max_odirs) -> list[Path]:
+    """Cleanup previous output directories.
+
+    When running jobs, we may want to maintain a limited history of previous run results.
+    This function cleans up the location where output directories are stored, making space
+    for the 'new_odir' to be created in a follow-up step.
+    This method does NOT create the new output directory.
+
+    This method:
+    - If an output directory already exists at the path of 'new_odir', move it to
+      a new path based on it's creation time
+    - If more than 'max_odirs' directories are present in the parent directory:
+      - Deletes directories with the oldest timestamps until only 'max_odirs'
+        are remaining
+
+    Args:
+        new_odir: The new output directory we want to create (after this routine completes)
+        max_odirs: Sets the limit of existing output dirs to keep.
+
+    Returns:
+        A list of output directories that remain after deletion.
     """
 
-    odir = Path(odir)
-
-    if os.path.exists(odir):
-        # If output directory exists, back it up.
-        ts = datetime.fromtimestamp(os.stat(odir).st_ctime).strftime(ts_format)
-        # Prior to Python 3.9, shutil may run into an error when passing in
-        # Path objects (see https://bugs.python.org/issue32689). While this
-        # has been fixed in Python 3.9, string casts are added so that this
-        # also works with older versions.
-        shutil.move(str(odir), str(odir.with_name(ts)))
-
-    # Get list of past output directories sorted by creation time.
-    pdir = odir.resolve().parent
+    pdir = Path(new_odir).parent.resolve()
     if not pdir.exists():
+        # Parent directory of the new output directory does not exist, hence there is nothing
+        # to clean.
         return []
 
-    dirs = sorted([old for old in pdir.iterdir() if (old.is_dir() and
-                                                     old != 'summary')],
+    # If there is an existing directory at the path of 'new_odir', move it now.
+    odir = Path(new_odir).resolve()
+    if odir.exists():
+        # Move it to a new unique path (based on its creation time).
+        ts = datetime.fromtimestamp(os.stat(odir).st_ctime).strftime(TS_FORMAT)
+        shutil.move(src=str(odir),
+                    dst=str(odir.with_name(ts)))
+
+    # Now prune existing output directories based on 'max_odirs'
+
+    # Get a list of all past output directories sorted by creation time. This allows us to
+    # filter and delete them by age.
+    dirs = sorted([odir for odir in pdir.iterdir()
+                   # Only filter for directories, and exclude any 'summary' directories
+                   if (odir.is_dir() and odir != 'summary')],
                   key=os.path.getctime,
                   reverse=True)
-
+    # Delete the oldest output directories until we have at most 'max_odirs' remaining
     for old in dirs[max(0, max_odirs - 1):]:
         shutil.rmtree(old, ignore_errors=True)
 
+    # Finally, return a list of remaining output directories after the cleanup.
     return [] if max_odirs == 0 else dirs[:max_odirs - 1]