From ef615708c812a41fee2749a7f4759de47dc5e1a6 Mon Sep 17 00:00:00 2001 From: Harry Callahan Date: Sun, 18 Aug 2024 20:13:05 +0100 Subject: [PATCH] [dvsim] Cleanup FlowCfg::write_results() / utils::clean_odirs() Signed-off-by: Harry Callahan --- util/dvsim/Deploy.py | 2 +- util/dvsim/FlowCfg.py | 20 ++++++------ util/dvsim/Launcher.py | 2 +- util/dvsim/LsfLauncher.py | 2 +- util/dvsim/utils.py | 65 ++++++++++++++++++++++++--------------- 5 files changed, 55 insertions(+), 36 deletions(-) diff --git a/util/dvsim/Deploy.py b/util/dvsim/Deploy.py index b5292116ca98d3..99466aa363e6d7 100644 --- a/util/dvsim/Deploy.py +++ b/util/dvsim/Deploy.py @@ -673,7 +673,7 @@ def __init__(self, run_items, sim_cfg): sim_cfg.__dict__) # Prune previous merged cov directories, keeping only the lastest 7. - prev_cov_db_dirs = clean_odirs(odir=self.cov_merge_db_dir, max_odirs=7) + prev_cov_db_dirs = clean_odirs(new_odir=self.cov_merge_db_dir, max_odirs=7) # If the --cov-merge-previous command line switch is passed, then # merge coverage with the previous runs. diff --git a/util/dvsim/FlowCfg.py b/util/dvsim/FlowCfg.py index c4b4979fa3f2ad..005119872a1bc7 100644 --- a/util/dvsim/FlowCfg.py +++ b/util/dvsim/FlowCfg.py @@ -488,18 +488,20 @@ def gen_results_summary(self): def write_results(self) -> None: """Write results to files. - This function converts text_md to HTML and writes the result to a file - in self.results_dir with the file name given by html_filename. If - json_str is not None, this function additionally writes json_str to a - file with the same path and base name as the HTML file but with '.json' - as suffix. + This function: + - Writes self.results_md to a file + - Converts self.results_md to HTML, and write it to a file + + If the intended new results directory already exists, rename it based on + a timestamp of when it was created. Then create the new directory again + and write the new results into it. """ - # Prepare reports directory, keeping 90 day history. - clean_odirs(odir=self.results_dir, max_odirs=89) - mk_path(self.results_dir) + # Prepare reports directory, keeping the 90 newest results. + clean_odirs(new_odir=self.results_dir, max_odirs=89) - # Write results to the report area. + # Now write the result files for this run into its result directory. + mk_path(self.results_dir) self.md_report_path.write_text(self.results_md) self.html_report_path.write_text(self.results_html) diff --git a/util/dvsim/Launcher.py b/util/dvsim/Launcher.py index a483e94e67db01..52175cc4a28cf5 100644 --- a/util/dvsim/Launcher.py +++ b/util/dvsim/Launcher.py @@ -172,7 +172,7 @@ def _make_odir(self): # If renew_odir flag is True - then move it. if self.renew_odir: - clean_odirs(odir=self.deploy.odir, max_odirs=self.max_odirs) + clean_odirs(new_odir=self.deploy.odir, max_odirs=self.max_odirs) os.makedirs(self.deploy.odir, exist_ok=True) def _link_odir(self, status): diff --git a/util/dvsim/LsfLauncher.py b/util/dvsim/LsfLauncher.py index 2260a01752d2d2..8030f1fb12bee9 100644 --- a/util/dvsim/LsfLauncher.py +++ b/util/dvsim/LsfLauncher.py @@ -63,7 +63,7 @@ def prepare_workspace_for_cfg(cfg): # Create the job dir. LsfLauncher.jobs_dir[cfg] = Path(cfg.scratch_path, "lsf", cfg.timestamp) - clean_odirs(odir=LsfLauncher.jobs_dir[cfg], max_odirs=2) + clean_odirs(new_odir=LsfLauncher.jobs_dir[cfg], max_odirs=2) os.makedirs(Path(LsfLauncher.jobs_dir[cfg]), exist_ok=True) @staticmethod diff --git a/util/dvsim/utils.py b/util/dvsim/utils.py index 8f0677186f8a2d..842422f70650a1 100644 --- a/util/dvsim/utils.py +++ b/util/dvsim/utils.py @@ -612,40 +612,57 @@ def mk_symlink(path, link): rm_path(link) -def clean_odirs(odir, max_odirs, ts_format=TS_FORMAT): - """Clean previous output directories. - - When running jobs, we may want to maintain a limited history of - previous invocations. This method finds and deletes the output - directories at the base of input arg 'odir' with the oldest timestamps, - if that limit is reached. It returns a list of directories that - remain after deletion. +def clean_odirs(new_odir, max_odirs) -> list[Path]: + """Cleanup previous output directories. + + When running jobs, we may want to maintain a limited history of previous run results. + This function cleans up the location where output directories are stored, making space + for the 'new_odir' to be created in a follow-up step. + This method does NOT create the new output directory. + + This method: + - If an output directory already exists at the path of 'new_odir', move it to + a new path based on it's creation time + - If more than 'max_odirs' directories are present in the parent directory: + - Deletes directories with the oldest timestamps until only 'max_odirs' + are remaining + + Args: + new_odir: The new output directory we want to create (after this routine completes) + max_odirs: Sets the limit of existing output dirs to keep. + + Returns: + A list of output directories that remain after deletion. """ - odir = Path(odir) - - if os.path.exists(odir): - # If output directory exists, back it up. - ts = datetime.fromtimestamp(os.stat(odir).st_ctime).strftime(ts_format) - # Prior to Python 3.9, shutil may run into an error when passing in - # Path objects (see https://bugs.python.org/issue32689). While this - # has been fixed in Python 3.9, string casts are added so that this - # also works with older versions. - shutil.move(str(odir), str(odir.with_name(ts))) - - # Get list of past output directories sorted by creation time. - pdir = odir.resolve().parent + pdir = Path(new_odir).parent.resolve() if not pdir.exists(): + # Parent directory of the new output directory does not exist, hence there is nothing + # to clean. return [] - dirs = sorted([old for old in pdir.iterdir() if (old.is_dir() and - old != 'summary')], + # If there is an existing directory at the path of 'new_odir', move it now. + odir = Path(new_odir).resolve() + if odir.exists(): + # Move it to a new unique path (based on its creation time). + ts = datetime.fromtimestamp(os.stat(odir).st_ctime).strftime(TS_FORMAT) + shutil.move(src=str(odir), + dst=str(odir.with_name(ts))) + + # Now prune existing output directories based on 'max_odirs' + + # Get a list of all past output directories sorted by creation time. This allows us to + # filter and delete them by age. + dirs = sorted([odir for odir in pdir.iterdir() + # Only filter for directories, and exclude any 'summary' directories + if (odir.is_dir() and odir != 'summary')], key=os.path.getctime, reverse=True) - + # Delete the oldest output directories until we have at most 'max_odirs' remaining for old in dirs[max(0, max_odirs - 1):]: shutil.rmtree(old, ignore_errors=True) + # Finally, return a list of remaining output directories after the cleanup. return [] if max_odirs == 0 else dirs[:max_odirs - 1]