- bugfix for un-aligned types (int v.s. float) caused by two OPs

- improve the usability for logfile_name when running several exps in the same workdir
modelscope · Dec 20, 2023 · 4320177 · 4320177
1 parent a137b30
commit 4320177
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/data_juicer/config/config.py b/data_juicer/config/config.py
@@ -324,14 +324,14 @@ def init_setup_from_cfg(cfg):
     """
 
     cfg.export_path = os.path.abspath(cfg.export_path)
-    export_path = cfg.export_path
-    cfg.work_dir = os.path.dirname(export_path)
+    cfg.work_dir = os.path.dirname(cfg.export_path)
+    export_rel_path = os.path.relpath(cfg.export_path, start=cfg.work_dir)
     log_dir = os.path.join(cfg.work_dir, 'log')
     if not os.path.exists(log_dir):
         os.makedirs(log_dir, exist_ok=True)
     timestamp = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
     cfg.timestamp = timestamp
-    logfile_name = f'export_{export_path}_time_{timestamp}.txt'
+    logfile_name = f'export_{export_rel_path}_time_{timestamp}.txt'
     setup_logger(save_dir=log_dir,
                  filename=logfile_name,
                  redirect=cfg.executor_type == 'default')