fix scripts (#235)

modelscope · Mar 11, 2024 · da6440a · da6440a
1 parent 9a3db9c
commit da6440a
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/scripts/dlc/partition_data_dlc.py b/scripts/dlc/partition_data_dlc.py
@@ -28,7 +28,7 @@ def partition_data(json_file_path: str, hostnames: List[str]):
         nodes_video_size[min_node] += video_sizes[video]
 
     for hostname in hostnames:
-        host_file_path = f"{json_file_path.rsplit('.', 1)[0]}_{hostname}.json"
+        host_file_path = f"{json_file_path.rsplit('.', 1)[0]}_{hostname}.jsonl"
         with open(host_file_path, 'w') as f:
             for entry in nodes_data[hostname]:
                 f.write(json.dumps(entry) + '\n')

diff --git a/scripts/dlc/run_on_dlc.sh b/scripts/dlc/run_on_dlc.sh
@@ -36,5 +36,13 @@ else
     sed -i$SED_I_SUFFIX "s|\(dataset_path: '\)\(.*\)'\(.*\)|\1\2_$hostname'\3|" "$new_config_file"
 fi
 
+if grep -q "export_path: .*\.json" "$new_config_file"; then
+    # .json data_path
+    sed -i$SED_I_SUFFIX "s|\(export_path: \)\(.*\)\(/[^/]*\)\(.json\)|\1\2\3_$hostname\4|" "$new_config_file"
+else
+    # dir export_path
+    sed -i$SED_I_SUFFIX "s|\(export_path: '\)\(.*\)'\(.*\)|\1\2_$hostname'\3|" "$new_config_file"
+fi
+
 # run to process data
 python tools/process_data.py --config "$new_config_file"