From 18da97174a6bb464150e3b699dde004da038d050 Mon Sep 17 00:00:00 2001 From: Vivian Chen Date: Wed, 7 Aug 2024 18:46:11 +0000 Subject: [PATCH] remove unrelated changes Signed-off-by: Vivian Chen --- .../multimodal_llm/neva/sequence_packing/preprocess_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/multimodal/multimodal_llm/neva/sequence_packing/preprocess_dataset.py b/examples/multimodal/multimodal_llm/neva/sequence_packing/preprocess_dataset.py index b670d171fd1d..60f882fa9821 100644 --- a/examples/multimodal/multimodal_llm/neva/sequence_packing/preprocess_dataset.py +++ b/examples/multimodal/multimodal_llm/neva/sequence_packing/preprocess_dataset.py @@ -271,7 +271,6 @@ def main(): logging.info(f"Output directory: {output_dir}") prefix_path = f"{output_dir}/packed_seq_dataset" - os.makedirs(prefix_path, exist_ok=True) # Original Datasets to Sequence Lengths Files builders = {} for item_dict in tqdm(train_dl, desc="Building indexed datasets"):