Reducing deepspeed timeout to 10mins (#132)

Signed-off-by: Mustafa Eyceoz <[email protected]>
instructlab · Jul 9, 2024 · 2b744af · 2b744af
1 parent 22639da
commit 2b744af
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/instructlab/training/main_ds.py b/src/instructlab/training/main_ds.py
@@ -490,7 +490,7 @@ def main(args):
     #### distributed init #####
     torch.cuda.set_device(int(os.environ["LOCAL_RANK"]))
     args.local_rank = int(os.environ["LOCAL_RANK"])
-    deepspeed.init_distributed(timeout=timedelta(minutes=360))
+    deepspeed.init_distributed(timeout=timedelta(minutes=10))
     args.global_rank = torch.distributed.get_rank()
     tensor = torch.ByteTensor([False]).cuda()
     torch.distributed.all_reduce(tensor)