diff --git a/src/instructlab/training/main_ds.py b/src/instructlab/training/main_ds.py index 2dbfa3f4..8b12ee7e 100644 --- a/src/instructlab/training/main_ds.py +++ b/src/instructlab/training/main_ds.py @@ -490,7 +490,7 @@ def main(args): #### distributed init ##### torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) args.local_rank = int(os.environ["LOCAL_RANK"]) - deepspeed.init_distributed(timeout=timedelta(minutes=360)) + deepspeed.init_distributed(timeout=timedelta(minutes=10)) args.global_rank = torch.distributed.get_rank() tensor = torch.ByteTensor([False]).cuda() torch.distributed.all_reduce(tensor)