Kick off anealing run for new 7b model at 2T.

allenai · Apr 9, 2024 · 4ee0089 · 4ee0089
1 parent 1bec77f
commit 4ee0089
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 10 deletions.
diff --git a/...new_7b-step_1T-warmup_true-steps_50B.yaml → ...g/v1.7_step_2T-warmup_true-steps_50B.yaml b/...new_7b-step_1T-warmup_true-steps_50B.yaml → ...g/v1.7_step_2T-warmup_true-steps_50B.yaml
@@ -1,11 +1,11 @@
-run_name: v0-new_7b-step_1T-warmup_true-steps_50B
+run_name: v1.7-step_2T-warmup_true-steps_50B
 seed: 61394
 dry_run: false
 
 wandb:
   name: ${run_name}
   project: olmo-annealing
-  group: v0-new_7b-step_1T-warmup_true-steps_50B
+  group: v1.7-step_2T-warmup_true-steps_50B
 
 model:
   d_model: 4096
@@ -69,8 +69,8 @@ save_num_unsharded_checkpoints_to_keep: -1
 
 restore_dataloader: false
 
-# 1T token checkpoint for new 7B model
-load_path: s3://ai2-llm/checkpoints/OLMo-medium/mitchish7/step239000-unsharded
+# 2T token checkpoint for new 7B model; we call this v1.7 to match the Dolma data.
+load_path: s3:///ai2-llm/checkpoints/OLMo-medium/mitchish7/step477000-unsharded
 
 no_pre_train_checkpoint: true
 reset_optimizer_state: true

diff --git a/...0-new_7b-step_1T-warmup_true-steps_50B.sh → ...ing/v1.7-step_2T-warmup_true-steps_50B.sh b/...0-new_7b-step_1T-warmup_true-steps_50B.sh → ...ing/v1.7-step_2T-warmup_true-steps_50B.sh
@@ -2,17 +2,17 @@
 
 set -ex
 
-CONFIG_PATH=configs/annealing/v0-new_7b-step_1T-warmup_true-steps_50B.yaml
-NUM_NODES=8
+CONFIG_PATH=configs/annealing/v1.7-step_2T-warmup_true-steps_50B.yaml
+NUM_NODES=4
 
 gantry run \
   --allow-dirty \
   --workspace ai2/davidw-oe-annealing \
-  --task-name v0-new_7b-step_1T-warmup_true-steps_50B \
-  --description v0-new_7b-step_1T-warmup_true-steps_50B \
-  --priority normal \
+  --task-name v1.7-step_2T-warmup_true-steps_50B \
+  --description v1.7-step_2T-warmup_true-steps_50B \
+  --priority high \
   --beaker-image petew/olmo-torch2-gantry \
-  --cluster ai2/pluto-cirrascale \
+  --cluster ai2/general-cirrascale-a100-80g-ib \
   --gpus 8 \
   --replicas "${NUM_NODES}" \
   --leader-selection \