try 2 node run

mwalmsley · Nov 3, 2023 · 257b4dc · 257b4dc
1 parent adf9f26
commit 257b4dc
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/only_for_me/narval/finetune.py b/only_for_me/narval/finetune.py
@@ -61,7 +61,7 @@
         os.path.join(os.environ['SLURM_TMPDIR'], 'walml/finetune/checkpoints'),
         accelerator='gpu',
         devices=2,
-        num_nodes=1,
+        num_nodes=2,
         strategy='ddp',
         precision='16-mixed',
         max_epochs=max_epochs,

diff --git a/only_for_me/narval/finetune.sh b/only_for_me/narval/finetune.sh
@@ -1,11 +1,18 @@
 #!/bin/bash
 #SBATCH --mem=32G
-#SBATCH --nodes=1
+#SBATCH --nodes=2
 #SBATCH --time=0:20:0  
 #SBATCH --tasks-per-node=2 
 #SBATCH --cpus-per-task=12
 #SBATCH --gres=gpu:a100:2
 
+#### SBATCH --mem=32G
+#### SBATCH --nodes=1
+#### SBATCH --time=0:20:0  
+#### SBATCH --tasks-per-node=2 
+#### SBATCH --cpus-per-task=12
+#### SBATCH --gres=gpu:a100:2
+
 #### 
 #### SBATCH --mem=16G
 #### SBATCH --nodes=1

diff --git a/only_for_me/narval/narval.md b/only_for_me/narval/narval.md
@@ -8,6 +8,7 @@ https://prashp.gitlab.io/post/compute-canada-tut/
 https://docs.alliancecan.ca/wiki/Python
 
 ssh [email protected]
+ssh-copy-id to avoid password in future
 
     module purge
     module avail
@@ -51,8 +52,12 @@ and my own cloned repos
 pip install --no-deps -e galaxy-datasets
 pip install --no-deps -e zoobot
 
+Run training
+
+sbatch only_for_me/narval/finetune.sh
+
+Works with simple images on multi-GPU, single node
 
-Multi-node notes
 
 https://lightning.ai/docs/pytorch/stable/clouds/cluster_intermediate_2.html#
 https://pytorch.org/docs/stable/elastic/run.html#environment-variables