back to local disk, still 11 workers and b256

mwalmsley · Nov 6, 2023 · dece4ab · dece4ab
1 parent 666241d
commit dece4ab
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/only_for_me/narval/train.py b/only_for_me/narval/train.py
@@ -126,8 +126,8 @@
         num_workers=11,
         random_state=random_state,
         learning_rate=1e-3,
-        # cache_dir=os.environ['SLURM_TMPDIR'] + '/cache'
-        cache_dir='/tmp/cache'
+        cache_dir=os.environ['SLURM_TMPDIR'] + '/cache'
+        # cache_dir='/tmp/cache'
         # /tmp for ramdisk (400GB total, vs 4TB total for nvme)
     )
 

diff --git a/only_for_me/narval/train.sh b/only_for_me/narval/train.sh
@@ -10,8 +10,8 @@ nvidia-smi
 
 PYTHON=/home/walml/envs/zoobot39_dev/bin/python
 
-# mkdir $SLURM_TMPDIR/cache
-mkdir /tmp/cache
+mkdir $SLURM_TMPDIR/cache
+# mkdir /tmp/cache
 
 # export NCCL_BLOCKING_WAIT=1  #Set this environment variable if you wish to use the NCCL backend for inter-GPU communication.
 # export MASTER_ADDR=$(hostname) #Store the master node’s IP address in the MASTER_ADDR environment variable.