diff --git a/configs/small-test-s3.yaml b/configs/small-test-s3.yaml
index 2c37d4805..82bf57ee4 100644
--- a/configs/small-test-s3.yaml
+++ b/configs/small-test-s3.yaml
@@ -7,9 +7,9 @@ wandb:
   project: olmo-small-test
 
 model:
-  d_model: 1024
+  d_model: 512
   n_heads: 16
-  n_layers: 12
+  n_layers: 8
   mlp_ratio: 8
   weight_tying: false
   alibi: false