context size

Lightning-AI · Apr 19, 2024 · 8eb4e11 · 8eb4e11
1 parent b7defe4
commit 8eb4e11
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/litgpt/config.py b/litgpt/config.py
@@ -846,6 +846,7 @@ def norm_class(self) -> Type:
     dict(
         name="Llama-3-8B{}",
         hf_config=dict(org="meta-llama", name="Meta-Llama-3-8B{}"),
+        block_size=8192,
         vocab_size=128256,
         padding_multiple=64,
         n_layer=32,
@@ -863,6 +864,7 @@ def norm_class(self) -> Type:
     dict(
         name="Llama-3-70B{}",
         hf_config=dict(org="meta-llama", name="Meta-Llama-3-70B{}"),
+        block_size=8192,
         vocab_size=128256,
         padding_multiple=64,
         n_layer=80,