Falcon3: minor fix on config.py

Lightning-AI · Dec 22, 2024 · 307e4b4 · 307e4b4
1 parent 33ad8af
commit 307e4b4
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/litgpt/config.py b/litgpt/config.py
@@ -454,6 +454,7 @@ def norm_class(self) -> Type:
         n_query_groups=4,
         n_embd=2048,
         rotary_percentage=1.0,
+        parallel_residual=False,
         rope_base=1000042,
         norm_eps=1e-6,
         bias=False,
@@ -473,6 +474,7 @@ def norm_class(self) -> Type:
         n_query_groups=4,
         n_embd=3072,
         rotary_percentage=1.0,
+        parallel_residual=False,
         rope_base=1000042,
         norm_eps=1e-6,
         bias=False,
@@ -492,6 +494,7 @@ def norm_class(self) -> Type:
         n_query_groups=4,
         n_embd=3072,
         rotary_percentage=1.0,
+        parallel_residual=False,
         rope_base=1000042,
         norm_eps=1e-6,
         bias=False,
@@ -511,6 +514,7 @@ def norm_class(self) -> Type:
         n_query_groups=4,
         n_embd=3072,
         rotary_percentage=1.0,
+        parallel_residual=False,
         rope_base=1000042,
         norm_eps=1e-6,
         bias=False,