diff --git a/pyproject.toml b/pyproject.toml
index 93387840..ca053385 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,7 @@ package-dir = { "" = "src" }
 dependencies = { file = ["requirements.txt"] }
 optional-dependencies.cuda = { file = ["requirements-cuda.txt"] }
 optional-dependencies.rocm = { file = ["requirements-rocm.txt"] }
+optional-dependencies.hpu = { file = ["requirements-hpu.txt"] }
 
 [tool.setuptools.packages.find]
 where = ["src"]
diff --git a/requirements-cuda.txt b/requirements-cuda.txt
index 1bed61eb..6230797e 100644
--- a/requirements-cuda.txt
+++ b/requirements-cuda.txt
@@ -1,2 +1,5 @@
 flash-attn>=2.4.0
 bitsandbytes>=0.43.1
+
+# required for FSDP updates
+accelerate>=0.34.2
diff --git a/requirements-hpu.txt b/requirements-hpu.txt
new file mode 100644
index 00000000..6c9ca0c2
--- /dev/null
+++ b/requirements-hpu.txt
@@ -0,0 +1,2 @@
+# required for optimum-habana's deps
+accelerate>=0.33.0
\ No newline at end of file
diff --git a/requirements-rocm.txt b/requirements-rocm.txt
index 21612c36..9d72f4d5 100644
--- a/requirements-rocm.txt
+++ b/requirements-rocm.txt
@@ -1 +1,3 @@
 flash-attn>=2.6.2,<2.7.0
+# required for FSDP updates
+accelerate>=0.34.2
diff --git a/requirements.txt b/requirements.txt
index 84da8460..7c6a00ca 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ py-cpuinfo
 # replace custom pytorch images with the 2.3.0
 torch>=2.3.0a0
 transformers>=4.45.2
-accelerate>=0.34.2
+
 datasets>=2.15.0
 numba
 # Note: numpy ranges copied from instructlab/instructlab
@@ -22,6 +22,6 @@ trl>=0.9.4
 peft
 pydantic>=2.7.0
 
-# deepspeed needs to be at the bottom or it'll break during installation
+# deepspeed needs to be at the end or it'll break stuff.
 deepspeed>=0.14.3
 aiofiles>=23.2.1