diff --git a/pyproject.toml b/pyproject.toml
index c55795eb1378dd..37bc1d7efe16da 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,6 +18,6 @@ line-length = 119
 lines-after-imports = 2
 known-first-party = ["transformers"]
 
-# This is ignored, maybe because of the header? If someone finds a fix, we can uncomment and remove setup.cfg
-# [tool.pytest]
-# doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
+[tool.pytest.ini_options]
+doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
+doctest_glob="**/*.md"
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index ffe8973dd21c61..00000000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-[tool:pytest]
-doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS
-doctest_glob=**/*.md
\ No newline at end of file