diff --git a/.github/lm-eval-configs/smoke-small-models.txt b/.github/lm-eval-configs/smoke-small-models.txt
index 9950a0d4fc01d..d884f36672a74 100644
--- a/.github/lm-eval-configs/smoke-small-models.txt
+++ b/.github/lm-eval-configs/smoke-small-models.txt
@@ -1,2 +1 @@
 Meta-Llama-3-8B-Instruct.yaml
-Qwen2-7B-Instruct.yaml