fix: switch vulkan images to the kompute backend

The latest versions of MoltenVK (tested with 1.2.11) are not able to deal properly with llama.cpp's Vulkan (GGML_VULKAN) backend. Switch our Vulkan images to use the Kompute (GGML_KOMPUTE) backend instead, which works fine with MoltenVK and will eventually achieve the same performance as Metal on macOS bare-metal. Signed-off-by: Sergio Lopez <[email protected]>
containers · Dec 11, 2024 · 665759f · 665759f
1 parent f1df6c3
commit 665759f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/chat/vulkan/amd64/Containerfile b/chat/vulkan/amd64/Containerfile
@@ -24,7 +24,7 @@ RUN dnf install -y python3-dnf-plugin-versionlock && \
     dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     dnf install -y git cmake ninja-build gcc gcc-c++ && \
     dnf copr enable -y ligenix/enterprise-sandbox epel-9-x86_64 && \
-    dnf install -y vulkan-headers vulkan-tools vulkan-loader-devel && \
+    dnf install -y vulkan-headers vulkan-tools vulkan-loader-devel fmt-devel && \
     dnf copr enable -y ligenix/enterprise-buildtools epel-9-x86_64 && \
     dnf copr enable -y jeffmaury/shaderc epel-9-x86_64 && \
     dnf install -y glslc && \
@@ -34,7 +34,7 @@ WORKDIR /locallm
 COPY requirements.txt ./
 COPY llama-cpp-python llama-cpp-python
 RUN pip install --upgrade pip
-ENV CMAKE_ARGS="-DGGML_VULKAN=on"
+ENV CMAKE_ARGS="-DGGML_KOMPUTE=1 -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF"
 ENV FORCE_CMAKE=1
 RUN pip install --target=/locallm --no-cache-dir --upgrade -r requirements.txt
 

diff --git a/chat/vulkan/arm64/Containerfile b/chat/vulkan/arm64/Containerfile
@@ -25,14 +25,14 @@ https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     dnf copr enable -y slp/mesa-krunkit epel-9-aarch64 && \
     dnf install -y mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
     dnf versionlock  mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
-    dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools && \
+    dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools fmt-devel && \
     dnf copr enable -y jeffmaury/shaderc epel-9-aarch64 && \
     dnf install -y glslc && \
     dnf clean all
 COPY requirements.txt ./
 COPY llama-cpp-python llama-cpp-python
 RUN pip install --upgrade pip
-ENV CMAKE_ARGS="-DGGML_VULKAN=on"
+ENV CMAKE_ARGS="-DGGML_KOMPUTE=1 -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF"
 ENV FORCE_CMAKE=1
 RUN pip install --target=/locallm --no-cache-dir --upgrade -r requirements.txt