From 665759fd6950fec9387777927a36306887a67cbd Mon Sep 17 00:00:00 2001 From: Sergio Lopez Date: Thu, 28 Nov 2024 18:39:10 +0100 Subject: [PATCH] fix: switch vulkan images to the kompute backend The latest versions of MoltenVK (tested with 1.2.11) are not able to deal properly with llama.cpp's Vulkan (GGML_VULKAN) backend. Switch our Vulkan images to use the Kompute (GGML_KOMPUTE) backend instead, which works fine with MoltenVK and will eventually achieve the same performance as Metal on macOS bare-metal. Signed-off-by: Sergio Lopez --- chat/vulkan/amd64/Containerfile | 4 ++-- chat/vulkan/arm64/Containerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chat/vulkan/amd64/Containerfile b/chat/vulkan/amd64/Containerfile index c4ffdad..b10b43e 100644 --- a/chat/vulkan/amd64/Containerfile +++ b/chat/vulkan/amd64/Containerfile @@ -24,7 +24,7 @@ RUN dnf install -y python3-dnf-plugin-versionlock && \ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ dnf install -y git cmake ninja-build gcc gcc-c++ && \ dnf copr enable -y ligenix/enterprise-sandbox epel-9-x86_64 && \ - dnf install -y vulkan-headers vulkan-tools vulkan-loader-devel && \ + dnf install -y vulkan-headers vulkan-tools vulkan-loader-devel fmt-devel && \ dnf copr enable -y ligenix/enterprise-buildtools epel-9-x86_64 && \ dnf copr enable -y jeffmaury/shaderc epel-9-x86_64 && \ dnf install -y glslc && \ @@ -34,7 +34,7 @@ WORKDIR /locallm COPY requirements.txt ./ COPY llama-cpp-python llama-cpp-python RUN pip install --upgrade pip -ENV CMAKE_ARGS="-DGGML_VULKAN=on" +ENV CMAKE_ARGS="-DGGML_KOMPUTE=1 -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF" ENV FORCE_CMAKE=1 RUN pip install --target=/locallm --no-cache-dir --upgrade -r requirements.txt diff --git a/chat/vulkan/arm64/Containerfile b/chat/vulkan/arm64/Containerfile index 4351c3a..31af112 100644 --- a/chat/vulkan/arm64/Containerfile +++ b/chat/vulkan/arm64/Containerfile @@ -25,14 +25,14 @@ https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ dnf copr enable -y slp/mesa-krunkit epel-9-aarch64 && \ dnf install -y mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \ dnf versionlock mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \ - dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools && \ + dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools fmt-devel && \ dnf copr enable -y jeffmaury/shaderc epel-9-aarch64 && \ dnf install -y glslc && \ dnf clean all COPY requirements.txt ./ COPY llama-cpp-python llama-cpp-python RUN pip install --upgrade pip -ENV CMAKE_ARGS="-DGGML_VULKAN=on" +ENV CMAKE_ARGS="-DGGML_KOMPUTE=1 -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF" ENV FORCE_CMAKE=1 RUN pip install --target=/locallm --no-cache-dir --upgrade -r requirements.txt