abstracting model downloads and file-normalization to models dir

Signed-off-by: greg pereira <[email protected]>
containers · Apr 9, 2024 · 1c0bc90 · 1c0bc90
1 parent 2fd9933
commit 1c0bc90
Show file tree

Hide file tree

Showing 14 changed files with 213 additions and 133 deletions.
diff --git a/.github/workflows/model_servers.yaml b/.github/workflows/model_servers.yaml
@@ -57,7 +57,7 @@ jobs:
 
       - name: Download model
         working-directory: ./model_servers/${{ matrix.image_name }}/
-        run: make ${{ matrix.model }}
+        run: make trigger-download-model-${{ matrix.model }}
 
       - name: Set up Python
         uses: actions/[email protected]
@@ -71,8 +71,6 @@ jobs:
       - name: Run tests
         working-directory: ./model_servers/${{ matrix.image_name }}/
         run: make test
-        env:
-          IMAGE_NAME: ${{ matrix.image_name }}
 
       - name: Login to Container Registry
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'

diff --git a/.gitignore b/.gitignore
@@ -4,8 +4,7 @@
 port_check.lock
 *build
 models/*
-model_servers/llamacpp_python/model.gguf
+model_servers/*/model.file
 !models/convert_models/*
-!models/Containerfile
 !models/README.md
 recipes/chromedriver
diff --git a/model_servers/common/Makefile.common b/model_servers/common/Makefile.common
@@ -0,0 +1,37 @@
+REGISTRY ?= quay.io
+REGISTRY_ORG ?= ai-lab
+COMPONENT_NAME ?= model_servers
+
+BIND_MOUNT_OPTIONS := ro
+OS := $(shell uname -s)
+ifeq ($(OS),Linux)
+    BIND_MOUNT_OPTIONS := Z,ro
+endif
+
+.PHONY: build
+build:
+	podman build --squash-all -t $(IMAGE) . -f base/Containerfile
+
+.PHONY: install
+install:
+	pip install -r tests/requirements.txt
+
+.PHONY: run-test
+run-test:
+	REGISTRY=$(REGISTRY) IMAGE_NAME=$(IMAGE_NAME) MODEL_NAME=$(MODEL_NAME) pytest --log-cli-level NOTSET
+
+.PHONY: test
+test:
+	@if [ ! -f "../../models/$(MODEL_NAME)" ]; then \
+		echo "Model file -- $(MODEL_NAME) -- not present in the models directory."; \
+		exit 1; \
+	else \
+        if [ ! -f "./$(MODEL_NAME)" ]; then \
+            ln -s ../../models/$(MODEL_NAME) ./$(MODEL_NAME); \
+        fi; \
+		$(MAKE) REGISTRY=$(REGISTRY) IMAGE_NAME=$(IMAGE_NAME) MODEL_NAME=$(MODEL_NAME)  -k run-test clean; \
+	fi;
+
+.PHONY: clean
+clean:
+	- rm ./$(MODEL_NAME) &> /dev/null
diff --git a/model_servers/llamacpp_python/Makefile b/model_servers/llamacpp_python/Makefile
@@ -1,41 +1,20 @@
 APP := llamacpp_python
-PORT := 8001
+PORT ?= 8001
 
-IMAGE := quay.io/ai-lab/$(APP):latest
-CUDA_IMAGE := quay.io/ai-lab/$(APP)_cuda:latest
-VULKAN_IMAGE := quay.io/ai-lab/$(APP)_vulkan:latest
+include ../common/Makefile.common
 
-# ----- MODEL OPTIONS -----
-
-LLAMA_MODEL_NAME := llama-2-7b-chat.Q5_K_S.gguf
-LLAMA_MODEL_URL := https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf
-
-TINY_LLAMA_MODEL_NAME := tinyllama-1.1b-chat-v1.0.Q4_K_S.gguf
-TINY_LLAMA_MODEL_URL := https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/blob/main/tinyllama-1.1b-chat-v1.0.Q4_K_S.gguf
+IMAGE_NAME ?= $(REGISTRY_ORG)/$(COMPONENT_NAME)/$(APP):latest
+IMAGE := $(REGISTRY)/$(IMAGE_NAME)
+CUDA_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT_NAME)/$(APP)_cuda:latest
+VULKAN_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT_NAME)/$(APP)_vulkan:latest
 
-MISTRAL_MODEL_NAME := mistral-7b-instruct-v0.1.Q4_K_M.gguf
-MISTRAL_MODEL_URL := https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q4_K_M.gguf
-
-# --- END MODEL OPTIONS ---
-
-SELECTED_MODEL_NAME := $(or $(SELECTED_MODEL),$(MISTRAL_MODEL_NAME))
-SELECTED_MODEL_URL := $(or $(SELECTED_MODEL_LINK),$(MISTRAL_MODEL_URL))
+# ----- MODEL OPTIONS -----
 
-RELATIVE_MODELS_PATH := ../../models
 MODELS_PATH := /locallm/models
-
-BIND_MOUNT_OPTIONS := ro
-OS := $(shell uname -s)
-ifeq ($(OS),Linux)
-    BIND_MOUNT_OPTIONS := ro,Z
-endif
+MODEL_NAME ?= mistral-7b-instruct-v0.1.Q4_K_M.gguf
 
 .Phony: all
-all: build mistral run
-
-.PHONY: build
-build:
-	podman build --squash-all -t $(IMAGE) . -f base/Containerfile
+all: build trigger-download-model-mistral run
 
 .PHONY: build-cuda
 build-cuda:
@@ -45,29 +24,25 @@ build-cuda:
 build-vulkan:
 	podman build --squash-all -t $(VULKAN_IMAGE) . -f cuda/Containerfile
 
-.PHONY: download-model-tiny-llama
-download-model-tiny-llama:
-	curl -H "Cache-Control: no-cache" -s -S -L -f $(TINY_LLAMA_MODEL_URL) -z $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME) -o $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME).tmp && mv -f $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME) 2>/dev/null || rm -f $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(LLAMA_MODEL_NAME)
+########################## TRIGGER DOWNLOAD MAKE TARGETS ##########################
+### NOTE: you should not add these trigger make targets for new models.
+# Adding new models should implemented in the [models directory](../../models).
+# These are just provided here for bootstrapping and testing different model types.
 
-.PHONY: download-model-llama
-download-model-llama:
-	curl -H "Cache-Control: no-cache" -s -S -L -f $(LLAMA_MODEL_URL) -z $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME) -o $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME).tmp && mv -f $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME) 2>/dev/null || rm -f $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(TINY_LLAMA_MODEL_NAME)
+.PHONY: trigger-download-model-tiny-llama # small .gguf model for testing
+trigger-download-model-tiny-llama:
+	cd ../../models && \
+	make -f Makefile download-model-tiny-llama
 
+.PHONY: trigger-download-model-mistral # default model
+trigger-download-model-mistral:
+	cd ../../models && \
+	make -f Makefile download-model-mistral
 
-.PHONY: mistral
-mistral:
-	curl -H "Cache-Control: no-cache" -s -S -L -f $(MISTRAL_MODEL_URL) -z $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME) -o $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME).tmp && mv -f $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME) 2>/dev/null || rm -f $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME).tmp $(RELATIVE_MODELS_PATH)/$(MISTRAL_MODEL_NAME)
+############################ END DOWNLOAD MAKE TARGETS ############################
 
-.PHONY: install
-install:
-	pip install -r tests/requirements.txt
 
 .PHONY: run
 run:
 	cd ../../models && \
-	podman run -it -d -p $(PORT):$(PORT) -v ./$(SELECTED_MODEL_NAME):$(MODELS_PATH)/model.gguf:$(BIND_MOUNT_OPTIONS) -e MODEL_PATH=$(MODELS_PATH)/model.gguf -e HOST=0.0.0.0 -e PORT=$(PORT) --net=host $(IMAGE)
-
-.PHONY: test
-test:
-	curl -H "Cache-Control: no-cache" -s -S -L -f $(SELECTED_MODEL_URL) -z ./model.gguf -o ./model.gguf.tmp && mv -f ./model.gguf.tmp ./model.gguf 2>/dev/null || rm -f ./model.gguf.tmp ./model.gguf
-	pytest --log-cli-level NOTSET
+	podman run --rm -d -it -p $(PORT):$(PORT) -e PORT=$(PORT) -v ./$(MODEL_NAME):$(MODELS_PATH)/$(MODEL_NAME):$(BIND_MOUNT_OPTIONS) -e MODEL_PATH=$(MODELS_PATH)/$(MODEL_NAME) -e HOST=0.0.0.0 --network=host $(IMAGE)
diff --git a/model_servers/llamacpp_python/README.md b/model_servers/llamacpp_python/README.md
@@ -59,6 +59,7 @@ podman pull quay.io/ai-lab/llamacpp-python-vulkan
 ```
 
 
+
 ## Download Model(s)
 
 There are many models to choose from these days, most of which can be found on [huggingface.co](https://huggingface.co). In order to use a model with the llamacpp_python model server, it must be in GGUF format. You can either download pre-converted GGUF models directly or convert them yourself with the [model converter utility](../../convert_models/) available in this repo.
@@ -69,20 +70,15 @@ Download URL: [https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/res
 
 Place all models in the [models](../../models/) directory.
 
-You can use this snippet below to download models. 
+You can use this snippet below to download models.:
 
 ```bash
 cd ../../models
-curl -sLO <Download URL> 
+make -f Makefile download-model-mistral
 cd model_servers/llamacpp_python
 ```
 
-or:
-
-```bash
-make -f Makefile download-model-mistral
-make -f Makefile download-model-llama
-```
+However we have also added targets [in this directory's Makefile](./Makefile) which will trigger model downloads: `make -f Makefile trigger-download-model-mistral`
 
 ## Deploy Model Service
 

diff --git a/model_servers/llamacpp_python/tests/conftest.py b/model_servers/llamacpp_python/tests/conftest.py
@@ -1,17 +1,31 @@
 import pytest_container
 import os
+import logging
+
+REGISTRY=os.environ['REGISTRY']
+IMAGE_NAME=os.environ['IMAGE_NAME']
+MODEL_NAME=os.environ['MODEL_NAME']
+
+logging.info("""
+Starting pytest with the following ENV vars:
+    REGISTRY: {REGISTRY}
+    IMAGE_NAME: {IMAGE_NAME}
+    MODEL_NAME: {MODEL_NAME}
+For:
+    model_server: whispercpp
+""".format(REGISTRY=REGISTRY, IMAGE_NAME=IMAGE_NAME, MODEL_NAME=MODEL_NAME))
 
 MS = pytest_container.Container(
-        url=f"containers-storage:{os.environ['REGISTRY']}/containers/{os.environ['IMAGE_NAME']}",
+        url=f"containers-storage:{REGISTRY}/{IMAGE_NAME}",
         volume_mounts=[
             pytest_container.container.BindMount(
-                container_path="/locallm/models/model.gguf",
-                host_path=f"./model.gguf",
+                container_path="/locallm/models/{MODEL_NAME}".format(MODEL_NAME=MODEL_NAME),
+                host_path=f"./{MODEL_NAME}",
                 flags=["ro"]
             )
         ],
         extra_environment_variables={
-            "MODEL_PATH": "/locallm/models/model.gguf",
+            "MODEL_PATH": "/locallm/models/{MODEL_NAME}".format(MODEL_NAME=MODEL_NAME),
             "HOST": "0.0.0.0",
             "PORT": "8001"
         },

diff --git a/model_servers/llamacpp_python/tests/test_alive.py b/model_servers/llamacpp_python/tests/test_alive.py
@@ -4,7 +4,6 @@
 
 CONTAINER_IMAGES = [MS]
 
-
 def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData):
     assert auto_container.connection.file("/etc/os-release").exists
 

diff --git a/model_servers/whispercpp/Makefile b/model_servers/whispercpp/Makefile
@@ -1,57 +1,41 @@
-PORT := 8001
 APP := whispercpp
-IMAGE := quay.io/ai-lab/model_servers/$(APP):latest
-CUDA_IMAGE := quay.io/ai-lab/model_servers/$(APP)_cuda:latest
-VULKAN_IMAGE :=quay.io/ai-lab/model_servers/$(APP)_vulkan:latest
-
-# ----- MODEL OPTIONS -----
-
-WHISPER_SMALL_MODEL_NAME := ggml-small.bin
-WHISPER_SMALL_MODEL_URL := https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin
+MODELS_PATH := /app/models
 
-WHISPER_BASE_MODEL_NAME := ggml-base.en.bin
-WHISPER_BASE_MODEL_URL := https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
+REGISTRY ?= quay.io
+PORT ?= 8001
 
-SELECTED_MODEL_NAME := $(or $(SELECTED_MODEL),$(WHISPER_SMALL_MODEL_NAME))
-SELECTED_MODEL_URL := $(or $(SELECTED_MODEL_LINK),$(WHISPER_SMALL_MODEL_URL))
+IMAGE_NAME ?= $(BASE_IMAGE_NAME)/$(APP):latest
+IMAGE ?= $(REGISTRY)/$(IMAGE_NAME)
 
-# --- END MODEL OPTIONS ---
+MODEL_NAME ?= ggml-small.bin
 
-MODELS_PATH := /app/models
+include ../common/Makefile.common
 
-BIND_MOUNT_OPTIONS :=  ro
-OS := $(shell uname -s)
-ifeq ($(OS),Linux)
-    BIND_MOUNT_OPTIONS := Z,ro
-endif
+# CUDA_IMAGE_NAME := $(REGISTRY)/$(BASE_IMAGE_NAME)/$(APP)_cuda:latest
+# VULKAN_IMAGE := $(REGISTRY)/$(BASE_IMAGE_NAME)/$(APP)_vulkan:latest
 
 .PHONY: all
-all: build whisper-small run
+all: build trigger-download-model-whisper-small run
 
-.PHONY: build
-build:
-	podman build -t $(IMAGE) . -f Containerfile
+########################## TRIGGER DOWNLOAD MAKE TARGETS ##########################
+### NOTE: you should not add these trigger make targets for new models.
+# Adding new models should implemented in the [models directory](../../models).
+# These are just provided here for bootstrapping and testing different model types.
 
-.PHONY: whisper-small
-whisper-small: 
+.PHONY: trigger-download-model-whisper-small # small .bin model type testing
+trigger-download-model-whisper-small:
 	cd ../../models && \
-	curl -s -S -L -f $(WHISPER_SMALL_MODEL_URL) -z $(WHISPER_SMALL_MODEL_NAME) -o $(WHISPER_SMALL_MODEL_NAME).tmp && mv -f $(WHISPER_SMALL_MODEL_NAME).tmp $(WHISPER_SMALL_MODEL_NAME) 2>/dev/null || rm -f $(WHISPER_SMALL_MODEL_NAME).tmp $(WHISPER_SMALL_MODEL_NAME)
-
-.PHONY: install
-install:
-	pip install -r tests/requirements.txt
+	make -f Makefile download-model-whisper-small
 
-.PHONY: download-model-whisper-base
-download-model-whisper-base: 
+.PHONY: trigger-download-model-whisper-base # default model
+trigger-download-model-whisper-base:
 	cd ../../models && \
-	curl -s -S -L -f $(WHISPER_BASE_MODEL_URL) -z $(WHISPER_BASE_MODEL_NAME) -o $(WHISPER_BASE_MODEL_NAME).tmp && mv -f $(WHISPER_BASE_MODEL_NAME).tmp $(WHISPER_BASE_MODEL_NAME) 2>/dev/null || rm -f $(WHISPER_BASE_MODEL_NAME).tmp $(WHISPER_BASE_MODEL_NAME)
+	make -f Makefile download-model-whisper-base
+
+############################ END DOWNLOAD MAKE TARGETS ############################
 
 .PHONY: run
 run:
 	cd ../../models && \
-	podman run -d --rm -it -p $(PORT):$(PORT) -v ./$(SELECTED_MODEL_NAME):$(MODELS_PATH)/$(SELECTED_MODEL_NAME):$(BIND_MOUNT_OPTIONS) -e HOST=0.0.0.0 -e MODEL_PATH=$(MODELS_PATH)/$(SELECTED_MODEL_NAME) -e PORT=$(PORT) $(IMAGE)
+	podman run -d -it -p $(PORT):$(PORT) -v ./$(MODEL_NAME):$(MODELS_PATH)/$(MODEL_NAME):$(BIND_MOUNT_OPTIONS) -e HOST=0.0.0.0 -e MODEL_PATH=$(MODELS_PATH)/$(MODEL_NAME) -e PORT=$(PORT) --network=host $(IMAGE)
 
-.PHONY: test
-test:
-	curl -H "Cache-Control: no-cache" -s -S -L -f $(SELECTED_MODEL_URL) -z ./model.gguf -o ./model.gguf.tmp && mv -f ./model.gguf.tmp ./model.gguf 2>/dev/null || rm -f ./model.gguf.tmp ./model.gguf
-	pytest --log-cli-level NOTSET
diff --git a/model_servers/whispercpp/tests/conftest.py b/model_servers/whispercpp/tests/conftest.py
@@ -1,17 +1,32 @@
 import pytest_container
 import os
+import logging
+import sys
+
+REGISTRY=os.environ['REGISTRY']
+IMAGE_NAME=os.environ['IMAGE_NAME']
+MODEL_NAME=os.environ['MODEL_NAME']
+
+logging.info("""
+Starting pytest with the following ENV vars:
+    REGISTRY: {REGISTRY}
+    IMAGE_NAME: {IMAGE_NAME}
+    MODEL_NAME: {MODEL_NAME}
+For:
+    model_server: whispercpp
+""".format(REGISTRY=REGISTRY, IMAGE_NAME=IMAGE_NAME, MODEL_NAME=MODEL_NAME))
 
 MS = pytest_container.Container(
-        url=f"containers-storage:{os.environ['REGISTRY']}/containers/{os.environ['IMAGE_NAME']}",
+        url=f"containers-storage:{REGISTRY}/{IMAGE_NAME}",
         volume_mounts=[
             pytest_container.container.BindMount(
-                container_path="/locallm/models/model.gguf",
-                host_path=f"./model.gguf",
+                container_path="/app/models/{MODEL_NAME}".format(MODEL_NAME=MODEL_NAME),
+                host_path=f"./{MODEL_NAME}",
                 flags=["ro"]
             )
         ],
         extra_environment_variables={
-            "MODEL_PATH": "/locallm/models/model.gguf",
+            "MODEL_PATH": f"/app/models/{MODEL_NAME}".format(MODEL_NAME=MODEL_NAME),
             "HOST": "0.0.0.0",
             "PORT": "8001"
         },

diff --git a/model_servers/whispercpp/tests/test_alive.py b/model_servers/whispercpp/tests/test_alive.py
@@ -4,7 +4,6 @@
 
 CONTAINER_IMAGES = [MS]
 
-
 def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData):
     assert auto_container.connection.file("/etc/os-release").exists
 

diff --git a/models/Containerfile b/models/Containerfile
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,7 +4,6 @@

		CONTAINER_IMAGES = [MS]


		def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData):
		assert auto_container.connection.file("/etc/os-release").exists

Expand Down