From 5d5d6be5eeef6c524f5c8e2880b48bec86262cff Mon Sep 17 00:00:00 2001 From: greg pereira Date: Sat, 13 Apr 2024 07:08:21 -0700 Subject: [PATCH] include common directory in tests on path changes Signed-off-by: greg pereira --- .github/workflows/chatbot.yaml | 2 ++ .github/workflows/codegen.yaml | 6 +++-- .github/workflows/model_servers.yaml | 2 +- .github/workflows/rag.yaml | 2 ++ .gitignore | 4 +-- recipes/common/Makefile.common | 27 +++++++++---------- recipes/common/bin/.gitkeep | 0 .../chatbot/Makefile | 7 ++++- 8 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 recipes/common/bin/.gitkeep diff --git a/.github/workflows/chatbot.yaml b/.github/workflows/chatbot.yaml index 14b726ded..f04be8350 100644 --- a/.github/workflows/chatbot.yaml +++ b/.github/workflows/chatbot.yaml @@ -5,12 +5,14 @@ on: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/chatbot/** - .github/workflows/chatbot.yaml push: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/chatbot/** - .github/workflows/chatbot.yaml diff --git a/.github/workflows/codegen.yaml b/.github/workflows/codegen.yaml index b3ec2c5ce..869ca70fb 100644 --- a/.github/workflows/codegen.yaml +++ b/.github/workflows/codegen.yaml @@ -5,12 +5,14 @@ on: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/codegen/** - .github/workflows/codegen.yaml push: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/codegen/** - .github/workflows/codegen.yaml @@ -59,8 +61,8 @@ jobs: run: make install - name: Download model - working-directory: ./model_servers/llamacpp_python - run: make mistral + working-directory: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }} + run: make download-model-mistral - name: Run Functional Tests shell: bash diff --git a/.github/workflows/model_servers.yaml b/.github/workflows/model_servers.yaml index 4bbc893ce..16f75b0f6 100644 --- a/.github/workflows/model_servers.yaml +++ b/.github/workflows/model_servers.yaml @@ -5,7 +5,7 @@ on: branches: - main paths: - - ./model_servers/ + - ./model_servers/** - .github/workflows/model_servers.yaml push: branches: diff --git a/.github/workflows/rag.yaml b/.github/workflows/rag.yaml index 3fbe96ef2..b8a43e8be 100644 --- a/.github/workflows/rag.yaml +++ b/.github/workflows/rag.yaml @@ -5,12 +5,14 @@ on: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/rag/** - .github/workflows/rag.yaml push: branches: - main paths: + - ./recipes/common/Makefile.common - ./recipes/natural_language_processing/rag/** - .github/workflows/rag.yaml diff --git a/.gitignore b/.gitignore index 0d0ab5b70..5e97f2843 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,5 @@ models/* !models/Makefile !models/README.md convert_models/converted_models -recipes/chromedriver -recipes/Google\ Chrome.app +recipes/common/bin/* +*/.venv/ diff --git a/recipes/common/Makefile.common b/recipes/common/Makefile.common index 712dde49c..d979d6ac6 100644 --- a/recipes/common/Makefile.common +++ b/recipes/common/Makefile.common @@ -31,7 +31,7 @@ endif CHROMEDRIVER_VERSION := 103.0.5060.53 CHROMEDRIVER_MIRROR := https://chromedriver.storage.googleapis.com CHROMEDRIVER_DOWNLOAD_PATH := -CHROMEDRIVER_INSTALLATION_PATH ?= +RECIPE_BINARIES_PATH ?= CHROME_DOWNLOAD_PATH ?= OS := $(shell uname -s) @@ -51,7 +51,7 @@ endif CHROME_MIRROR := https://www.slimjet.com/chrome/files/$(CHROMEDRIVER_VERSION)/$(CHROME_DOWNLOAD_PATH) -LOCAL_CHROMEDRIVER_EXISTS ?= $(shell command -v $(CHROMEDRIVER_INSTALLATION_PATH)/chromedriver) +LOCAL_CHROMEDRIVER_EXISTS ?= $(shell command -v $(RECIPE_BINARIES_PATH)/chromedriver) UNZIP_EXISTS ?= $(shell command -v unzip) RELATIVE_MODELS_PATH := ?= @@ -68,9 +68,7 @@ download-model-mistral: make MODEL_NAME=mistral-7b-instruct-v0.1.Q4_K_M.gguf MODEL_URL=https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q4_K_M.gguf -f Makefile download-model .PHONY: install -install: - $(MAKE) install-chromedriver CHROMEDRIVER_INSTALLATION_PATH=${CHROMEDRIVER_INSTALLATION_PATH} - $(MAKE) install-chrome CHROMEDRIVER_INSTALLATION_PATH=${CHROMEDRIVER_INSTALLATION_PATH} +install:: pip install -q -r ${RELATIVE_TESTS_PATH}/requirements.txt .PHONY: build @@ -129,9 +127,9 @@ bootc-image-builder: bootc install-chromedriver: @if [[ -z "$(LOCAL_CHROMEDRIVER_EXISTS)" ]]; then \ if [[ -n "$(UNZIP_EXISTS)" ]]; then \ - curl -sLO $(CHROMEDRIVER_MIRROR)/$(CHROMEDRIVER_VERSION)/$(CHROMEDRIVER_DOWNLOAD_PATH); \ + curl --max-time 300 --connect-timeout 60 -sLO $(CHROMEDRIVER_MIRROR)/$(CHROMEDRIVER_VERSION)/$(CHROMEDRIVER_DOWNLOAD_PATH); \ unzip $(CHROMEDRIVER_DOWNLOAD_PATH); \ - mv chromedriver $(CHROMEDRIVER_INSTALLATION_PATH); \ + mv chromedriver $(RECIPE_BINARIES_PATH/); \ rm ./$(CHROMEDRIVER_DOWNLOAD_PATH); \ elif [[ -z "$(UNZIP_EXISTS)" ]]; then \ echo "Install make target requires unizp binary."; \ @@ -141,14 +139,15 @@ install-chromedriver: .PHONY: install-chrome install-chrome: - curl -sLO $(CHROME_MIRROR) + curl --max-time 300 --connect-timeout 60 -sLO $(CHROME_MIRROR) @if [[ "$(OS)" == "Linux" ]]; then \ sudo dpkg -i $(CHROME_DOWNLOAD_PATH); \ elif [[ "$(OS)" == "Darwin" ]]; then \ open $(CHROME_DOWNLOAD_PATH); \ + ls "/Volumes/Google Chrome/Google Chrome.app/Contents/MacOS/Google Chrome"; \ + cp -r "/Volumes/Google Chrome/Google Chrome.app" "$(RECIPE_BINARIES_PATH)/"; \ + diskutil unmount "/Volumes/Google Chrome" || true; \ rm $(CHROME_DOWNLOAD_PATH); \ - cp -r /Volumes/Google\ Chrome/Google\ Chrome.app $(CHROMEDRIVER_INSTALLATION_PATH); \ - diskutil unmount /Volumes/Google\ Chrome; \ fi; .PHONY: quadlet @@ -193,20 +192,20 @@ check-model-in-path: functional-tests: $(MAKE) MODEL_NAME=$(MODEL_NAME) check-model-in-path @if [[ -n "$(LOCAL_CHROMEDRIVER_EXISTS)" ]]; then \ - IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} MODEL_NAME=${MODEL_NAME} pytest -vvv --driver=Chrome --driver-path=$(CHROMEDRIVER_INSTALLATION_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/functional; \ + IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} MODEL_NAME=${MODEL_NAME} pytest -vvv --driver=Chrome --driver-path=$(RECIPE_BINARIES_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/functional; \ else \ echo "fetching chromedriver"; \ make install; \ - IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} MODEL_NAME=${MODEL_NAME} pytest -vvv --driver=Chrome --driver-path=$(CHROMEDRIVER_INSTALLATION_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/functional; \ + IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} MODEL_NAME=${MODEL_NAME} pytest -vvv --driver=Chrome --driver-path=$(RECIPE_BINARIES_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/functional; \ fi; .PHONY: integration-tests integration-tests: @if [[ -n "$(LOCAL_CHROMEDRIVER_EXISTS)" ]]; then \ - URL=${URL} IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} pytest -vvv --driver=Chrome --driver-path=$(CHROMEDRIVER_INSTALLATION_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/integration; \ + URL=${URL} IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} pytest -vvv --driver=Chrome --driver-path=$(RECIPE_BINARIES_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/integration; \ else \ echo "fetching chromedriver"; \ make install; \ - URL=${URL} IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} pytest -vvv --driver=Chrome --driver-path=$(CHROMEDRIVER_INSTALLATION_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/integration; \ + URL=${URL} IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} pytest -vvv --driver=Chrome --driver-path=$(RECIPE_BINARIES_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/integration; \ fi; diff --git a/recipes/common/bin/.gitkeep b/recipes/common/bin/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/recipes/natural_language_processing/chatbot/Makefile b/recipes/natural_language_processing/chatbot/Makefile index 9cd8adf09..0fb079042 100644 --- a/recipes/natural_language_processing/chatbot/Makefile +++ b/recipes/natural_language_processing/chatbot/Makefile @@ -4,6 +4,11 @@ PORT ?= 8501 include ../../common/Makefile.common -CHROMEDRIVER_INSTALLATION_PATH := $(shell realpath ../..) +RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) RELATIVE_MODELS_PATH := ../../../models RELATIVE_TESTS_PATH := ../tests + +.PHONY: install +install:: + # $(MAKE) install-chromedriver RECIPE_BINARIES_PATH=${RECIPE_BINARIES_PATH} + $(MAKE) install-chrome RECIPE_BINARIES_PATH=${RECIPE_BINARIES_PATH} \ No newline at end of file