fix: windows build with -DBUILD_SHARED_LIBS=OFF (#399)

janhq · Feb 1, 2024 · a2ce055 · a2ce055
1 parent 937ec23
commit a2ce055
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 53 deletions.
diff --git a/.github/scripts/e2e-test-llama-windows.bat b/.github/scripts/e2e-test-llama-windows.bat
@@ -25,10 +25,6 @@ set /a max=11000
 set /a range=max-min+1
 set /a PORT=%min% + %RANDOM% %% %range%
 
-rem Kill any existing Nitro processes
-echo Killing any existing Nitro processes...
-taskkill /f /im nitro.exe 2>nul
-
 rem Start the binary file
 start /B "" "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > %TEMP%\nitro.log 2>&1
 
@@ -64,9 +60,9 @@ echo curl_data1=%curl_data1%
 echo curl_data2=%curl_data2%
 
 rem Run the curl commands and capture the status code
-curl.exe -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/loadModel" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1
+curl.exe --connect-timeout 60 -o "%TEMP%\response1.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/loadModel" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1.log 2>&1
 
-curl.exe -o "%TEMP%\response2.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/chat_completion" ^
+curl.exe --connect-timeout 60 -o "%TEMP%\response2.log" -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/inferences/llamacpp/chat_completion" ^
 --header "Content-Type: application/json" ^
 --header "Accept: text/event-stream" ^
 --header "Access-Control-Allow-Origin: *" ^
@@ -78,13 +74,13 @@ rem Read the status codes from the log files
 for /f %%a in (%TEMP%\response1.log) do set "response1=%%a"
 for /f %%a in (%TEMP%\response2.log) do set "response2=%%a"
 
-if "%response1%" neq "000" (
+if "%response1%" neq "200" (
     echo The first curl command failed with status code: %response1%
     type %TEMP%\response1.log
     set "error_occurred=1"
 )
 
-if "%response2%" neq "000" (
+if "%response2%" neq "200" (
     echo The second curl command failed with status code: %response2%
     type %TEMP%\response2.log
     set "error_occurred=1"
@@ -111,4 +107,4 @@ echo Nitro test run successfully!
 
 rem Kill the server process
 @REM taskkill /f /pid %pid%
-taskkill /f /im nitro.exe 2>nul || exit /B 0
+taskkill /f /im nitro.exe 2>nul || exit /B 0
diff --git a/.github/scripts/e2e-test-whisper-windows.bat b/.github/scripts/e2e-test-whisper-windows.bat
@@ -25,10 +25,6 @@ set /a max=11000
 set /a range=max-min+1
 set /a PORT=%min% + %RANDOM% %% %range%
 
-rem Kill any existing Nitro processes
-echo Killing any existing Nitro processes...
-taskkill /f /im nitro.exe 2>nul
-
 rem Start the binary file
 start /B "" "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > %TEMP%\nitro.log 2>&1
 
@@ -49,28 +45,21 @@ if not defined pid (
 
 rem Wait for a few seconds to let the server start
 
-rem Check if %TEMP%\testmodel exists, if not, download it
+rem Check if %TEMP%\testwhisper exists, if not, download it
 if not exist "%MODEL_PATH%" (
     bitsadmin.exe /transfer "DownloadTestModel" %DOWNLOAD_URL% "%MODEL_PATH%"
 )
 
 rem Define JSON strings for curl data
 call set "MODEL_PATH_STRING=%%MODEL_PATH:\=\\%%"
-set "curl_data1={\"model_path\":\"%MODEL_PATH_STRING%\",\"model_id\":\"whisper.cpp\"}"
-
-rem Print the values of curl_data1 for debugging
-echo curl_data1=%curl_data1%
+set "curl_data1={\"model_path\":\"%MODEL_PATH_STRING%\",\"model_id\":\"whisper\"}"
 
 rem Run the curl commands and capture the status code
-curl.exe -o %TEMP%\response1_code.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/load_model" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1_code.log 2>&1
+curl.exe -o %TEMP%\response1.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/load_model" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1_code.log 2>&1
 
-curl.exe -o %TEMP%\response2_code.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/transcriptions" ^
---header "Access-Control-Allow-Origin: *" ^
---form 'model_id="whisper.cpp"' ^
---form 'file=@"..\whisper.cpp\samples\jfk.wav"' ^
---form 'temperature="0.0"' ^
---form 'prompt="The transcript is about OpenAI which makes technology like DALL·E, GPT-3, and ChatGPT with the hope of one day building an AGI system that benefits all of humanity. The president is trying to raly people to support the cause."' ^
-> %TEMP%\response2_code.log 2>&1
+curl -o %TEMP%\response2.log -s -w "%%{http_code}" --location "http://localhost:%PORT%/v1/audio/transcriptions" ^
+--form "file=@../..//whisper.cpp/samples/jfk.wav" ^
+--form "model_id=whisper" > %TEMP%\response2_code.log 2>&1
 
 set "error_occurred=0"
 
@@ -80,13 +69,13 @@ for /f %%a in (%TEMP%\response2_code.log) do set "response2=%%a"
 
 if "%response1%" neq "200" (
     echo The first curl command failed with status code: %response1%
-    type %TEMP%\response1_code.log
+    type %TEMP%\response1.log
     set "error_occurred=1"
 )
 
-if "%response2%" neq "000" (
+if "%response2%" neq "200" (
     echo The second curl command failed with status code: %response2%
-    type %TEMP%\response2_code.log
+    type %TEMP%\response2.log
     set "error_occurred=1"
 )
 
@@ -101,14 +90,13 @@ if "%error_occurred%"=="1" (
 
 echo ----------------------
 echo Log load model:
-type %TEMP%\response1_code.log
+type %TEMP%\response1.log
 
 echo ----------------------
 echo "Log run test:"
-type %TEMP%\response2_code.log
+type %TEMP%\response2.log
 
 echo Nitro test run successfully!
 
 rem Kill the server process
-@REM taskkill /f /pid %pid%
 taskkill /f /im nitro.exe 2>nul || exit /B 0
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -50,7 +50,7 @@ on:
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   LLM_MODEL_URL: https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/resolve/main/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
-  WHISPER_MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin
+  WHISPER_MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin
 
 jobs:
   create-draft-release:
@@ -377,11 +377,11 @@ jobs:
         run: |
           # To test with CoreML
           if [[ ! -f "/tmp/testwhisper-encoder.mlmodelc" ]]; then
-            wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-encoder.mlmodelc.zip
-            unzip ggml-tiny.en-encoder.mlmodelc.zip
-            rm ggml-tiny.en-encoder.mlmodelc.zip
+            wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-encoder.mlmodelc.zip
+            unzip ggml-tiny-encoder.mlmodelc.zip
+            rm ggml-tiny-encoder.mlmodelc.zip
             rm -rf /tmp/testwhisper-encoder.mlmodelc
-            mv ggml-tiny.en-encoder.mlmodelc /tmp/testwhisper-encoder.mlmodelc
+            mv ggml-tiny-encoder.mlmodelc /tmp/testwhisper-encoder.mlmodelc
           fi
           # run e2e testing
           cd nitro
@@ -586,7 +586,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts
@@ -602,15 +602,19 @@ jobs:
           7z a -ttar temp.tar .\build\Release\*
           7z a -tgzip nitro.tar.gz temp.tar
 
-      # - name: Run e2e testing - Llama.cpp
-      #   shell: cmd
-      #   run: |
-      #     .\.github\scripts\e2e-test-llama-windows.bat .\build\Release\nitro.exe ${{ env.LLM_MODEL_URL }}
+      - name: Run e2e testing - Llama.cpp
+        shell: cmd
+        run: |
+          cd build\Release
+          ..\..\.github\scripts\e2e-test-llama-windows.bat nitro.exe ${{ env.LLM_MODEL_URL }}
+          rmdir /S /Q .\build\Release\uploads
 
-      # - name: Run e2e testing - Whisper.cpp
-      #   shell: cmd
-      #   run: |
-      #     .\.github\scripts\e2e-test-whisper-windows.bat .\build\Release\nitro.exe ${{ env.WHISPER_MODEL_URL }}
+      - name: Run e2e testing - Whisper.cpp
+        shell: cmd
+        run: |
+          cd build\Release
+          ..\..\.github\scripts\e2e-test-whisper-windows.bat nitro.exe ${{ env.WHISPER_MODEL_URL }}
+          rmdir /S /Q .\build\Release\uploads
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v2
@@ -679,7 +683,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts
@@ -770,7 +774,7 @@ jobs:
           cmake --build ./build_deps/nitro_deps --config Release
           mkdir -p build
           cd build
-          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DWHISPER_CUBLAS=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
+          cmake .. -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DWHISPER_SDL2=ON -DWHISPER_CUBLAS=ON -DNITRO_VERSION=${{ needs.set-nitro-version.outputs.version }}
           cmake --build . --config Release -j "%NUMBER_OF_PROCESSORS%"
 
       - name: Pack artifacts

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -59,7 +59,7 @@ add_executable(${PROJECT_NAME} main.cc)
 #
 # and comment out the following lines
 find_package(Drogon CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama llava whisper
+target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama whisper llava
                                               ${CMAKE_THREAD_LIBS_INIT})
 
 # ##############################################################################

diff --git a/controllers/whisperCPP.cc b/controllers/whisperCPP.cc
@@ -872,7 +872,9 @@ void whisperCPP::load_model(
         (*jsonBody)["warm_up_audio_path"].asString();
     // Return 400 error if warm up audio path is not found
     if (!is_file_exist(warm_up_audio_path.c_str())) {
-      std::string error_msg = "Warm up audio " + warm_up_audio_path + " not found, please provide a valid path or don't specify it at all";
+      std::string error_msg =
+          "Warm up audio " + warm_up_audio_path +
+          " not found, please provide a valid path or don't specify it at all";
       LOG_INFO << error_msg;
       Json::Value jsonResp;
       jsonResp["message"] = error_msg;
@@ -881,9 +883,10 @@ void whisperCPP::load_model(
       callback(resp);
       return;
     } else {
-      LOG_INFO << "Warming up model " << model_id << " with audio " << warm_up_audio_path << " ...";
-      std::string warm_up_result = whisper.inference(
-          warm_up_audio_path, "en", "", text_format, 0, false);
+      LOG_INFO << "Warming up model " << model_id << " with audio "
+               << warm_up_audio_path << " ...";
+      std::string warm_up_result = whisper.inference(warm_up_audio_path, "en",
+                                                     "", text_format, 0, false);
       LOG_INFO << "Warm up model " << model_id << " completed";
     }
   } else {