From 52b692d2b2b70bae325a9e810d3092e40bc12483 Mon Sep 17 00:00:00 2001 From: Hien To Date: Wed, 15 May 2024 16:45:39 +0700 Subject: [PATCH 1/2] Add codesign for cortex cpp --- .../{build.yml => cortex-cpp-build.yml} | 31 ++++++++++-- ...y-gate.yml => cortex-cpp-quality-gate.yml} | 7 ++- cortex-cpp/Makefile | 49 ++++++++++++++++--- 3 files changed, 77 insertions(+), 10 deletions(-) rename .github/workflows/{build.yml => cortex-cpp-build.yml} (85%) rename .github/workflows/{quality-gate.yml => cortex-cpp-quality-gate.yml} (97%) diff --git a/.github/workflows/build.yml b/.github/workflows/cortex-cpp-build.yml similarity index 85% rename from .github/workflows/build.yml rename to .github/workflows/cortex-cpp-build.yml index 716dfd679..e75be1c33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/cortex-cpp-build.yml @@ -1,4 +1,4 @@ -name: CI +name: CI Cortex CPP on: push: @@ -25,7 +25,8 @@ jobs: steps: - name: Extract tag name without v prefix id: get_version - run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}" + run: | + echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}" env: GITHUB_REF: ${{ github.ref }} - name: Create Draft Release @@ -91,7 +92,7 @@ jobs: - os: "mac" name: "arm64" - runs-on: "mac-silicon" + runs-on: "macos-latest" cmake-flags: "-DMAC_ARM64=ON" run-e2e: true @@ -166,11 +167,35 @@ jobs: run: | choco install make -y + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 + shell: bash + env: + CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + + - uses: apple-actions/import-codesign-certs@v2 + if: runner.os == 'macOS' + with: + p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} + - name: Build run: | cd cortex-cpp make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" + - name: Pre-package + run: | + cd cortex-cpp + make pre-package + + - name: Code Signing + run: | + cd cortex-cpp + make codesign CODE_SIGN=true AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}" DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}" + + - name: Package run: | cd cortex-cpp diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml similarity index 97% rename from .github/workflows/quality-gate.yml rename to .github/workflows/cortex-cpp-quality-gate.yml index 909ab7e77..33c8a4533 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -1,4 +1,4 @@ -name: CI Quality Gate +name: CI Quality Gate Cortex CPP on: pull_request: @@ -145,6 +145,11 @@ jobs: cd cortex-cpp make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" + - name: Pre-package + run: | + cd cortex-cpp + make pre-package + - name: Package run: | cd cortex-cpp diff --git a/cortex-cpp/Makefile b/cortex-cpp/Makefile index e9f1d85b0..80614f14d 100644 --- a/cortex-cpp/Makefile +++ b/cortex-cpp/Makefile @@ -6,6 +6,13 @@ CMAKE_EXTRA_FLAGS ?= "" RUN_TESTS ?= false LLM_MODEL_URL ?= "https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" EMBEDDING_MODEL_URL ?= "https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf" +CODE_SIGN ?= false +AZURE_KEY_VAULT_URI ?= xxxx +AZURE_CLIENT_ID ?= xxxx +AZURE_TENANT_ID ?= xxxx +AZURE_CLIENT_SECRET ?= xxxx +AZURE_CERT_NAME ?= xxxx +DEVELOPER_ID ?= xxxx # Default target, does nothing all: @@ -29,7 +36,7 @@ else make -j4; endif -package: +pre-package: ifeq ($(OS),Windows_NT) @powershell -Command "mkdir -p cortex-cpp\engines\cortex.llamacpp\; cp -r build\engines\cortex.llamacpp\engine.dll cortex-cpp\engines\cortex.llamacpp\;" @powershell -Command "cp -r build\Release\cortex-cpp.exe .\cortex-cpp\;" @@ -37,16 +44,38 @@ ifeq ($(OS),Windows_NT) @powershell -Command "cp -r ..\.github\patches\windows\msvcp140.dll .\cortex-cpp\;" @powershell -Command "cp -r ..\.github\patches\windows\vcruntime140_1.dll .\cortex-cpp\;" @powershell -Command "cp -r ..\.github\patches\windows\vcruntime140.dll .\cortex-cpp\;" - @powershell -Command "7z a -ttar temp.tar cortex-cpp\*; 7z a -tgzip cortex-cpp.tar.gz temp.tar;" else ifeq ($(shell uname -s),Linux) @mkdir -p cortex-cpp/engines/cortex.llamacpp; \ cp build/engines/cortex.llamacpp/libengine.so cortex-cpp/engines/cortex.llamacpp/; \ - cp build/cortex-cpp cortex-cpp/; \ - tar -czvf cortex-cpp.tar.gz cortex-cpp; + cp build/cortex-cpp cortex-cpp/; else @mkdir -p cortex-cpp/engines/cortex.llamacpp; \ cp build/engines/cortex.llamacpp/libengine.dylib cortex-cpp/engines/cortex.llamacpp/; \ - cp build/cortex-cpp cortex-cpp/; \ + cp build/cortex-cpp cortex-cpp/; +endif + +codesign: +ifeq ($(CODE_SIGN),false) + @echo "Skipping Code Sign" + @exit 0 +endif + +ifeq ($(OS),Windows_NT) + @powershell -Command "dotnet tool install --global AzureSignTool;" + @powershell -Command "Get-ChildItem -Path .\cortex-cpp -Recurse | ForEach-Object { & 'azuresigntool.exe' sign -kvu '$(AZURE_KEY_VAULT_URI)' -kvi '$(AZURE_CLIENT_ID)' -kvt '$(AZURE_TENANT_ID)' -kvs '$(AZURE_CLIENT_SECRET)' -kvc '$(AZURE_CERT_NAME)' -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v '$_.FullName' };" +else ifeq ($(shell uname -s),Linux) + @echo "Skipping Code Sign for linux" + @exit 0 +else + find "cortex-cpp" -type f -exec codesign --force -s "$(DEVELOPER_ID)" --options=runtime {} \; +endif + +package: +ifeq ($(OS),Windows_NT) + @powershell -Command "7z a -ttar temp.tar cortex-cpp\*; 7z a -tgzip cortex-cpp.tar.gz temp.tar;" +else ifeq ($(shell uname -s),Linux) + tar -czvf cortex-cpp.tar.gz cortex-cpp; +else tar -czvf cortex-cpp.tar.gz cortex-cpp; endif @@ -65,4 +94,12 @@ else @cd cortex-cpp; \ chmod +x ../../.github/scripts/e2e-test-llama-linux-and-mac.sh && ../../.github/scripts/e2e-test-llama-linux-and-mac.sh ./cortex-cpp $(LLM_MODEL_URL) $(EMBEDDING_MODEL_URL); \ rm -rf uploads/; -endif \ No newline at end of file +endif + +clean: +ifeq ($(OS),Windows_NT) + @powershell -Command "rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz;" +else ifeq ($(shell uname -s),Linux) + @rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz; +else + @rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz; \ No newline at end of file From 33a0e96a0887cb3e35b553345a7e4b2d83e3f00c Mon Sep 17 00:00:00 2001 From: Hien To Date: Thu, 16 May 2024 16:55:14 +0700 Subject: [PATCH 2/2] Add CI for cortex js --- .github/workflows/cortex-js.yml | 44 +++++++++++++++++++++++++++++++++ cortex-js/package.json | 2 +- package.json | 21 ---------------- 3 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/cortex-js.yml delete mode 100644 package.json diff --git a/.github/workflows/cortex-js.yml b/.github/workflows/cortex-js.yml new file mode 100644 index 000000000..31175b1be --- /dev/null +++ b/.github/workflows/cortex-js.yml @@ -0,0 +1,44 @@ +name: Publish cortex js Package to npmjs +on: + push: + tags: ["v[0-9]+.[0-9]+.[0-9]+-cortex-js"] + paths: + [ + "cortex-js/**", + ] +jobs: + build-and-publish-plugins: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: "0" + + - name: Install jq + uses: dcarbone/install-jq-action@v2.0.1 + + - name: "Update version by tag" + run: | + cd cortex-js + # Remove the v prefix + tag_version=${GITHUB_REF#refs/tags/v} + # Remove the -cortex-js suffix + new_version=${tag_version%-cortex-js} + + # Replace the old version with the new version in package.json + jq --arg version "$new_version" '.version = $version' ./package.json > /tmp/package.json && mv /tmp/package.json ./package.json + + # Print the new version + echo "Updated package.json version to: $new_version" + + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v3 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + - run: yarn install && yarn build + working-directory: ./cortex-js + - run: npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + working-directory: ./cortex-js diff --git a/cortex-js/package.json b/cortex-js/package.json index 3b3989381..f5681e447 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -1,5 +1,5 @@ { - "name": "cortex-js", + "name": "@janhq/cortex", "version": "0.0.1", "description": "", "author": "", diff --git a/package.json b/package.json deleted file mode 100644 index a782f0925..000000000 --- a/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@janhq/cortex", - "version": "1.0.0", - "license": "AGPL-3.0", - "scripts": { - "preinstall": "npm pre-install script; platform specific (MacOS / Windows / Linux)", - "dev": "cd cortex-js && yarn start", - "build": "cd cortex-js && yarn build" - }, - "dependencies": { - "express": "^4.17.1", - "typeorm": "^0.2.37", - "pg": "^8.7.1", - "dotenv": "^10.0.0" - }, - "devDependencies": { - "@types/express": "^4.17.13", - "@types/node": "^14.14.33", - "typescript": "^4F3.5" - } -}