Initiate CICD nightly

janhq · Sep 9, 2024 · 25173dd · 25173dd
1 parent 722ec0c
commit 25173dd
Show file tree

Hide file tree

Showing 2 changed files with 200 additions and 0 deletions.
diff --git a/.github/workflows/cortex-nightly.yml b/.github/workflows/cortex-nightly.yml
@@ -0,0 +1,142 @@
+name: CI Cortex CPP Nightly Build
+
+on:
+  schedule:
+      - cron: '0 20 * * *' # At 8 PM UTC everyday
+  workflow_dispatch:
+    inputs:
+      public_provider:
+        type: choice
+        description: 'Public Provider'
+        options:
+          - none
+          - aws-s3
+        default: none
+
+env:
+  LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
+  EMBEDDING_MODEL_URL: https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf
+
+jobs:
+  set-public-provider:
+    runs-on: ubuntu-latest
+    outputs:
+      public_provider: ${{ steps.set-public-provider.outputs.public_provider }}
+      ref: ${{ steps.set-public-provider.outputs.ref }}
+    steps:
+      - name: Set public provider
+        id: set-public-provider
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "::set-output name=public_provider::${{ github.event.inputs.public_provider }}"
+            echo "::set-output name=ref::${{ github.ref }}"
+          else
+            if [ "${{ github.event_name }}" == "schedule" ]; then
+              echo "::set-output name=public_provider::aws-s3"
+              echo "::set-output name=ref::refs/heads/dev"
+            elif [ "${{ github.event_name }}" == "push" ]; then
+              echo "::set-output name=public_provider::aws-s3"
+              echo "::set-output name=ref::${{ github.ref }}"
+            else
+              echo "::set-output name=public_provider::none"
+              echo "::set-output name=ref::${{ github.ref }}"
+            fi
+          fi
+
+  # Job create Update app version based on latest release tag with build number and save to output
+  get-update-version:
+    uses: ./.github/workflows/template-get-update-version.yml
+
+  build-and-test:
+    runs-on: ${{ matrix.runs-on }}
+    needs: [get-update-version, set-public-provider]
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: "linux"
+            name: "amd64"
+            runs-on: "ubuntu-20-04-cuda-12-0"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "amd64"
+            runs-on: "macos-13"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "arm64"
+            runs-on: "macos-latest"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DMAC_ARM64=ON -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "windows"
+            name: "amd64"
+            runs-on: "windows-cuda-12-0"
+            cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DBUILD_SHARED_LIBS=OFF -DCMAKE_TOOLCHAIN_FILE=C:/w/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja"
+            build-deps-cmake-flags: "-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja"
+            ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'          
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install choco on Windows
+        if: runner.os == 'Windows'
+        run: |
+          choco install make pkgconfiglite ccache awscli 7zip ninja -y
+
+      - name: Download ccache from s3
+        if: runner.os == 'Windows'
+        continue-on-error: true
+        run: |
+          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+          refreshenv
+          aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-${{ matrix.os }}-${{ matrix.name }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
+          AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
+
+      - name: Configure vcpkg
+        run: |
+          cd engine
+          make configure-vcpkg
+      
+      - name: Build
+        run: |
+          cd engine
+          make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}"
+ 
+      - name: Pre-package
+        run: |
+          cd engine
+          make pre-package
+          
+      - name: Package
+        run: |
+          cd engine
+          make package
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: cortex-cpp-${{ matrix.os }}-${{ matrix.name }}
+          path: ./engine/cortex-cpp
+
+      - name: Upload ccache to s3
+        continue-on-error: true
+        if: always() && runner.os == 'Windows'
+        run: |
+          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+          refreshenv
+          aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-${{ matrix.os }}-${{ matrix.name }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }}
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
+          AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
diff --git a/.github/workflows/template-get-update-version.yml b/.github/workflows/template-get-update-version.yml
@@ -0,0 +1,58 @@
+name: get-update-version
+on:
+  workflow_call:
+    outputs:
+      new_version:
+        description: 'The new version of the app'
+        value: ${{ jobs.get-update-version.outputs.new_version }}
+
+jobs:
+  get-update-version:
+    runs-on: ubuntu-latest
+    environment: production
+    outputs:
+      new_version: ${{ steps.version_update.outputs.new_version }}
+    steps:
+    - name: Install jq
+      uses: dcarbone/[email protected]
+
+    - name: Get tag
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+      id: tag
+      uses: dawidd6/action-get-tag@v1
+
+    - name: Update app version based on latest release tag with build number
+      id: version_update
+      run: |
+        # Function to get the latest release tag
+        get_latest_tag() {
+          local retries=0
+          local max_retries=3
+          local tag
+          while [ $retries -lt $max_retries ]; do
+            tag=$(curl -s https://api.github.com/repos/janhq/cortex.cpp/releases/latest | jq -r .tag_name)
+            if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+              echo $tag
+              return
+            else
+              let retries++
+              echo "Retrying... ($retries/$max_retries)"
+              sleep 2
+            fi
+          done
+          echo "Failed to fetch latest tag after $max_retries attempts."
+          exit 1
+        }
+
+        if ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}; then
+          echo "Tag detected, set output follow tag"
+          echo "::set-output name=new_version::${{ steps.tag.outputs.tag }}"
+        else
+          # Get the latest release tag from GitHub API
+          LATEST_TAG=$(get_latest_tag)
+          
+          # Remove the 'v' and append the build number to the version
+          new_version="${LATEST_TAG#v}-${GITHUB_RUN_NUMBER}"
+          echo "New version: $new_version"
+          echo "::set-output name=new_version::$new_version"
+        fi