diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index cf55b6cc2..b1b3a770c 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -90,7 +90,7 @@ jobs: make package - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: cortex-${{ matrix.os }}-${{ matrix.name }} path: ./engine/cortex diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 2b45b8d62..eaba16cb0 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -13,10 +13,6 @@ on: - aws-s3 default: none -env: - LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf - EMBEDDING_MODEL_URL: https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf - jobs: set-public-provider: runs-on: ubuntu-latest @@ -47,188 +43,69 @@ jobs: get-update-version: uses: ./.github/workflows/template-get-update-version.yml - build-and-test: - runs-on: ${{ matrix.runs-on }} + build-macos-x64: + uses: ./.github/workflows/template-build-macos.yml needs: [get-update-version, set-public-provider] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - os: "linux" - name: "amd64" - runs-on: "ubuntu-20-04-cuda-12-0" - cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" - build-deps-cmake-flags: "" - ccache-dir: '' - - os: "mac" - name: "amd64" - runs-on: "macos-13" - cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" - build-deps-cmake-flags: "" - ccache-dir: '' - - os: "mac" - name: "arm64" - runs-on: "macos-latest" - cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DMAC_ARM64=ON -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" - build-deps-cmake-flags: "" - ccache-dir: '' - - os: "windows" - name: "amd64" - runs-on: "windows-cuda-12-0" - cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DBUILD_SHARED_LIBS=OFF -DCMAKE_TOOLCHAIN_FILE=C:/w/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja" - build-deps-cmake-flags: "-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja" - ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' - steps: - - name: Clone - id: checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Get Cer for code signing - if: runner.os == 'macOS' - run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 - shell: bash - env: - NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} - - - name: Install gettext-bast linux - if: runner.os == 'Linux' - run: sudo apt update && sudo apt install gettext-base -y - - - uses: apple-actions/import-codesign-certs@v2 - continue-on-error: true - if: runner.os == 'macOS' - with: - p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} - p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} - - - uses: actions/setup-dotnet@v3 - if: runner.os == 'Windows' - with: - dotnet-version: "8.0.x" - - - name: Install choco on Windows - if: runner.os == 'Windows' - run: | - choco install make pkgconfiglite ccache awscli 7zip ninja -y - dotnet tool install --global AzureSignTool - - - name: Download ccache from s3 - if: runner.os == 'Windows' - continue-on-error: true - run: | - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - aws s3 cp s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-cpp-${{ matrix.os }}-${{ matrix.name }} ${{ matrix.ccache-dir }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }} - env: - AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}" - AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" - - - name: Configure vcpkg - run: | - cd engine - make configure-vcpkg - - - name: Build - run: | - cd engine - make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}" - - - name: Pre-package - run: | - cd engine - make pre-package - - - name: Code Signing binaries - run: | - cd engine - make codesign-binary CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}" AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}" - - - name: Notary macOS Binary - if: runner.os == 'macOS' - run: | - curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b /usr/local/bin - cd engine/cortex - # Notarize the binary - quill notarize ./cortex - env: - QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} - QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} - QUILL_NOTARY_KEY: "/tmp/notary-key.p8" - - - name: Build Installers - shell: bash - run: | - cd engine - make build-installer PACKAGE_NAME=cortexcpp-nightly VERSION=${{ needs.get-update-version.outputs.new_version }} SOURCE_BINARY_PATH="../../cortex/cortex" DESTINATION_BINARY_NAME="cortex-nightly" DATA_FOLDER_NAME=".cortex-nightly" CONFIGURATION_FILE_NAME=".cortexrc-nightly" UNINSTALLER_FILE_NAME="cortex-nightly-uninstall.sh" - - - name: Codesign and notary for macos installer - if: runner.os == 'macOS' - run: | - cd engine - productsign --sign "Developer ID Installer: ${{ secrets.DEVELOPER_ID }}" cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}-signed.pkg - rm cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg - mv cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}-signed.pkg cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg - xcrun notarytool submit cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg --apple-id ${{ secrets.APPLE_ID }} --password ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} --team-id ${{ secrets.APPLE_TEAM_ID }} --wait - - - name: Compile .ISS to .EXE Installer - uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 - if: runner.os == 'Windows' - with: - path: engine/templates/windows/installer-nightly.iss - options: /O+ - - - name: Codesign for windows installer - if: runner.os == 'Windows' - run: | - cd engine + secrets: inherit + with: + ref: ${{ needs.set-public-provider.outputs.ref }} + public_provider: ${{ needs.set-public-provider.outputs.public_provider }} + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: macos-12 + cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: nightly + arch: amd64 + + build-macos-arm64: + uses: ./.github/workflows/template-build-macos.yml + needs: [get-update-version, set-public-provider] + secrets: inherit + with: + ref: ${{ needs.set-public-provider.outputs.ref }} + public_provider: ${{ needs.set-public-provider.outputs.public_provider }} + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: macos-latest + cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DMAC_ARM64=ON -DCMAKE_TOOLCHAIN_FILE=/Users/runner/work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: nightly + arch: arm64 + + build-windows-x64: + uses: ./.github/workflows/template-build-windows-x64.yml + secrets: inherit + needs: [get-update-version, set-public-provider] + with: + ref: ${{ needs.set-public-provider.outputs.ref }} + public_provider: ${{ needs.set-public-provider.outputs.public_provider }} + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: windows-cuda-11-7 + cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DBUILD_SHARED_LIBS=OFF -DCMAKE_TOOLCHAIN_FILE=C:/w/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja" + build-deps-cmake-flags: "-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja" + ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' + channel: nightly - set PATH=%PATH%;%USERPROFILE%\.dotnet\tools - azuresigntool.exe sign -kvu ${{ secrets.AZURE_KEY_VAULT_URI }} -kvi ${{ secrets.AZURE_CLIENT_ID }} -kvt ${{ secrets.AZURE_TENANT_ID }} -kvs ${{ secrets.AZURE_CLIENT_SECRET }} -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\templates\windows\setup.exe";' - - - name: Package - run: | - cd engine - make package - - - name: Upload Artifact - uses: actions/upload-artifact@v2 - with: - name: cortex-${{ matrix.os }}-${{ matrix.name }} - path: ./engine/cortex - - - name: Upload windwos installer - if: runner.os == 'Windows' - uses: actions/upload-artifact@v2 - with: - name: cortex-${{ matrix.os }}-installer-${{ matrix.name }} - path: ./engine/templates/windows/setup.exe - - - name: Upload macos installer - if: runner.os == 'macOS' - uses: actions/upload-artifact@v2 - with: - name: cortex-${{ matrix.os }}-installer-${{ matrix.name }} - path: ./engine/cortexcpp-nightly.pkg - - - name: Upload linux installer - if: runner.os == 'Linux' - uses: actions/upload-artifact@v2 - with: - name: cortex-${{ matrix.os }}-installer-${{ matrix.name }} - path: ./engine/cortexcpp-nightly.deb - - - name: Upload ccache to s3 - continue-on-error: true - if: always() && runner.os == 'Windows' + build-linux-x64: + uses: ./.github/workflows/template-build-linux-x64.yml + secrets: inherit + needs: [get-update-version, set-public-provider] + with: + ref: ${{ needs.set-public-provider.outputs.ref }} + public_provider: ${{ needs.set-public-provider.outputs.public_provider }} + new_version: ${{ needs.get-update-version.outputs.new_version }} + runs-on: ubuntu-20-04 + cmake-flags: "-DCORTEX_CPP_VERSION=${{ needs.get-update-version.outputs.new_version }} -DCMAKE_TOOLCHAIN_FILE=/home/runner/actions-runner/_work/cortex.cpp/cortex.cpp/engine/vcpkg/scripts/buildsystems/vcpkg.cmake" + channel: nightly + + update-latest-version: + runs-on: ubuntu-latest + needs: [get-update-version, set-public-provider, build-linux-x64, build-macos-x64, build-macos-arm64, build-windows-x64] + steps: + - name: Update latest version + id: update-latest-version run: | - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - aws s3 cp ${{ matrix.ccache-dir }} s3://${{ secrets.MINIO_BUCKET_NAME }}/cortex-${{ matrix.os }}-${{ matrix.name }} --recursive --endpoint ${{ secrets.MINIO_ENDPOINT }} + echo "{\"tag_name\": \"v${{ needs.get-update-version.outputs.new_version }}\"}" > version.json + aws s3 cp version.json s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/version.json env: - AWS_ACCESS_KEY_ID: "${{ secrets.MINIO_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}" - AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" \ No newline at end of file + AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.DELTA_AWS_REGION }} + AWS_EC2_METADATA_DISABLED: "true" \ No newline at end of file diff --git a/.github/workflows/platform-openai-coverage.yml b/.github/workflows/platform-openai-coverage.yml index 010113f53..c432066a6 100644 --- a/.github/workflows/platform-openai-coverage.yml +++ b/.github/workflows/platform-openai-coverage.yml @@ -95,7 +95,7 @@ jobs: AWS_EC2_METADATA_DISABLED: "true" - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: report path: | diff --git a/.github/workflows/template-build-linux-x64.yml b/.github/workflows/template-build-linux-x64.yml index e694cd3eb..5c43c2d60 100644 --- a/.github/workflows/template-build-linux-x64.yml +++ b/.github/workflows/template-build-linux-x64.yml @@ -1,4 +1,4 @@ -name: build-windows-x64 +name: build-linux-x64 on: workflow_call: inputs: @@ -16,7 +16,7 @@ on: type: string default: '' upload_url: - required: true + required: false type: string default: '' runs-on: @@ -45,28 +45,6 @@ on: default: 'nightly' description: 'The channel to use for this job' secrets: - MINIO_BUCKET_NAME: - required: false - MINIO_ENDPOINT: - required: false - MINIO_ACCESS_KEY_ID: - required: false - MINIO_SECRET_ACCESS_KEY: - required: false - MINIO_REGION: - required: false - DEVELOPER_ID: - required: false - AZURE_KEY_VAULT_URI: - required: false - AZURE_CLIENT_ID: - required: false - AZURE_TENANT_ID: - required: false - AZURE_CLIENT_SECRET: - required: false - AZURE_CERT_NAME: - required: false DELTA_AWS_S3_BUCKET_NAME: required: false DELTA_AWS_ACCESS_KEY_ID: @@ -77,7 +55,7 @@ on: required: false jobs: - build-windows-x64: + build-linux-x64: runs-on: ${{ inputs.runs-on }} permissions: contents: write @@ -86,11 +64,12 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ inputs.ref }} + submodules: 'recursive' - - uses: actions/setup-dotnet@v3 - if: runner.os == 'Windows' + - name: use python 3.9 + uses: actions/setup-python@v4 with: - dotnet-version: "8.0.x" + python-version: '3.9' - name: Set output params for each channel id : set-output-params @@ -127,4 +106,79 @@ jobs: fi - name: Install jq - uses: dcarbone/install-jq-action@v2.0.1 \ No newline at end of file + uses: dcarbone/install-jq-action@v2.0.1 + + - name: Install dependencies linux + run: | + sudo apt update && sudo apt install gettext-base -y + python3 -m pip install awscli + + - name: Configure vcpkg + run: | + cd engine + make configure-vcpkg + + - name: Build + run: | + cd engine + make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}" + + - name: Pre-package + run: | + cd engine + make pre-package + + - name: Build Installers + shell: bash + run: | + cd engine + make build-installer PACKAGE_NAME="${{ steps.set-output-params.outputs.package_name }}" VERSION=${{ inputs.new_version }} DESTINATION_BINARY_NAME="${{ steps.set-output-params.outputs.destination_binary_name }}" DATA_FOLDER_NAME="${{ steps.set-output-params.outputs.data_folder_name }}" CONFIGURATION_FILE_NAME="${{ steps.set-output-params.outputs.configuration_file_name }}" UNINSTALLER_FILE_NAME="${{ steps.set-output-params.outputs.uninstaller_file_name }}" + + - name: Package + run: | + cd engine + make package + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: cortex-${{ inputs.new_version }}-linux-amd64 + path: ./engine/cortex + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: cortex-${{ inputs.new_version }}-linux-amd64-installer + path: ./engine/${{ steps.set-output-params.outputs.package_name }}.deb + + - name: upload to aws s3 if public provider is aws + if: inputs.public_provider == 'aws-s3' + run: | + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-amd64/cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/linux-amd64/cortex-linux-amd64-installer-${{ inputs.new_version }}.deb + + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/linux-amd64/cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}.deb s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/linux-amd64/cortex-linux-amd64-installer-${{ inputs.new_version }}.deb + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.DELTA_AWS_REGION }} + AWS_EC2_METADATA_DISABLED: "true" + + - name: Upload release assert if public provider is github + if: inputs.public_provider == 'github' + uses: actions/upload-release-asset@v1.0.1 + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ./engine/cortex.tar.gz + asset_name: cortex-${{ inputs.new_version }}-linux-amd64.tar.gz + asset_content_type: application/zip + + - name: Upload release assert if public provider is github + if: inputs.public_provider == 'github' + uses: actions/upload-release-asset@v1.0.1 + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ./engine/${{ steps.set-output-params.outputs.package_name }}.deb + asset_name: cortex-${{ inputs.new_version }}-linux-amd64-installer.deb + asset_content_type: application/octet-stream \ No newline at end of file diff --git a/.github/workflows/template-build-macos.yml b/.github/workflows/template-build-macos.yml new file mode 100644 index 000000000..ccb0a2e79 --- /dev/null +++ b/.github/workflows/template-build-macos.yml @@ -0,0 +1,233 @@ +name: build-mac-x64 +on: + workflow_call: + inputs: + ref: + required: true + type: string + default: 'refs/heads/main' + public_provider: + required: true + type: string + default: none + description: 'none: build only, github: build and publish to github, aws s3: build and publish to aws s3' + new_version: + required: true + type: string + default: '' + upload_url: + required: false + type: string + default: '' + runs-on: + required: false + type: string + default: 'macos-latest' + description: 'The runner to use for this job' + cmake-flags: + required: false + type: string + default: '' + description: 'The cmake flags to use for this job' + build-deps-cmake-flags: + required: false + type: string + default: '' + description: 'The cmake flags to use for this job' + ccache-dir: + required: false + type: string + default: '' + description: 'The ccache directory to use for this job' + channel: + required: true + type: string + default: 'nightly' + description: 'The channel to use for this job' + arch: + required: true + type: string + default: 'arm64' + description: 'The architecture to use for this job' + secrets: + DELTA_AWS_S3_BUCKET_NAME: + required: false + DELTA_AWS_ACCESS_KEY_ID: + required: false + DELTA_AWS_SECRET_ACCESS_KEY: + required: false + DELTA_AWS_REGION: + required: false + NOTARIZE_P8_BASE64: + required: false + CODE_SIGN_P12_BASE64: + required: false + CODE_SIGN_P12_PASSWORD: + required: false + DEVELOPER_ID: + required: false + NOTARY_KEY_ID: + required: false + NOTARY_ISSUER: + required: false + APPLE_ID: + required: false + APPLE_APP_SPECIFIC_PASSWORD: + required: false + APPLE_TEAM_ID: + required: false + +jobs: + build-mac-x64: + runs-on: ${{ inputs.runs-on }} + permissions: + contents: write + steps: + - name: Getting the repo + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + submodules: 'recursive' + + - name: Set output params for each channel + id : set-output-params + shell: bash + run: | + # Set output for stable channel + if [ "${{ inputs.channel }}" == "stable" ]; then + echo "::set-output name=package_name::cortexcpp" + echo "::set-output name=destination_binary_name::cortex" + echo "::set-output name=data_folder_name::.cortex" + echo "::set-output name=configuration_file_name::.cortexrc" + echo "::set-output name=uninstaller_file_name::cortex-uninstall.sh" + echo "::set-output name=iss_file_name::installer.iss" + fi + + # Set output for beta channel + if [ "${{ inputs.channel }}" == "beta" ]; then + echo "::set-output name=package_name::cortexcpp-beta" + echo "::set-output name=destination_binary_name::cortex-beta" + echo "::set-output name=data_folder_name::.cortex-beta" + echo "::set-output name=configuration_file_name::.cortexrc-beta" + echo "::set-output name=uninstaller_file_name::cortex-beta-uninstall.sh" + echo "::set-output name=iss_file_name::installer-beta.iss" + fi + + # Set output for nightly channel + if [ "${{ inputs.channel }}" == "nightly" ]; then + echo "::set-output name=package_name::cortexcpp-nightly" + echo "::set-output name=destination_binary_name::cortex-nightly" + echo "::set-output name=data_folder_name::.cortex-nightly" + echo "::set-output name=configuration_file_name::.cortexrc-nightly" + echo "::set-output name=uninstaller_file_name::cortex-nightly-uninstall.sh" + echo "::set-output name=iss_file_name::installer-nightly.iss" + fi + + - name: Install jq + uses: dcarbone/install-jq-action@v2.0.1 + + - name: Get Cer for code signing + run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 + shell: bash + env: + NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} + + - uses: apple-actions/import-codesign-certs@v2 + continue-on-error: true + with: + p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} + + - name: Configure vcpkg + run: | + cd engine + make configure-vcpkg + + - name: Build + run: | + cd engine + make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}" + + - name: Pre-package + run: | + cd engine + make pre-package + + - name: Code Signing binaries + run: | + cd engine + make codesign-binary CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}" + + - name: Notary macOS Binary + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b /usr/local/bin + cd engine/cortex + # Notarize the binary + quill notarize ./cortex + env: + QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} + QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} + QUILL_NOTARY_KEY: "/tmp/notary-key.p8" + + - name: Build Installers + shell: bash + run: | + cd engine + make build-installer PACKAGE_NAME="${{ steps.set-output-params.outputs.package_name }}" VERSION=${{ inputs.new_version }} DESTINATION_BINARY_NAME="${{ steps.set-output-params.outputs.destination_binary_name }}" DATA_FOLDER_NAME="${{ steps.set-output-params.outputs.data_folder_name }}" CONFIGURATION_FILE_NAME="${{ steps.set-output-params.outputs.configuration_file_name }}" UNINSTALLER_FILE_NAME="${{ steps.set-output-params.outputs.uninstaller_file_name }}" + + - name: Codesign and notary for macos installer + run: | + cd engine + productsign --sign "Developer ID Installer: ${{ secrets.DEVELOPER_ID }}" ${{ steps.set-output-params.outputs.package_name }}.pkg ${{ steps.set-output-params.outputs.package_name }}$-signed.pkg + rm ${{ steps.set-output-params.outputs.package_name }}.pkg + mv ${{ steps.set-output-params.outputs.package_name }}$-signed.pkg ${{ steps.set-output-params.outputs.package_name }}.pkg + xcrun notarytool submit ${{ steps.set-output-params.outputs.package_name }}.pkg --apple-id ${{ secrets.APPLE_ID }} --password ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} --team-id ${{ secrets.APPLE_TEAM_ID }} --wait + + - name: Package + run: | + cd engine + make package + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: cortex-${{ inputs.new_version }}-mac-${{ inputs.arch}} + path: ./engine/cortex + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: cortex-${{ inputs.new_version }}-mac-${{ inputs.arch}}-installer + path: ./engine/${{ steps.set-output-params.outputs.package_name }}.pkg + + - name: upload to aws s3 if public provider is aws + if: inputs.public_provider == 'aws-s3' + run: | + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/mac-${{ inputs.arch}}/cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}.pkg s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/mac-${{ inputs.arch}}/cortex-mac-${{ inputs.arch}}-installer-${{ inputs.new_version }}.pkg + + aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/mac-${{ inputs.arch}}/cortex-nightly.tar.gz + aws s3 cp ./engine/${{ steps.set-output-params.outputs.package_name }}.pkg s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/mac-${{ inputs.arch}}/cortex-mac-${{ inputs.arch}}-installer-${{ inputs.new_version }}.pkg + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.DELTA_AWS_REGION }} + AWS_EC2_METADATA_DISABLED: "true" + + - name: Upload release assert if public provider is github + if: inputs.public_provider == 'github' + uses: actions/upload-release-asset@v1.0.1 + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ./engine/cortex.tar.gz + asset_name: cortex-${{ inputs.new_version }}-mac-${{ inputs.arch}}.tar.gz + asset_content_type: application/zip + + - name: Upload release assert if public provider is github + if: inputs.public_provider == 'github' + uses: actions/upload-release-asset@v1.0.1 + with: + upload_url: ${{ inputs.upload_url }} + asset_path: ./engine/${{ steps.set-output-params.outputs.package_name }}.pkg + asset_name: cortex-${{ inputs.new_version }}-mac-${{ inputs.arch}}-installer.pkg + asset_content_type: application/octet-stream diff --git a/.github/workflows/template-build-windows-x64.yml b/.github/workflows/template-build-windows-x64.yml index 11e0a6b45..a7bcb6e4e 100644 --- a/.github/workflows/template-build-windows-x64.yml +++ b/.github/workflows/template-build-windows-x64.yml @@ -16,7 +16,7 @@ on: type: string default: '' upload_url: - required: true + required: false type: string default: '' runs-on: @@ -86,9 +86,9 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ inputs.ref }} + submodules: 'recursive' - uses: actions/setup-dotnet@v3 - if: runner.os == 'Windows' with: dotnet-version: "8.0.x" @@ -135,7 +135,6 @@ jobs: dotnet tool install --global AzureSignTool - name: Download ccache from s3 - if: runner.os == 'Windows' continue-on-error: true run: | Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" @@ -147,6 +146,7 @@ jobs: AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" - name: Configure vcpkg + shell: cmd run: | cd engine make configure-vcpkg @@ -164,34 +164,33 @@ jobs: - name: Code Signing binaries run: | cd engine - make codesign-binary CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}" AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}" + make codesign-binary AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}" - name: Update version in installer.iss using sed - if: runner.os == 'Windows' shell: bash run: | cd engine/templates/windows sed -i "s/AppVersion=1.0/AppVersion=${{ inputs.new_version }}/g" ${{ steps.set-output-params.outputs.iss_file_name }} cat ${{ steps.set-output-params.outputs.iss_file_name }} + cp ${{ steps.set-output-params.outputs.iss_file_name }} ../../../ + ls ../../../ - name: Build Installers shell: bash run: | cd engine make build-installer PACKAGE_NAME=${{ steps.set-output-params.outputs.package_name }} VERSION=${{ inputs.new_version }} DESTINATION_BINARY_NAME="${{ steps.set-output-params.outputs.destination_binary_name }}" + ls ../ - name: Compile .ISS to .EXE Installer uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 - if: runner.os == 'Windows' with: - path: engine/templates/windows/${{ steps.set-output-params.outputs.iss_file_name }} + path: ${{ steps.set-output-params.outputs.iss_file_name }} options: /O+ - name: Codesign for windows installer - if: runner.os == 'Windows' run: | - cd engine - %USERPROFILE%\.dotnet\tools\azuresigntool.exe sign -kvu ${{ secrets.AZURE_KEY_VAULT_URI }} -kvi ${{ secrets.AZURE_CLIENT_ID }} -kvt ${{ secrets.AZURE_TENANT_ID }} -kvs ${{ secrets.AZURE_CLIENT_SECRET }} -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\templates\windows\setup.exe";' + %USERPROFILE%\.dotnet\tools\azuresigntool.exe sign -kvu ${{ secrets.AZURE_KEY_VAULT_URI }} -kvi ${{ secrets.AZURE_CLIENT_ID }} -kvt ${{ secrets.AZURE_TENANT_ID }} -kvs ${{ secrets.AZURE_CLIENT_SECRET }} -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\setup.exe";' - name: Package run: | @@ -199,34 +198,25 @@ jobs: make package - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: cortex-windows-amd64-${{ inputs.new_version }} + name: cortex-${{ inputs.new_version }}-windows-amd64 path: ./engine/cortex - name: Upload Artifact - uses: actions/upload-artifact@v2 - with: - name: cortex-windows-amd64-installer-${{ inputs.new_version }} - path: ./engine/templates/windows/setup.exe - - - name: Upload release assert if public provider is github - if: ${{ inputs.public_provider }} == 'github' - uses: actions/upload-release-asset@v1.0.1 + uses: actions/upload-artifact@v4 with: - upload_url: ${{ inputs.upload_url }} - asset_path: ./engine/cortex.tar.gz - asset_name: cortex-${{ inputs.new_version }}-windows-amd64.tar.gz - asset_content_type: application/zip + name: cortex-${{ inputs.new_version }}-windows-amd64-installer + path: ./setup.exe - name: upload to aws s3 if public provider is aws - if: ${{ inputs.public_provider }} == 'aws-s3' + if: inputs.public_provider == 'aws-s3' run: | aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/windows-amd64/cortex-nightly.tar.gz - aws s3 cp ./engine/templates/windows/setup.exe s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/windows-amd64/cortex-windows-amd64-installer-${{ inputs.new_version }}.exe + aws s3 cp ./setup.exe s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/latest/windows-amd64/cortex-${{ inputs.new_version }}-windows-amd64-installer.exe aws s3 cp ./engine/cortex.tar.gz s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/windows-amd64/cortex-nightly.tar.gz - aws s3 cp ./engine/templates/windows/setup.exe s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/windows-amd64/cortex-windows-amd64-installer-${{ inputs.new_version }}.exe + aws s3 cp ./setup.exe s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/cortex/${{ inputs.new_version }}/windows-amd64/cortex-${{ inputs.new_version }}-windows-amd64-installer.exe env: AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} @@ -234,7 +224,7 @@ jobs: AWS_EC2_METADATA_DISABLED: "true" - name: Upload release assert if public provider is github - if: ${{ inputs.public_provider }} == 'github' + if: inputs.public_provider == 'github' uses: actions/upload-release-asset@v1.0.1 with: upload_url: ${{ inputs.upload_url }} @@ -243,10 +233,10 @@ jobs: asset_content_type: application/zip - name: Upload release assert if public provider is github - if: ${{ inputs.public_provider }} == 'github' + if: inputs.public_provider == 'github' uses: actions/upload-release-asset@v1.0.1 with: upload_url: ${{ inputs.upload_url }} - asset_path: ./engine/templates/windows/setup.exe - asset_name: cortex-windows-amd64-installer-${{ inputs.new_version }}.exe - asset_content_type: application/zip \ No newline at end of file + asset_path: ./setup.exe + asset_name: cortex-${{ inputs.new_version }}-windows-amd64-installer.exe + asset_content_type: application/octet-stream \ No newline at end of file diff --git a/engine/Makefile b/engine/Makefile index 877eba4fd..9f72f8355 100644 --- a/engine/Makefile +++ b/engine/Makefile @@ -29,8 +29,8 @@ all: configure-vcpkg: ifeq ($(OS),Windows_NT) - @cd vcpkg && bootstrap-vcpkg.bat; - @cd vcpkg && vcpkg install + @cd vcpkg && cmd /c bootstrap-vcpkg.bat; + @cd vcpkg && cmd /c vcpkg install else ifeq ($(shell uname -s),Linux) @cd vcpkg && ./bootstrap-vcpkg.sh; @cd vcpkg && ./vcpkg install; @@ -76,7 +76,7 @@ endif ifeq ($(OS),Windows_NT) @powershell -Command "dotnet tool install --global AzureSignTool;" - @powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex\cortex.exe";' + @powershell -Command '~\.dotnet\tools\azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex\cortex.exe";' else ifeq ($(shell uname -s),Linux) @echo "Skipping Code Sign for linux" @exit 0 @@ -95,20 +95,20 @@ endif build-installer: ifeq ($(OS),Windows_NT) - @echo "Building installer for linux"; \ - cp cortex/* templates/windows/; \ - mv templates/windows/cortex.exe templates/windows/$(DESTINATION_BINARY_NAME).exe; + @echo "Building installer for Windows"; \ + cp cortex/* ../; \ + mv ../cortex.exe ../$(DESTINATION_BINARY_NAME).exe; else ifeq ($(shell uname -s),Linux) @echo "Building installer for linux"; \ cd templates/linux; \ chmod +x create_deb.sh; \ - ./create_deb.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME); + ./create_deb.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME); \ cp $(PACKAGE_NAME).deb ../../ else - @echo "Building installer for linux"; \ + @echo "Building installer for Macos"; \ cd templates/macos; \ chmod +x create_pkg.sh; \ - ./create_pkg.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME) $(UNINSTALLER_FILE_NAME); + ./create_pkg.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME) $(UNINSTALLER_FILE_NAME); \ cp $(PACKAGE_NAME).pkg ../../ endif diff --git a/engine/templates/macos/create_pkg.sh b/engine/templates/macos/create_pkg.sh index ed3c80cfe..61adbd2b3 100644 --- a/engine/templates/macos/create_pkg.sh +++ b/engine/templates/macos/create_pkg.sh @@ -13,13 +13,13 @@ cp $SOURCE_BINARY_PATH installer/$DESTINATION_BINARY_NAME export DESTINATION_BINARY_NAME cp post-installer.sh scripts/post-installer.sh -sed -i '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' $PACKAGE_NAME/DEBIAN/postinst +sed -i '' '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' $PACKAGE_NAME/DEBIAN/postinst export DATA_FOLDER_NAME CONFIGURATION_FILE_NAME UNINSTALLER_FILE_NAME cp cortex-uninstall.sh scripts/$UNINSTALLER_FILE_NAME -sed -i '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' scripts/$UNINSTALLER_FILE_NAME -sed -i '3s/.*/DATA_FOLDER_NAME=\$DATA_FOLDER_NAME/' scripts/$UNINSTALLER_FILE_NAME -sed -i '4s/.*/CONFIGURATION_FILE_NAME=\$CONFIGURATION_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME -sed -i '5s/.*/UNINSTALLER_FILE_NAME=\$UNINSTALLER_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME +sed -i '' '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' scripts/$UNINSTALLER_FILE_NAME +sed -i '' '3s/.*/DATA_FOLDER_NAME=\$DATA_FOLDER_NAME/' scripts/$UNINSTALLER_FILE_NAME +sed -i '' '4s/.*/CONFIGURATION_FILE_NAME=\$CONFIGURATION_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME +sed -i '' '5s/.*/UNINSTALLER_FILE_NAME=\$UNINSTALLER_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME pkgbuild --identifier ai.cortexcpp.pkg --version $VERSION --scripts scripts --install-location /usr/local/bin --root ./installer ${PACKAGE_NAME}.pkg