Makefile add build-installer

.github/workflows/cortex-nightly.yml → .github/workflows/nightly-build.yml

@@ -86,11 +86,30 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Get Cer for code signing
+        if: runner.os == 'macOS'
+        run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8
+        shell: bash
+        env:
+          NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }}
+
+      - uses: apple-actions/import-codesign-certs@v2
+        continue-on-error: true
+        if: runner.os == 'macOS'
+        with:
+          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+
       - name: Install choco on Windows
         if: runner.os == 'Windows'
         run: |
           choco install make pkgconfiglite ccache awscli 7zip ninja -y
 
+      - uses: actions/setup-dotnet@v3
+        if: runner.os == 'Windows'
+        with:
+          dotnet-version: "8.0.x"
+
       - name: Download ccache from s3
         if: runner.os == 'Windows'
         continue-on-error: true
@@ -117,7 +136,54 @@ jobs:
         run: |
           cd engine
           make pre-package
-          
+      
+      - name: Code Signing binaries
+        run: |
+          cd engine
+          make codesign-binary CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }} AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}"
+
+      - name: Notary macOS Binary
+        if: runner.os == 'macOS'
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b /usr/local/bin
+          cd engine/cortex
+          # Notarize the binary
+          quill notarize ./cortex
+        env:
+          QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }}
+          QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }}
+          QUILL_NOTARY_KEY: "/tmp/notary-key.p8"
+
+      - name: Build Installers
+        shell: bash
+        run: |
+          cd engine
+          make build-installer PACKAGE_NAME=cortexcpp-nightly VERSION=${{ needs.get-update-version.outputs.new_version }} SOURCE_BINARY_PATH="../../cortex/cortex" DESTINATION_BINARY_NAME="cortex-nightly" DATA_FOLDER_NAME=".cortex-nightly" CONFIGURATION_FILE_NAME=".cortexrc-nightly" UNINSTALLER_FILE_NAME="cortex-nightly-uninstall.sh"
+
+      - name: Codesign and notary for macos installer
+        if: runner.os == 'macOS'
+        run: |
+          cd engine
+          productsign --sign "Developer ID Installer: ${{ secrets.DEVELOPER_ID }}" cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}-signed.pkg
+          rm cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg
+          mv cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}-signed.pkg cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg
+          xcrun notarytool submit cortexcpp-nightly-${{ needs.get-update-version.outputs.new_version }}.pkg --apple-id ${{ secrets.APPLE_ID }} --password ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} --team-id ${{ secrets.APPLE_TEAM_ID }} --wait
+
+      - name: Compile .ISS to .EXE Installer
+        uses: Minionguyjpro/[email protected]
+        if: runner.os == 'Windows'
+        with:
+          path: engine/templates/windows/installer-nightly.iss
+          options: /O+
+
+      - name: Codesign for windows installer
+        if: runner.os == 'Windows'
+        run: |
+          cd engine
+          dotnet tool install --global AzureSignTool
+          set PATH=%PATH%;%USERPROFILE%\.dotnet\tools
+          azuresigntool.exe sign -kvu ${{ secrets.AZURE_KEY_VAULT_URI }} -kvi ${{ secrets.AZURE_CLIENT_ID }} -kvt ${{ secrets.AZURE_TENANT_ID }} -kvs ${{ secrets.AZURE_CLIENT_SECRET }} -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\templates\windows\setup.exe";' 
+
       - name: Package
         run: |
           cd engine
@@ -129,6 +195,27 @@ jobs:
           name: cortex-${{ matrix.os }}-${{ matrix.name }}
           path: ./engine/cortex
 
+      - name: Upload windwos installer
+        if: runner.os == 'Windows'
+        uses: actions/upload-artifact@v2
+        with:
+          name: cortex-${{ matrix.os }}-installer-${{ matrix.name }}
+          path: ./engine/templates/windows/setup.exe
+
+      - name: Upload macos installer
+        if: runner.os == 'macOS'
+        uses: actions/upload-artifact@v2
+        with:
+          name: cortex-${{ matrix.os }}-installer-${{ matrix.name }}
+          path: ./engine/cortexcpp-nightly.pkg
+
+      - name: Upload linux installer
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: cortex-${{ matrix.os }}-installer-${{ matrix.name }}
+          path: ./engine/cortexcpp-nightly.deb
+
       - name: Upload ccache to s3
         continue-on-error: true
         if: always() && runner.os == 'Windows'

engine/Makefile

@@ -14,6 +14,15 @@ AZURE_CLIENT_SECRET ?= xxxx
 AZURE_CERT_NAME ?= xxxx
 DEVELOPER_ID ?= xxxx
 
+# Arguments for build installer
+PACKAGE_NAME ?= cortexcpp
+VERSION ?= 0.1.1
+SOURCE_BINARY_PATH ?= ../../cortex/cortex
+DESTINATION_BINARY_NAME ?= cortex
+DATA_FOLDER_NAME ?= .cortex
+CONFIGURATION_FILE_NAME ?= .cortexrc
+UNINSTALLER_FILE_NAME ?= cortex-uninstall.sh
+
 # Default target, does nothing
 all:
 	@echo "Specify a target to run"
@@ -59,7 +68,7 @@ else
 	cp build/cortex cortex/;
 endif
 
-codesign:
+codesign-binary:
 ifeq ($(CODE_SIGN),false)
 	@echo "Skipping Code Sign"
 	@exit 0
@@ -68,12 +77,11 @@ endif
 ifeq ($(OS),Windows_NT)
 	@powershell -Command "dotnet tool install --global AzureSignTool;"
 	@powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex\cortex.exe";'
-	@powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex\engines\cortex.llamacpp\engine.dll";'
 else ifeq ($(shell uname -s),Linux)
 	@echo "Skipping Code Sign for linux"
 	@exit 0
 else
-	find "cortex" -type f -exec codesign --force -s "$(DEVELOPER_ID)" --options=runtime {} \;
+	codesign --force -s "$(DEVELOPER_ID)" --options=runtime --entitlements="./templates/macos/entitlements.plist" ./cortex/cortex;
 endif
 
 package:
@@ -85,6 +93,23 @@ else
 	tar -czvf cortex.tar.gz cortex;
 endif
 
+build-installer:
+ifeq ($(OS),Windows_NT)
+	@echo "Building installer for linux"; \
+	cp cortex/* templates/windows/; \
+	mv templates/windows/cortex.exe templates/windows/$(DESTINATION_BINARY_NAME).exe;
+else ifeq ($(shell uname -s),Linux)
+	@echo "Building installer for linux"; \
+	cd templates/linux; \
+	./create_deb.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME);
+	cp $(PACKAGE_NAME).deb ../../
+else
+	@echo "Building installer for linux"; \
+	cd templates/macos; \
+	./create_pkg.sh $(PACKAGE_NAME) $(VERSION) $(SOURCE_BINARY_PATH) $(DESTINATION_BINARY_NAME) $(DATA_FOLDER_NAME) $(CONFIGURATION_FILE_NAME) $(UNINSTALLER_FILE_NAME);
+	cp $(PACKAGE_NAME).pkg ../../
+endif
+
 clean:
 ifeq ($(OS),Windows_NT)
 	@powershell -Command "rm -rf build; rm -rf build-deps; rm -rf cortex; rm -rf cortex.tar.gz;"

engine/templates/linux/create_deb.sh

@@ -2,15 +2,27 @@ PACKAGE_NAME=$1
 VERSION=$2
 SOURCE_BINARY_PATH=$3
 DESTINATION_BINARY_NAME=$4
+DATA_FOLDER_NAME=$5
+CONFIGURATION_FILE_NAME=$6
 
 mkdir -p $PACKAGE_NAME/DEBIAN
 
 mkdir -p $PACKAGE_NAME/usr/bin
 cp $SOURCE_BINARY_PATH $PACKAGE_NAME/usr/bin/$DESTINATION_BINARY_NAME
 
+export DESTINATION_BINARY_NAME
+
 cp postinst $PACKAGE_NAME/DEBIAN/postinst
-cp postrm $PACKAGE_NAME/DEBIAN/postrm
+sed -i '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' $PACKAGE_NAME/DEBIAN/postinst
+
 cp prerm $PACKAGE_NAME/DEBIAN/prerm
+sed -i '3s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' $PACKAGE_NAME/DEBIAN/prerm
+
+export DATA_FOLDER_NAME CONFIGURATION_FILE_NAME
+
+cp postrm $PACKAGE_NAME/DEBIAN/postrm
+sed -i '3s/.*/DATA_FOLDER_NAME=\$DATA_FOLDER_NAME/' $PACKAGE_NAME/DEBIAN/postrm
+sed -i '4s/.*/CONFIGURATION_FILE_NAME=\$CONFIGURATION_FILE_NAME/' $PACKAGE_NAME/DEBIAN/postrm
 
 chmod 755 $PACKAGE_NAME/DEBIAN/postinst
 chmod 755 $PACKAGE_NAME/DEBIAN/postrm
@@ -20,4 +32,4 @@ export PACKAGE_NAME VERSION
 
 envsubst < control > $PACKAGE_NAME/DEBIAN/control
 
-dpkg-deb --build $PACKAGE_NAME $PACKAGE_NAME_$VERSION.deb
+dpkg-deb --build $PACKAGE_NAME $PACKAGE_NAME.deb

engine/templates/linux/postinst

@@ -1,2 +1,5 @@
 #!/bin/sh
-echo "Running post-install script..."
+DESTINATION_BINARY_NAME=cortex
+echo "Download cortex.llamacpp engines by default"
+USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
+sudo -u "$USER_TO_RUN_AS" /usr/bin/$DESTINATION_BINARY_NAME engines cortex.llamacpp install

engine/templates/linux/postrm

@@ -1,18 +1,24 @@
 #!/bin/sh
 set +e
+DATA_FOLDER_NAME=.cortex
+CONFIGURATION_FILE_NAME=.cortexrc
 
-echo "Do you want to delete the 'cortex' data folder for all users? (yes/no)"
+echo "Do you want to delete the '~/${DATA_FOLDER_NAME}' data folder and file '~/${CONFIGURATION_FILE_NAME}'? (yes/no)"
 read -r answer
 
+USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
+
 case "$answer" in
     [yY][eE][sS]|[yY])
-        echo "Deleting 'cortex' data folders..."
-        for userdir in /home/*; do
-            if [ -d "$userdir/cortex" ]; then
-                echo "Removing $userdir/cortex"
-                rm -rf "$userdir/cortex" > /dev/null 2>&1
-            fi
-        done
+        echo "Deleting cortex data folders..."
+        if [ -d "/home/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}" ]; then
+            echo "Removing /home/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}"
+            rm -rf "/home/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}" > /dev/null 2>&1
+        fi
+        if [ -f "/home/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}" ]; then
+            echo "Removing /home/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}"
+            rm -f "/home/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}" > /dev/null 2>&1
+        fi
         ;;
     [nN][oO]|[nN])
         echo "Keeping the 'cortex' data folders."
@@ -21,5 +27,4 @@ case "$answer" in
         echo "Invalid response. Please type 'yes' or 'no'."
         ;;
 esac
-
 exit 0

engine/templates/linux/prerm

@@ -1,6 +1,6 @@
 #!/bin/sh
 set +e
-
-/usr/bin/cortex stop > /dev/null 2>&1
-
+DESTINATION_BINARY_NAME=cortex
+USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
+sudo -u "$USER_TO_RUN_AS"  /usr/bin/$DESTINATION_BINARY_NAME stop > /dev/null 2>&1
 exit 0

engine/templates/macos/cortex-uninstall.sh

@@ -1,25 +1,33 @@
 #!/bin/bash
+DESTINATION_BINARY_NAME=cortex
+DATA_FOLDER_NAME=.cortex
+CONFIGURATION_FILE_NAME=.cortexrc
+UNINSTALLER_FILE_NAME=cortex-uninstall.sh
 
 # required root privileges
 if [ "$EUID" -ne 0 ]
   then echo "Please run as root with sudo"
   exit
 fi
 
-rm /usr/local/bin/cortex
+USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
 
-echo "Do you want to delete the 'cortex' data folder for all users? (yes/no)"
+rm /usr/local/bin/$DESTINATION_BINARY_NAME
+
+echo "Do you want to delete the '~/${DATA_FOLDER_NAME}' data folder and file '~/${CONFIGURATION_FILE_NAME}'? (yes/no)"
 read -r answer
 
 case "$answer" in
     [yY][eE][sS]|[yY])
-        echo "Deleting 'cortex' data folders..."
-        for userdir in /Users/*; do
-            if [ -d "$userdir/cortex" ]; then
-                echo "Removing $userdir/cortex"
-                rm -rf "$userdir/cortex" > /dev/null 2>&1
-            fi
-        done
+        echo "Deleting cortex data folders..."
+        if [ -d "/Users/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}" ]; then
+            echo "Removing /Users/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}"
+            rm -rf "/Users/${USER_TO_RUN_AS}/${DATA_FOLDER_NAME}" > /dev/null 2>&1
+        fi
+        if [ -f "/Users/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}" ]; then
+            echo "Removing /Users/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}"
+            rm -f "/Users/${USER_TO_RUN_AS}/${CONFIGURATION_FILE_NAME}" > /dev/null 2>&1
+        fi
         ;;
     [nN][oO]|[nN])
         echo "Keeping the 'cortex' data folders."
@@ -29,4 +37,4 @@ case "$answer" in
         ;;
 esac
 
-rm /usr/local/bin/cortex-uninstall.sh
+rm /usr/local/bin/$UNINSTALLER_FILE_NAME

engine/templates/macos/create_pkg.sh

@@ -0,0 +1,25 @@
+PACKAGE_NAME=$1
+VERSION=$2
+SOURCE_BINARY_PATH=$3
+DESTINATION_BINARY_NAME=$4
+DATA_FOLDER_NAME=$5
+CONFIGURATION_FILE_NAME=$6
+UNINSTALLER_FILE_NAME=$7
+
+mkdir installer
+mkdir scripts
+
+cp $SOURCE_BINARY_PATH installer/$DESTINATION_BINARY_NAME
+
+export DESTINATION_BINARY_NAME
+cp post-installer.sh scripts/post-installer.sh
+sed -i '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' $PACKAGE_NAME/DEBIAN/postinst
+
+export DATA_FOLDER_NAME CONFIGURATION_FILE_NAME UNINSTALLER_FILE_NAME
+cp cortex-uninstall.sh scripts/$UNINSTALLER_FILE_NAME
+sed -i '2s/.*/DESTINATION_BINARY_NAME=\$DESTINATION_BINARY_NAME/' scripts/$UNINSTALLER_FILE_NAME
+sed -i '3s/.*/DATA_FOLDER_NAME=\$DATA_FOLDER_NAME/' scripts/$UNINSTALLER_FILE_NAME
+sed -i '4s/.*/CONFIGURATION_FILE_NAME=\$CONFIGURATION_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME
+sed -i '5s/.*/UNINSTALLER_FILE_NAME=\$UNINSTALLER_FILE_NAME/' scripts/$UNINSTALLER_FILE_NAME
+
+pkgbuild --identifier ai.cortexcpp.pkg --version $VERSION --scripts scripts --install-location /usr/local/bin --root ./installer ${PACKAGE_NAME}.pkg

engine/templates/macos/post-installer.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+DESTINATION_BINARY_NAME=cortex
+echo "Download cortex.llamacpp engines by default"
+USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
+sudo -u "$USER_TO_RUN_AS" /usr/local/bin/$DESTINATION_BINARY_NAME engines cortex.llamacpp install