chore: upgrade to turbo v2 and reduce ci quality gate runtime

.github/workflows/jan-electron-linter-and-test.yml

@@ -66,7 +66,7 @@ jobs:
 
   test-on-macos:
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-    runs-on: [self-hosted, macOS, macos-desktop]
+    runs-on: macos-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -100,18 +100,13 @@ jobs:
 
       - name: Linter and test
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
         env:
           CSC_IDENTITY_AUTO_DISCOVERY: 'false'
-          # TURBO_API: '${{ secrets.TURBO_API }}'
-          # TURBO_TEAM: 'macos'
-          # TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
   test-on-macos-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
-    runs-on: [self-hosted, macOS, macos-desktop]
+    runs-on: macos-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -131,8 +126,6 @@ jobs:
 
       - name: Linter and test
         run: |
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
           make test
         env:
           CSC_IDENTITY_AUTO_DISCOVERY: 'false'
@@ -182,13 +175,8 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'windows'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
+
   test-on-windows-pr:
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: windows-desktop-default-windows-security
@@ -230,17 +218,11 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'windows'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
   test-on-windows-pr-target:
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
-    runs-on: windows-desktop-default-windows-security
+    runs-on: windows-latest
     steps:
       - name: Getting the repo
         uses: actions/checkout@v3
@@ -268,12 +250,10 @@ jobs:
       - name: Linter and test
         shell: powershell
         run: |
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
           make test
 
   test-on-ubuntu:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     steps:
       - name: Getting the repo
@@ -307,17 +287,15 @@ jobs:
         run: |
           make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
 
+      - name: Setup Xvfb
+        run: |
+          sudo apt-get install xvfb
+
       - name: Linter and test
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
-          make test
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'linux'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
+          make test-headless
 
       - uses: actions/upload-artifact@v4
         if: always()
@@ -327,7 +305,7 @@ jobs:
           retention-days: 2
 
   coverage-check:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     needs: base_branch_cov
     if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     steps:
@@ -341,6 +319,9 @@ jobs:
         with:
           node-version: 20
 
+      - name: Install yarn
+        run: npm install -g yarn
+
       - name: 'Cleanup cache'
         continue-on-error: true
         run: |
@@ -356,15 +337,9 @@ jobs:
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry ${{ secrets.NPM_PROXY }} --global
-          yarn config set registry ${{ secrets.NPM_PROXY }} --global
           make lint
           yarn build:test
           yarn test:coverage
-        # env:
-        #   TURBO_API: '${{ secrets.TURBO_API }}'
-        #   TURBO_TEAM: 'linux'
-        #   TURBO_TOKEN: '${{ secrets.TURBO_TOKEN }}'
 
       - name: Generate Code Coverage report
         id: code-coverage
@@ -377,7 +352,7 @@ jobs:
           show-annotations: 'warning'
 
   test-on-ubuntu-pr-target:
-    runs-on: [self-hosted, Linux, ubuntu-desktop]
+    runs-on: ubuntu-latest
     if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Getting the repo
@@ -396,10 +371,12 @@ jobs:
           rm -rf ~/jan
           make clean
 
+      - name: Setup Xvfb
+        run: |
+          sudo apt-get install xvfb
+
       - name: Linter and test
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
           echo -e "Display ID: $DISPLAY"
-          npm config set registry https://registry.npmjs.org --global
-          yarn config set registry https://registry.npmjs.org --global
-          make test
+          make test-headless

.github/workflows/publish-npm-core.yml

@@ -1,4 +1,4 @@
-name: Publish plugin models Package to npmjs
+name: Publish core Package to npmjs
 on:
   push:
     tags: ["v[0-9]+.[0-9]+.[0-9]+-core"]
@@ -45,7 +45,7 @@ jobs:
           node-version: "20.x"
           registry-url: "https://registry.npmjs.org"
 
-      - run: cd core && yarn install && yarn build
+      - run: cd core && corepack enable && corepack prepare [email protected] --activate && yarn --version && yarn install && yarn build
 
       - run: cd core && yarn publish --access public
         if: github.event_name == 'push'

.github/workflows/publish-npm-joi.yml

@@ -1,4 +1,4 @@
-name: Publish plugin models Package to npmjs
+name: Publish joi Package to npmjs
 on:
   push:
     tags: ["v[0-9]+.[0-9]+.[0-9]+-joi"]
@@ -45,7 +45,7 @@ jobs:
           node-version: "20.x"
           registry-url: "https://registry.npmjs.org"
 
-      - run: cd joi && yarn install && yarn build
+      - run: cd joi && corepack enable && corepack prepare [email protected] --activate && yarn --version && yarn install && yarn build
 
       - run: cd joi && yarn publish --access public
         if: github.event_name == 'push'

.gitignore

@@ -8,7 +8,6 @@ error.log
 node_modules
 *.tgz
 !charts/server/charts/*.tgz
-yarn.lock
 dist
 build
 .DS_Store
@@ -48,3 +47,6 @@ coverage
 test_results.html
 *.tsbuildinfo
 electron/shared/**
+
+# docs
+docs/yarn.lock

.yarnrc.yml

@@ -0,0 +1,3 @@
+nmHoistingLimits: workspaces
+nodeLinker: node-modules
+checksumBehavior: update

Makefile

@@ -10,20 +10,23 @@ REPORT_PORTAL_DESCRIPTION ?= "Jan App report"
 all:
 	@echo "Specify a target to run"
 
+# Config yarn version
+
+config-yarn:
+	corepack enable
+	corepack prepare [email protected] --activate
+	yarn --version
+	yarn config set -H enableImmutableInstalls false
+
 # Builds the UI kit
-build-joi:
-ifeq ($(OS),Windows_NT)
-	cd joi && yarn config set network-timeout 300000 && yarn install && yarn build
-else
+build-joi: config-yarn
 	cd joi && yarn install && yarn build
-endif
 
 # Installs yarn dependencies and builds core and extensions
 install-and-build: build-joi
 ifeq ($(OS),Windows_NT)
-	yarn config set network-timeout 300000
+	echo "skip"
 endif
-	yarn global add [email protected]
 	yarn build:core
 	yarn build:server
 	yarn install
@@ -107,6 +110,11 @@ test: lint
 	yarn test:coverage
 	yarn test
 
+test-headless: lint
+	yarn build:test
+	yarn test:coverage
+	yarn test:e2e:headless
+
 # Builds and publishes the app
 build-and-publish: check-file-counts
 	yarn build:publish
@@ -117,9 +125,8 @@ build: check-file-counts
 
 clean:
 ifeq ($(OS),Windows_NT)
-	-powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo -Recurse -Directory | Remove-Item -Recurse -Force"
-	-powershell -Command "Get-ChildItem -Path . -Include package-lock.json -Recurse -File | Remove-Item -Recurse -Force"
-	-powershell -Command "Get-ChildItem -Path . -Include yarn.lock -Recurse -File | Remove-Item -Recurse -Force"
+	-powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo, .yarn -Recurse -Directory | Remove-Item -Recurse -Force"
+	-powershell -Command "Get-ChildItem -Path . -Include package-lock.json, tsconfig.tsbuildinfo -Recurse -File | Remove-Item -Recurse -Force"
 	-powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz"
 	-powershell -Command "Remove-Item -Recurse -Force ./extensions/*/*.tgz"
 	-powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz"
@@ -131,8 +138,8 @@ else ifeq ($(shell uname -s),Linux)
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
 	find . -name ".turbo" -type d -exec rm -rf '{}' +
+	find . -name ".yarn" -type d -exec rm -rf '{}' +
 	find . -name "packake-lock.json" -type f -exec rm -rf '{}' +
-	find . -name "yarn.lock" -type f -exec rm -rf '{}' +
 	find . -name "package-lock.json" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./extensions/*/*.tgz
@@ -146,8 +153,8 @@ else
 	find . -name "build" -type d -exec rm -rf '{}' +
 	find . -name "out" -type d -exec rm -rf '{}' +
 	find . -name ".turbo" -type d -exec rm -rf '{}' +
+	find . -name ".yarn" -type d -exec rm -rf '{}' +
 	find . -name "package-lock.json" -type f -exec rm -rf '{}' +
-	find . -name "yarn.lock" -type f -exec rm -rf '{}' +
 	rm -rf ./pre-install/*.tgz
 	rm -rf ./extensions/*/*.tgz
 	rm -rf ./electron/pre-install/*.tgz

core/package.json

@@ -8,8 +8,9 @@
   ],
   "homepage": "https://jan.ai",
   "license": "AGPL-3.0",
+  "browser": "dist/core.es5.js",
   "main": "dist/core.es5.js",
-  "module": "dist/core.cjs.js",
+  "module": "dist/index.cjs.js",
   "typings": "dist/types/index.d.ts",
   "files": [
     "dist",
@@ -36,18 +37,24 @@
     "lint": "tslint --project tsconfig.json -t codeFrame 'src/**/*.ts' 'test/**/*.ts'",
     "test": "jest",
     "prebuild": "rimraf dist",
-    "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "start": "rollup -c rollup.config.ts -w"
+    "build": "rollup -c rollup.config.ts"
   },
   "devDependencies": {
+    "@npmcli/arborist": "^9.0.0",
     "@rollup/plugin-replace": "^5.0.5",
-    "@types/jest": "^29.5.12",
-    "@types/node": "^20.11.4",
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.10.0",
+    "@types/pacote": "^11.1.8",
+    "@types/request": "^2.48.12",
+    "electron": "33.2.1",
     "eslint": "8.57.0",
     "eslint-plugin-jest": "^27.9.0",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0",
     "jest-runner": "^29.7.0",
+    "pacote": "^21.0.0",
+    "request": "^2.88.2",
+    "request-progress": "^3.0.0",
     "rimraf": "^3.0.2",
     "rollup": "^2.38.5",
     "rollup-plugin-commonjs": "^9.1.8",
@@ -62,5 +69,6 @@
   "dependencies": {
     "rxjs": "^7.8.1",
     "ulidx": "^2.3.0"
-  }
+  },
+  "packageManager": "[email protected]"
 }

core/rollup.config.ts

@@ -23,9 +23,12 @@ export default [
       // Allow json resolution
       json(),
       // Compile TypeScript files
-      typescript({ useTsconfigDeclarationDir: true }),
+      typescript({
+        useTsconfigDeclarationDir: true,
+        exclude: ['**/*.test.ts', 'src/node/**'],
+      }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
-      commonjs(),
+      // commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control
       // which external modules to include in the bundle
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
@@ -44,7 +47,9 @@ export default [
   },
   {
     input: `src/node/index.ts`,
-    output: [{ file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }],
+    output: [
+      { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
+    ],
     // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
     external: [
       'fs/promises',
@@ -70,7 +75,10 @@ export default [
       // Allow json resolution
       json(),
       // Compile TypeScript files
-      typescript({ useTsconfigDeclarationDir: true }),
+      typescript({
+        useTsconfigDeclarationDir: true,
+        exclude: ['**/*.test.ts', 'src/browser/**'],
+      }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
       commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control