From d43f50dededc90aa491dc5546468cd3f7df152cc Mon Sep 17 00:00:00 2001 From: InNoobWeTrust Date: Tue, 6 Feb 2024 13:21:01 +0700 Subject: [PATCH] Revert "chore(nitro-node): split github workflow files into separate commit" This reverts commit 1d79084046cf07b7c11a23199b3eaed2a98b40d3. --- .../scripts/e2e-test-install-nitro-node.js | 138 +++++++ .github/workflows/build-nitro-node.yml | 355 ++++++++++++++++++ .github/workflows/test-install-nitro-node.yml | 163 ++++++++ 3 files changed, 656 insertions(+) create mode 100644 .github/scripts/e2e-test-install-nitro-node.js create mode 100644 .github/workflows/build-nitro-node.yml create mode 100644 .github/workflows/test-install-nitro-node.yml diff --git a/.github/scripts/e2e-test-install-nitro-node.js b/.github/scripts/e2e-test-install-nitro-node.js new file mode 100644 index 000000000..ae2339aec --- /dev/null +++ b/.github/scripts/e2e-test-install-nitro-node.js @@ -0,0 +1,138 @@ +const os = require("node:os"); +const path = require("node:path"); +const fs = require("node:fs"); +const { spawn } = require("node:child_process"); + +// Test on both npm and yarn +const PACKAGE_MANAGERS = ["npm", "yarn"]; +const ADD_DEP_CMDS = { + // Need to copy dependency instead of linking so test logic can check the bin + npm: "install --install-links", + yarn: "add", +}; +// Path to the package to install +const NITRO_NODE_PKG = + process.env.NITRO_NODE_PKG || + path.resolve(path.normalize(path.join(__dirname, "..", "..", "nitro-node"))); +// Prefixes of downloaded nitro bin subdirectories +const BIN_DIR_PREFIXES = { + darwin: "mac", + win32: "win", + linux: "linux", +}; + +// Utility to check for a file with nitro in name in the corresponding directory +const checkBinaries = (repoDir) => { + // FIXME: Check for unsupported platforms + const binDirPrefix = BIN_DIR_PREFIXES[process.platform]; + const searchRoot = path.join( + repoDir, + "node_modules", + "@janhq", + "nitro-node", + "bin", + ); + // Get the dir and files that indicate successful download of binaries + const matched = fs.readdirSync(searchRoot, { recursive: true }).filter( + // FIXME: the result of readdirSync with recursive option is filename + // with intermediate subdirectories so this logic might not be correct + (fname) => fname.startsWith(binDirPrefix) || fname.includes("nitro"), + ); + console.log(`Downloaded bin paths:`, matched); + + // Must have both the directory for the platform and the binary + return matched.length > 1; +}; + +// Wrapper to wait for child process to finish +const childProcessPromise = (childProcess) => + new Promise((resolve, reject) => { + childProcess.on("exit", (exitCode) => { + const exitNum = Number(exitCode); + if (0 == exitNum) { + resolve(); + } else { + reject(exitNum); + } + }); + }); + +// Create a temporary directory for testing +const createTestDir = () => + fs.mkdtempSync(path.join(os.tmpdir(), "dummy-project-")); + +// First test, create empty project dir and add nitro-node as dependency +const firstTest = async (packageManager, repoDir) => { + console.log(`[First test @ ${repoDir}] install with ${packageManager}`); + // Init project with default package.json + const cmd1 = `npm init -y`; + console.log("🖥️ " + cmd1); + await childProcessPromise( + spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); + + // Add nitro-node as dependency + const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${NITRO_NODE_PKG}`; + console.log("🖥️ " + cmd2); + await childProcessPromise( + spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); + + // Check that the binaries exists + if (!checkBinaries(repoDir)) process.exit(3); + + // Cleanup node_modules after success + //fs.rmSync(path.join(repoDir, "node_modules"), { recursive: true }); +}; + +// Second test, install the wrapper from another project dir +const secondTest = async (packageManager, repoDir, wrapperDir) => { + console.log( + `[Second test @ ${repoDir}] install ${wrapperDir} with ${packageManager}`, + ); + // Init project with default package.json + const cmd1 = `npm init -y`; + console.log("🖥️ " + cmd1); + await childProcessPromise( + spawn(cmd1, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); + + // Add wrapper as dependency + const cmd2 = `${packageManager} ${ADD_DEP_CMDS[packageManager]} ${wrapperDir}`; + console.log("🖥️ " + cmd2); + await childProcessPromise( + spawn(cmd2, [], { cwd: repoDir, shell: true, stdio: "inherit" }), + ); + + // Check that the binaries exists + if (!checkBinaries(repoDir)) process.exit(3); +}; + +// Run all the tests for the chosen package manger +const run = async (packageManager) => { + const firstRepoDir = createTestDir(); + + // Run first test + await firstTest(packageManager, firstRepoDir); + console.log("First test ran success"); + + // FIXME: Currently failed with npm due to wrong path being resolved. + //const secondRepoDir = createTestDir(); + + // Run second test + //await secondTest(packageManager, secondRepoDir, firstRepoDir); + //console.log("Second test ran success"); +}; + +// Main, run tests for npm and yarn +const main = async () => { + await PACKAGE_MANAGERS.reduce( + (p, pkgMng) => p.then(() => run(pkgMng)), + Promise.resolve(), + ); +}; + +// Run script if called directly instead of import as module +if (require.main === module) { + main(); +} diff --git a/.github/workflows/build-nitro-node.yml b/.github/workflows/build-nitro-node.yml new file mode 100644 index 000000000..cce5bc310 --- /dev/null +++ b/.github/workflows/build-nitro-node.yml @@ -0,0 +1,355 @@ +name: Build nitro-node + +on: + #schedule: + # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 + push: + branches: + - main + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] + pull_request: + types: [opened, synchronize, reopened] + paths: [".github/workflows/build-nitro-node.yml", "nitro-node"] + workflow_dispatch: + +jobs: + ubuntu-amd64-non-cuda-build: + runs-on: ubuntu-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf + + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download model file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: nitro-node/test/test_assets/ + auto-match: true + retry-times: 3 + + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test-ci + + #ubuntu-amd64-build: + # runs-on: ubuntu-18-04-cuda-11-7 + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download model file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: nitro-node/test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test-ci + + #ubuntu-amd64-cuda-build: + # runs-on: ubuntu-18-04-cuda-${{ matrix.cuda }} + # strategy: + # matrix: + # cuda: ["12-0", "11-7"] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download model file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: nitro-node/test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test-ci + + macOS-M-build: + runs-on: macos-14 + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf + + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download model file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: nitro-node/test/test_assets/ + auto-match: true + retry-times: 3 + + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test-ci + + macOS-Intel-build: + runs-on: macos-latest + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Restore cached model file + id: cache-model-restore + uses: actions/cache/restore@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ runner.os }}-model-gguf + + - uses: suisei-cn/actions-download-file@v1.4.0 + id: download-model-file + name: Download model file + with: + url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + target: nitro-node/test/test_assets/ + auto-match: true + retry-times: 3 + + - name: Save downloaded model file to cache + id: cache-model-save + uses: actions/cache/save@v4 + with: + path: | + nitro-node/test/test_assets/*.gguf + key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + - name: Run tests + id: test_nitro_node + run: | + cd nitro-node + make clean test-ci + + #windows-amd64-build: + # runs-on: windows-latest + # steps: + # - name: Clone + + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: Setup VSWhere.exe + # uses: warrenbuckley/Setup-VSWhere@v1 + # with: + # version: latest + # silent: true + # env: + # ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download model file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: nitro-node/test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test-ci + + #windows-amd64-cuda-build: + # runs-on: windows-cuda-${{ matrix.cuda }} + # strategy: + # matrix: + # cuda: ["12-0", "11-7"] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - uses: actions/setup-node@v4 + # with: + # node-version: 18 + + # - name: actions-setup-cmake + # uses: jwlawson/actions-setup-cmake@v1.14.1 + + # - name: Setup VSWhere.exe + # uses: warrenbuckley/Setup-VSWhere@v1 + # with: + # version: latest + # silent: true + # env: + # ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + # - uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: "6.0.x" + + # - name: Restore cached model file + # id: cache-model-restore + # uses: actions/cache/restore@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ runner.os }}-model-gguf + + # - uses: suisei-cn/actions-download-file@v1.4.0 + # id: download-model-file + # name: Download model file + # with: + # url: "The model we are using is [tinyllama-1.1b](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)!" + # target: nitro-node/test/test_assets/ + # auto-match: true + # retry-times: 3 + + # - name: Save downloaded model file to cache + # id: cache-model-save + # uses: actions/cache/save@v4 + # with: + # path: | + # nitro-node/test/test_assets/*.gguf + # key: ${{ steps.cache-model-restore.outputs.cache-primary-key }} + + # - name: Run tests + # id: test_nitro_node + # run: | + # cd nitro-node + # make clean test-ci diff --git a/.github/workflows/test-install-nitro-node.yml b/.github/workflows/test-install-nitro-node.yml new file mode 100644 index 000000000..de9abb29f --- /dev/null +++ b/.github/workflows/test-install-nitro-node.yml @@ -0,0 +1,163 @@ +name: Test install nitro-node + +on: + #schedule: + # - cron: "0 20 * * *" # At 8 PM UTC, which is 3 AM UTC+7 + push: + branches: + - main + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + paths: + - ".github/scripts/e2e-test-install-nitro-node.js" + - ".github/workflows/test-install-nitro-node.yml" + - "nitro-node/**" + pull_request: + types: [opened, synchronize, reopened] + paths: + - ".github/scripts/e2e-test-install-nitro-node.js" + - ".github/workflows/test-install-nitro-node.yml" + - "nitro-node/**" + workflow_dispatch: + +jobs: + linux-pack-tarball: + runs-on: ubuntu-latest + outputs: + tarball-url: ${{ steps.upload.outputs.artifact-url }} + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Build tarball + id: build + run: | + cd nitro-node + make pack + find . -type f -name 'janhq-nitro-node-*.tgz' -exec mv {} janhq-nitro-node.tgz \; + + - name: Upload tarball as artifact + id: upload + uses: actions/upload-artifact@master + with: + name: janhq-nitro-node + path: nitro-node/janhq-nitro-node.tgz + if-no-files-found: error + + ubuntu-install: + runs-on: ubuntu-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + + - name: Run tests + id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js + + macOS-install: + runs-on: macos-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + + - name: Run tests + id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}/.github/scripts/janhq-nitro-node.tgz + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js + + windows-install: + runs-on: windows-latest + needs: [linux-pack-tarball] + if: always() && needs.linux-pack-tarball.result == 'success' + steps: + - name: Clone + + id: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Download prebuilt tarball + uses: actions/download-artifact@master + with: + name: janhq-nitro-node + path: .github/scripts/ + + - name: List tarball content + id: tar-tf + run: | + cd .github + cd scripts + tar tf janhq-nitro-node.tgz + + - name: Run tests + id: test_install_nitro_node + env: + NITRO_NODE_PKG: ${{ github.workspace }}\.github\scripts\janhq-nitro-node.tgz + run: | + cd .github + cd scripts + node e2e-test-install-nitro-node.js