From a2d5b94e2f9b27329acbb7834c473fa1585615c0 Mon Sep 17 00:00:00 2001 From: Siim Kallas Date: Thu, 11 Nov 2021 15:50:24 +0200 Subject: [PATCH] CI: add prebuilds (#385) * native runtime info collection * metrics sending * add tests * lint * fix: cumulative counters * add deps for node-gyp * actually call apk add * apk sudo * apk add as the correct user * apk add python3 * add binding.gyp to docker image * test: spin with hrtime * test: debug output * test: format eventloop test message * test: copy native module to the correct path * fix: native builds for packaging * test: fix getSignalFxClient * fix: lint * test: separate e2e Dockerfiles * test: run compilation on windows * fix: set metrics disabled by default * doc: add docs * Change files * test: remove event loop collection flakiness * test: add a small timeout to native counters test to ensure a poll step happens between and counters are updated * docs: improve advanced-config wording * docs: mention future changes * refactor: add missing newlines, remove the need for a loop * add prebuild os script * linux prebuild jobs * add build stage * linux: specific prebuild versions * disable scripts in before_script * disable caching * prebuild node >= 14 on the proper image * debug prebuilds dir * tgz as artifact * macos runner * windows build * linux prebuilds in gh actions * fix yaml typo * debug * create-package job * upload the final npm package as an artifact * fix set-output * fix tgz upload path * windows, macos prebuilds * fix prebuild-os without any target args * use node14 on win, macos * fix needs * use c++14 on macos * run unit tests on macos * fix typo * rename linux prebuilds * use node 8.17 for unit tests * don't run on node 8 * remove codecov from tests setup * artifact fetch attempt 1 * fetch-release-artifact script * fix fetch-release-artifact invocation * install deps before ghfetch * debug artifact * wait for artifacts * increase timeout for waiting * use public artifacts token * prepare release dir * add artifact prepare debug message * add run conclusion debug message on failure * remove unnecessary Dockerfile * add node 17 to prebuild matrix * attempt node 17 * attempt 17 on node 16 * node 17 * upgrade prebuildify, attempt node 17 builds * only build 17 on node 14 * run unit tests for node 17 * remove test-release stage from gitlab * strip release binaries * refactor * remove release-test --- .github/workflows/ci.yml | 94 ++++++++++++++++++++- .gitlab-ci.yml | 41 +--------- binding.gyp | 2 +- package-lock.json | 54 +++++------- package.json | 6 +- scripts/prebuild-os.js | 17 ++++ scripts/prepare-release-artifact.js | 123 ++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+), 81 deletions(-) create mode 100644 scripts/prebuild-os.js create mode 100644 scripts/prepare-release-artifact.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fd43af7..70a5381c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,14 +7,102 @@ on: permissions: read-all jobs: + prebuilds-linux: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - container: node:10.13.0 + versions: '8.5.0 9.0.0 10.0.0 11.0.0 12.0.0 13.0.0' + label: node10 + - container: node:14.0.0 + versions: '14.0.0 15.0.0 16.0.0 17.0.1' + label: node14 + container: ${{ matrix.container }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + - name: Install npm dependencies + run: npm ci + - name: Prebuild + run: npm run prebuild:os ${{ matrix.versions }} + - name: Run tests + run: npm run test + - name: upload prebuilds + uses: actions/upload-artifact@v2 + with: + name: prebuilds-linux-${{ matrix.label }} + path: prebuilds + + prebuilds: + strategy: + fail-fast: false + matrix: + include: + - os: windows-2019 + label: windows + - os: macos-10.15 + label: macos + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '14' + - name: Install npm dependencies + run: npm ci + - name: Prebuild + run: npm run prebuild:os + - name: Run tests + run: npm run test + - name: upload prebuilds + uses: actions/upload-artifact@v2 + with: + name: prebuilds-${{ matrix.label }} + path: prebuilds + + create-package: + needs: [prebuilds-linux, prebuilds] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + - name: download prebuilds + uses: actions/download-artifact@v2 + - name: copy prebuilds + run: | + mkdir -p prebuilds + cp -r prebuilds-linux-node10/* prebuilds + cp -r prebuilds-linux-node14/* prebuilds + cp -r prebuilds-windows/* prebuilds + cp -r prebuilds-macos/* prebuilds + - name: Install npm dependencies + run: npm ci + - name: Build + run: npm run compile + - name: Pack + id: pack + run: | + echo "::set-output name=package_file::$(npm pack)" + - name: Upload package + uses: actions/upload-artifact@v2 + with: + name: ${{ steps.pack.outputs.package_file }} + path: ${{ steps.pack.outputs.package_file }} + unit-tests: + needs: [prebuilds-linux, prebuilds] runs-on: ${{ matrix.os }} permissions: read-all strategy: fail-fast: false matrix: - os: ['ubuntu-latest', 'windows-latest'] - nodejs: ['10', '12', '14', '16'] + os: ['ubuntu-latest', 'windows-latest', 'macos-latest'] + nodejs: ['10', '12', '14', '16', '17'] steps: - name: Checkout uses: actions/checkout@v2 @@ -25,8 +113,6 @@ jobs: run: npm ci - name: Test run: npm run test - - name: Report coverage - run: npm run codecov lint: if: ${{ github.event_name == 'pull_request' }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4c1802f..10ab2bdb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,45 +1,9 @@ default: image: 'cimg/node:lts' -cache: - key: - files: - - package-lock.json - paths: - - .npm/ - - node_modules/ - -# tell npm to store cache in .npm so it can be cached by Gitlab -before_script: - - npm ci --cache .npm --prefer-offline - stages: - - setup - - test - release -install: - stage: setup - script: - - npm install - -test: - script: - - npm test - -lint: - script: - - npm run lint - - npm run version:check - -build: - artifacts: - paths: - - dist/ - script: - - npm run compile - - npm pack - release: stage: release artifacts: @@ -51,10 +15,7 @@ release: - branches script: - npm install - - npm run compile - - npm pack - - mkdir -p dist - - mv splunk-otel-${CI_COMMIT_REF_NAME:1}.tgz dist/ + - npm run prepare-release-artifact - shasum -a 256 dist/* > dist/checksums.txt # release in Github diff --git a/binding.gyp b/binding.gyp index 492c69b3..0e8e893c 100644 --- a/binding.gyp +++ b/binding.gyp @@ -27,7 +27,7 @@ "xcode_settings": { "MACOSX_DEPLOYMENT_TARGET": "10.10", "OTHER_CFLAGS": [ - "-std=c++11", + "-std=c++14", "-stdlib=libc++", "-Wall", "-Werror" diff --git a/package-lock.json b/package-lock.json index 19334d2e..23aaf9d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,7 @@ "nyc": "15.1.0", "octokit": "^1.7.0", "pino": "^6.13.2", - "prebuildify": "4.2.1", + "prebuildify": "5.0.0", "prettier": ">=2.3.2", "rewire": "^5.0.0", "rimraf": "3.0.2", @@ -7941,21 +7941,15 @@ } }, "node_modules/node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.3.0.tgz", + "integrity": "sha512-/+2sCVPXmj07GY/l0ggRr7+trqzX7F9d4QVfSArqIVYmRzc/LkXKr5FlRO6U8uZ/gVVclDDaBxBNITj1z1Z/Zw==", "dev": true, "dependencies": { - "semver": "^5.4.1" - } - }, - "node_modules/node-abi/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true, - "bin": { - "semver": "bin/semver" + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" } }, "node_modules/node-fetch": { @@ -8600,15 +8594,15 @@ } }, "node_modules/prebuildify": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-4.2.1.tgz", - "integrity": "sha512-FFgf3jHbh404ZuM++Cr0nMhK/VIgpyzscEXXiZCX1gbQz1ktg0s4hFKr9nXQKDLA3De98BvqNZODfqvm0maA2w==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-5.0.0.tgz", + "integrity": "sha512-XhuFIeZx8Tk8e8yn3h5e+CE572pecpdKPrVubUIW0HctP3fpzh4eSWoHR1eOoQNTtxBUt1ixPLHPLbOTYi6STw==", "dev": true, "dependencies": { "execspawn": "^1.0.1", "minimist": "^1.2.5", "mkdirp-classic": "^0.5.3", - "node-abi": "^2.19.1", + "node-abi": "^3.3.0", "npm-run-path": "^3.1.0", "pump": "^3.0.0", "tar-fs": "^2.1.0" @@ -17605,20 +17599,12 @@ } }, "node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.3.0.tgz", + "integrity": "sha512-/+2sCVPXmj07GY/l0ggRr7+trqzX7F9d4QVfSArqIVYmRzc/LkXKr5FlRO6U8uZ/gVVclDDaBxBNITj1z1Z/Zw==", "dev": true, "requires": { - "semver": "^5.4.1" - }, - "dependencies": { - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - } + "semver": "^7.3.5" } }, "node-fetch": { @@ -18128,15 +18114,15 @@ "dev": true }, "prebuildify": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-4.2.1.tgz", - "integrity": "sha512-FFgf3jHbh404ZuM++Cr0nMhK/VIgpyzscEXXiZCX1gbQz1ktg0s4hFKr9nXQKDLA3De98BvqNZODfqvm0maA2w==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-5.0.0.tgz", + "integrity": "sha512-XhuFIeZx8Tk8e8yn3h5e+CE572pecpdKPrVubUIW0HctP3fpzh4eSWoHR1eOoQNTtxBUt1ixPLHPLbOTYi6STw==", "dev": true, "requires": { "execspawn": "^1.0.1", "minimist": "^1.2.5", "mkdirp-classic": "^0.5.3", - "node-abi": "^2.19.1", + "node-abi": "^3.3.0", "npm-run-path": "^3.1.0", "pump": "^3.0.0", "tar-fs": "^2.1.0" diff --git a/package.json b/package.json index 39b9e735..7eb6a170 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,14 @@ "clean": "tsc --build --clean", "codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p .", "compile": "tsc --build", + "prepare-release-artifact": "node scripts/prepare-release-artifact.js", "install": "node-gyp-build", "lint:fix": "eslint . --ext .ts --fix", "lint": "eslint . --ext .ts", "lint:commits": "commitlint", "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'", "prebuild:current": "node scripts/prebuild-current.js", + "prebuild:os": "node scripts/prebuild-os.js", "release:github": "node scripts/release-github.js", "version:check": "node scripts/version-check.js", "version:generate": "node scripts/version-generate.js", @@ -37,7 +39,7 @@ "stats" ], "engines": { - "node": ">=8.5.0 <17" + "node": ">=8.5.0 <18" }, "files": [ "binding.gyp", @@ -94,7 +96,7 @@ "ts-node": "10.4.0", "typescript": "4.4.4", "winston": "3.3.3", - "prebuildify": "4.2.1" + "prebuildify": "5.0.0" }, "dependencies": { "@opentelemetry/api": "^1.0.3", diff --git a/scripts/prebuild-os.js b/scripts/prebuild-os.js new file mode 100644 index 00000000..d7eca249 --- /dev/null +++ b/scripts/prebuild-os.js @@ -0,0 +1,17 @@ +const prebuildify = require('prebuildify'); + +let targets = process.argv.slice(2); + +if (targets.length == 0) { + targets = ['8.5.0', '9.0.0', '10.0.0', '11.0.0', '12.0.0', '13.0.0', '14.0.0', '15.0.0', '16.0.0', '17.0.1']; +} + +prebuildify({ + strip: true, + targets: targets, +}, err => { + if (err) { + console.error(err.message || err); + process.exit(1); + } +}); diff --git a/scripts/prepare-release-artifact.js b/scripts/prepare-release-artifact.js new file mode 100644 index 00000000..31bd4004 --- /dev/null +++ b/scripts/prepare-release-artifact.js @@ -0,0 +1,123 @@ +const fs = require('fs'); +const { execSync } = require('child_process'); +const path = require('path'); +const { Octokit } = require('octokit'); + +const { version } = require('../package.json'); + +const WORKFLOW_TIMEOUT_MS = 15 * 60 * 1000; + +function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +async function fetchWorkflowRun(context) { + const { data: workflows } = await context.octokit.rest.actions.listWorkflowRunsForRepo({ + owner: context.owner, + repo: context.repo, + branch: process.env.CI_COMMIT_BRANCH ?? 'main', + }); + + const runs = workflows.workflow_runs; + + const commitSha = process.env.CI_COMMIT_SHA; + const run = runs.find(wf => wf.head_sha === commitSha); + + if (run === undefined) { + throw new Error(`Workflow not found for commit ${commitSha}`); + } + + return run; +} + +async function waitForWorkflowRun(context) { + const waitUntil = Date.now() + WORKFLOW_TIMEOUT_MS; + + for (;;) { + let run = await fetchWorkflowRun(context); + + if (run.status !== 'completed') { + if (Date.now() > waitUntil) { + throw new Error('Timed out waiting for workflow to finish'); + } + + console.log('run not yet completed, waiting...'); + await sleep(10_000); + + continue; + } + + if (run.status === 'completed' && run.conclusion !== 'success') { + throw new Error(`Workflow not successful conclusion=${run.conclusion}`); + } + + return run; + } +} + +async function getBuildArtifact() { + const octokit = new Octokit({ auth: process.env.PUBLIC_ARTIFACTS_TOKEN }); + const owner = process.env.GITHUB_OWNER ?? 'signalfx'; + const repo = process.env.GITHUB_REPO ?? 'splunk-otel-js'; + + console.log('waiting for workflow results'); + + const run = await waitForWorkflowRun({ octokit, owner, repo }); + + console.log('found finished workflow run', run); + + const { data: artifacts } = await octokit.rest.actions.listWorkflowRunArtifacts({ + owner, + repo, + run_id: run.id, + }); + + console.log('found artifacts for workflow', artifacts); + + const tgzName = `splunk-otel-${version}.tgz` + const packageArtifact = artifacts.artifacts.find(artifact => artifact.name === tgzName); + + if (packageArtifact === undefined) { + throw new Error(`unable to find artifact named ${tgzName}`); + } + + const dlArtifact = await octokit.rest.actions.downloadArtifact({ + owner, + repo, + artifact_id: packageArtifact.id, + archive_format: 'zip', + }); + + console.log('downloaded', dlArtifact); + + const tempFile = 'artifact-temp.zip'; + + console.log(`writing content to ${tempFile} and unzipping`); + + fs.writeFileSync(tempFile, Buffer.from(dlArtifact.data)); + + execSync(`unzip -o ${tempFile}`); + + const exists = fs.existsSync(packageArtifact.name); + console.log(`${packageArtifact.name} was extracted: ${exists}`); + + if (!exists) { + throw new Error(`${packageArtifact.name} was not found after extraction`); + } + + return packageArtifact.name; +} + +async function prepareReleaseArtifact() { + const artifactName = await getBuildArtifact(); + + fs.mkdirSync('dist', { recursive: true }); + fs.renameSync(artifactName, path.join('dist', artifactName)); + + console.log('successfully prepared artifacts'); +} + +prepareReleaseArtifact().catch(e => { + console.error(e); + process.exit(1); +});