From ea1f4a6267a4e5232148d2d8288d755393884880 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 20 Jun 2024 11:23:39 -0400 Subject: [PATCH] chore: add a fastDownload flag to build script (#26) --- .evergreen/check-lint.sh | 29 -------- .evergreen/init-node-and-npm-env.sh | 21 ------ .evergreen/install-dependencies.sh | 111 ---------------------------- .evergreen/prebuild.sh | 89 ---------------------- .evergreen/test.sh | 21 ------ .github/scripts/libmongocrypt.mjs | 84 ++++++++++++++++----- README.md | 25 ++++++- 7 files changed, 87 insertions(+), 293 deletions(-) delete mode 100644 .evergreen/check-lint.sh delete mode 100644 .evergreen/init-node-and-npm-env.sh delete mode 100644 .evergreen/install-dependencies.sh delete mode 100755 .evergreen/prebuild.sh delete mode 100755 .evergreen/test.sh diff --git a/.evergreen/check-lint.sh b/.evergreen/check-lint.sh deleted file mode 100644 index c78acc1..0000000 --- a/.evergreen/check-lint.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${LINT_TARGET+omitted} ]; then echo "LINT_TARGET is unset, must have value typescript or cpp" && exit 1; fi - -# set -o xtrace # Write all commands first to stderr -set -o errexit # Exit the script with error if any of the commands fail - -echo "Setting up environment" - -export PATH="/opt/mongodbtoolchain/v2/bin:$PATH" -hash -r - -NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} -export NODE_LTS_VERSION=${NODE_LTS_VERSION} -source ./.evergreen/install-dependencies.sh - -# install dependencies but intentionally do not -# run prebuild or Typescript, since we are only -# linting. -npm install --ignore-scripts - -if [ "$LINT_TARGET" == "typescript" ]; then - npm run check:eslint -elif [ "$LINT_TARGET" == "cpp" ]; then - npm run check:clang-format -else - echo "unsupported value for LINT_TARGET: $LINT_TARGET" - exit 1 -fi diff --git a/.evergreen/init-node-and-npm-env.sh b/.evergreen/init-node-and-npm-env.sh deleted file mode 100644 index b3cecf5..0000000 --- a/.evergreen/init-node-and-npm-env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env bash -## -## This script add the location of `npm` and `node` to the path. -## This is necessary because evergreen uses separate bash scripts for -## different functions in a given CI run but doesn't persist the environment -## across them. So we manually invoke this script everywhere we need -## access to `npm`, `node`, or need to install something globally from -## npm. - -NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts" -if [[ "$OS" == "Windows_NT" ]]; then - NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH") -fi - -export NODE_ARTIFACTS_PATH -# npm uses this environment variable to determine where to install global packages -export npm_global_prefix=$NODE_ARTIFACTS_PATH/npm_global -export PATH="$npm_global_prefix/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH" -hash -r - -export NODE_OPTIONS="--trace-deprecation --trace-warnings" diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh deleted file mode 100644 index 1153d12..0000000 --- a/.evergreen/install-dependencies.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -set -o errexit # Exit the script with error if any of the commands fail - -NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} -# npm version can be defined in the environment for cases where we need to install -# a version lower than latest to support EOL Node versions. -NPM_VERSION=${NPM_VERSION:-latest} - -source "${PROJECT_DIRECTORY}/libmongocrypt/bindings/node/.evergreen/init-node-and-npm-env.sh" - -if [[ -z "${npm_global_prefix}" ]]; then echo "npm_global_prefix is unset" && exit 1; fi -if [[ -z "${NODE_ARTIFACTS_PATH}" ]]; then echo "NODE_ARTIFACTS_PATH is unset" && exit 1; fi - -CURL_FLAGS=( - --fail # Exit code 1 if request fails - --compressed # Request a compressed response should keep fetching fast - --location # Follow a redirect - --retry 8 # Retry HTTP 408, 429, 500, 502, 503 or 504, 8 times - --silent # Do not print a progress bar - --show-error # Despite the silent flag still print out errors - --max-time 900 # 900 seconds is 15 minutes, evergreen times out at 20 - --continue-at - # If a download is interrupted it can figure out where to resume -) - -mkdir -p "$NODE_ARTIFACTS_PATH/npm_global" - -# Comparisons are all case insensitive -shopt -s nocasematch - -# index.tab is a sorted tab separated values file with the following headers -# 0 1 2 3 4 5 6 7 8 9 10 -# version date files npm v8 uv zlib openssl modules lts security -curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab - -while IFS=$'\t' read -r -a row; do - node_index_version="${row[0]}" - node_index_major_version=$(echo $node_index_version | sed -E 's/^v([0-9]+).*$/\1/') - node_index_date="${row[1]}" - node_index_lts="${row[9]}" - [[ "$node_index_version" = "version" ]] && continue # skip tsv header - [[ "$NODE_LTS_VERSION" = "latest" ]] && break # first line is latest - [[ "$NODE_LTS_VERSION" = "$node_index_major_version" ]] && break # case insensitive compare -done < node_index.tab - -if [[ "$OS" = "Windows_NT" ]]; then - operating_system="win" -elif [[ $(uname) = "darwin" ]]; then - operating_system="darwin" -elif [[ $(uname) = "linux" ]]; then - operating_system="linux" -else - echo "Unable to determine operating system: $operating_system" - exit 1 -fi - -architecture=$(uname -m) -if [[ $architecture = "x86_64" ]]; then - architecture="x64" -elif [[ $architecture = "arm64" ]]; then - architecture="arm64" -elif [[ $architecture = "aarch64" ]]; then - architecture="arm64" -elif [[ $architecture == s390* ]]; then - architecture="s390x" -elif [[ $architecture == ppc* ]]; then - architecture="ppc64le" -else - echo "Unable to determine operating system: $architecture" - exit 1 -fi - -file_extension="tar.gz" -if [[ "$OS" = "Windows_NT" ]]; then file_extension="zip"; fi - -node_directory="node-${node_index_version}-${operating_system}-${architecture}" -node_archive="${node_directory}.${file_extension}" -node_archive_path="$NODE_ARTIFACTS_PATH/${node_archive}" -node_download_url="https://nodejs.org/dist/${node_index_version}/${node_archive}" - -echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}" - -set -o xtrace - -curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" - -if [[ "$file_extension" = "zip" ]]; then - unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}" - mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs" - # Windows "bins" are at the top level - mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin" - # Need to add executable flag ourselves - chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe" - chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm" -else - tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}" - mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs" -fi - -if [[ $operating_system != "win" ]]; then - npm install --global npm@$NPM_VERSION - hash -r -fi - -echo "npm location: $(which npm)" -echo "npm version: $(npm -v)" - -# other repos that use this script run npm install after installing Node. -# we can't in mongodb-client-encryption, because when releasing -# npm install will attempt to build from source, which fails -# because we haven't built libmongocrypt yet. -# npm install "${NPM_OPTIONS}" diff --git a/.evergreen/prebuild.sh b/.evergreen/prebuild.sh deleted file mode 100755 index 1ded5ae..0000000 --- a/.evergreen/prebuild.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${DISTRO_ID+omitted} ]; then echo "DISTRO_ID is unset" && exit 1; fi - -set -o errexit -set +o xtrace - -echo "Setting up environment" - -export PATH="/opt/mongodbtoolchain/v2/bin:$PATH" -hash -r - -NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} -export NODE_LTS_VERSION=${NODE_LTS_VERSION} -source ./.evergreen/install-dependencies.sh - -# install node dependencies -echo "Installing package dependencies (includes a static build)" -bash ./etc/build-static.sh - -# FLE platform matrix (as of 22 May 2023) -# macos arm64 (compiled on 11.00) -# macos x86_64 (compiled on 10.14) -# windows x86_64 (compiled on vs2019) -# linux x86_64 (compiled on Ubuntu 16.04) -# linux arm64 (compiled on Ubuntu 16.04) - -# Determines the OS name through uname results -# Returns 'windows' 'linux' 'macos' or 'unknown' -os_name() { - local WINDOWS_REGEX="cygwin|windows|mingw|msys" - local UNAME - UNAME=$(uname | tr '[:upper:]' '[:lower:]') - - local OS_NAME="unknown" - - if [[ $UNAME =~ $WINDOWS_REGEX ]]; then - OS_NAME="windows" - elif [[ $UNAME == "darwin" ]]; then - OS_NAME="macos" - elif [[ $UNAME == "linux" ]]; then - OS_NAME="linux" - fi - - echo $OS_NAME -} - -OS=$(os_name) - -get_version_at_git_rev () { - local REV=$1 - local VERSION - VERSION=$(node -r child_process -e "console.log(JSON.parse(child_process.execSync('git show $REV:./package.json', { encoding: 'utf8' })).version);") - echo "$VERSION" -} - -run_prebuild() { - if [ -z ${NODE_GITHUB_TOKEN+omitted} ]; then echo "NODE_GITHUB_TOKEN is unset" && exit 1; fi - echo "Github token detected. Running prebuild." - npm run prebuild -- -u "$NODE_GITHUB_TOKEN" - echo "Prebuild's successfully submitted" -} - -VERSION_AT_HEAD=$(get_version_at_git_rev "HEAD") -VERSION_AT_HEAD_1=$(get_version_at_git_rev "HEAD~1") - -if [[ "$OS" == "macos" ]]; then - ARCH=$(uname -m) - if [[ "$ARCH" == "arm64" ]]; then - # TODO(NODE-5174): node-gyp fails to run prebuild if Python 3.11+ - echo "Exporting PYTHON location for version $(/opt/homebrew/opt/python@3.9/bin/python3.9 --version)" - export PYTHON="/opt/homebrew/opt/python@3.9/bin/python3.9" - fi -fi - -if [[ -n $NODE_FORCE_PUBLISH ]]; then - echo "\$NODE_FORCE_PUBLISH=${NODE_FORCE_PUBLISH} detected" - echo "Beginning prebuild" - run_prebuild -elif [[ "$VERSION_AT_HEAD" != "$VERSION_AT_HEAD_1" ]]; then - echo "Difference is package version ($VERSION_AT_HEAD_1 -> $VERSION_AT_HEAD)" - echo "Beginning prebuild" - run_prebuild -else - echo "No difference is package version ($VERSION_AT_HEAD_1 -> $VERSION_AT_HEAD)" - echo "Will prebuild without submit ($OS - $DISTRO_ID)" - npm run prebuild - echo "Local prebuild successful." -fi diff --git a/.evergreen/test.sh b/.evergreen/test.sh deleted file mode 100755 index 47fb4bb..0000000 --- a/.evergreen/test.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# set -o xtrace # Write all commands first to stderr -set -o errexit # Exit the script with error if any of the commands fail - -echo "Setting up environment" - -export PATH="/opt/mongodbtoolchain/v2/bin:$PATH" -hash -r - -NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} -export NODE_LTS_VERSION=${NODE_LTS_VERSION} -source ./.evergreen/install-dependencies.sh - -# install node dependencies -echo "Installing package dependencies (includes a static build)" -bash ./etc/build-static.sh - -# Run tests -echo "Running tests" -npm test diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 54284cb..df963cc 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -1,3 +1,4 @@ +//@ts-check import util from 'node:util'; import process from 'node:process'; import fs from 'node:fs/promises'; @@ -23,6 +24,7 @@ async function parseArguments() { libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] }, clean: { short: 'c', type: 'boolean', default: false }, build: { short: 'b', type: 'boolean', default: false }, + fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient help: { short: 'h', type: 'boolean', default: false } }; @@ -39,7 +41,9 @@ async function parseArguments() { } return { - libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion }, + url: args.values.gitURL, + ref: args.values.libVersion, + fastDownload: args.values.fastDownload, clean: args.values.clean, build: args.values.build, pkg @@ -136,7 +140,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { }); } -export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) { const downloadURL = ref === 'latest' ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' @@ -164,61 +168,95 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`); - const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`]; + const downloadDestination = `${prebuild}/nocrypto`; + const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination]; console.error(`+ tar ${unzipArgs.join(' ')}`); const unzip = child_process.spawn('tar', unzipArgs, { - stdio: ['pipe', 'inherit'], + stdio: ['pipe', 'inherit', 'pipe'], cwd: resolveRoot('.') }); + if (unzip.stdin == null) throw new Error('Tar process must have piped stdin'); const [response] = await events.once(https.get(downloadURL), 'response'); const start = performance.now(); - await stream.pipeline(response, unzip.stdin); + + let signal; + if (fastDownload) { + /** + * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto) + * For each file it prints, we give it a deadline of 3 seconds to print the next one. + * If nothing prints after 3 seconds we exit early. + * This depends on the tar file being in order and un-tar-able in under 3sec. + */ + const controller = new AbortController(); + signal = controller.signal; + let firstMemberSeen = true; + let timeout; + unzip.stderr.on('data', chunk => { + process.stderr.write(chunk, () => { + if (firstMemberSeen) { + firstMemberSeen = false; + timeout = setTimeout(() => { + clearTimeout(timeout); + unzip.stderr.removeAllListeners('data'); + controller.abort(); + }, 3_000); + } + timeout?.refresh(); + }); + }); + } + + try { + await stream.pipeline(response, unzip.stdin, { signal }); + } catch { + await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination)); + } + const end = performance.now(); console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`); await fs.rm(nodeDepsRoot, { recursive: true, force: true }); await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true }); - const currentPath = path.join(nodeDepsRoot, 'lib64'); + const potentialLib64Path = path.join(nodeDepsRoot, 'lib64'); try { - await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib')); + await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib')); } catch (error) { - console.error(`error renaming ${currentPath}: ${error.message}`); + await fs.access(path.join(nodeDepsRoot, 'lib')); // Ensure there is a "lib" directory } } async function main() { - const { libmongocrypt, build, clean, pkg } = await parseArguments(); + const { pkg, ...args } = await parseArguments(); + console.log(args); const nodeDepsDir = resolveRoot('deps'); - if (build) { + if (args.build) { const libmongocryptCloneDir = resolveRoot('_libmongocrypt'); const currentLibMongoCryptBranch = await fs .readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8') .catch(() => ''); - const isClonedAndCheckedOut = currentLibMongoCryptBranch - .trim() - .endsWith(`r-${libmongocrypt.ref}`); + const isClonedAndCheckedOut = currentLibMongoCryptBranch.trim().endsWith(`r-${args.ref}`); - if (clean || !isClonedAndCheckedOut) { - await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt); + if (args.clean || !isClonedAndCheckedOut) { + await cloneLibMongoCrypt(libmongocryptCloneDir, args); } const libmongocryptBuiltVersion = await fs .readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8') .catch(() => ''); - const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; + const isBuilt = libmongocryptBuiltVersion.trim() === args.ref; - if (clean || !isBuilt) { + if (args.clean || !isBuilt) { await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir); } } else { // Download - await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt); + await downloadLibMongoCrypt(nodeDepsDir, args); } await fs.rm(resolveRoot('build'), { force: true, recursive: true }); @@ -235,8 +273,14 @@ async function main() { if (process.platform === 'darwin') { // The "arm64" build is actually a universal binary await fs.copyFile( - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-arm64.tar.gz`), - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-x64.tar.gz`) + resolveRoot( + 'prebuilds', + `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-arm64.tar.gz` + ), + resolveRoot( + 'prebuilds', + `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-x64.tar.gz` + ) ); } } diff --git a/README.md b/README.md index 50f51ca..b7cd896 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,31 @@ npm install mongodb-client-encryption #### Setup -Run the following command to build libmongocrypt and setup the node bindings for development: + +Run the following command to build libmongocrypt and you are setup to develop the node bindings: ```shell -bash ./etc/build-static.sh +npm run install:libmongocrypt +``` + +#### `libmongocrypt.mjs` + +``` +node libmongocrypt.mjs [--gitURL=string] [--libVersion=string] [--clean] [--build] [--no-crypto] [--fastDownload] + +By default attempts to download and compile the bindings with the crypto prebuilds of libmongocrypt. +Can be configured to clone and build without crypto. + +--gitURL=string A custom remote git repository to clone libmongocrypt from. You must also set --build to use this. +--libVersion=string A custom version reference to either download or checkout after cloning. + You may use "latest" to get current libmongocrypt `HEAD`. +--clean Combined with --build, the script will not skip cloning and rebuilding libmongocrypt. +--build Instead of downloading, clone and build libmongocrypt along with the bindings. + +Only suitable for local development: + +--fastDownload If you are improving this script or otherwise repeatedly downloading libmongocrypt, + this flag will interrupt the un-tar operation as early as possible. It should work, most of the time. ``` #### Prebuild Platforms