diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 7299ac1..e402c46 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -1,3 +1,4 @@ +// @ts-check import util from 'node:util'; import process from 'node:process'; import fs from 'node:fs/promises'; @@ -23,6 +24,8 @@ async function parseArguments() { libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] }, clean: { short: 'c', type: 'boolean', default: false }, build: { short: 'b', type: 'boolean', default: false }, + 'no-crypto': { type: 'boolean', default: false }, // Use Node.js builtin crypto + fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient help: { short: 'h', type: 'boolean', default: false } }; @@ -39,7 +42,10 @@ async function parseArguments() { } return { - libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion }, + url: args.values.gitURL, + ref: args.values.libVersion, + crypto: !args.values['no-crypto'], + fastDownload: args.values.fastDownload, clean: args.values.clean, build: args.values.build, pkg @@ -77,7 +83,7 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) { } } -export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { +export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) { console.error('building libmongocrypt...'); const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build'); @@ -88,7 +94,6 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { const CMAKE_FLAGS = toFlags({ /** * We provide crypto hooks from Node.js binding to openssl (so disable system crypto) - * TODO: NODE-5455 * * One thing that is not obvious from the build instructions for libmongocrypt * and the Node.js bindings is that the Node.js driver uses libmongocrypt in @@ -101,7 +106,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { * have a copy of OpenSSL available directly, but for now it seems to make sense * to stick with what the Node.js addon does here. */ - DDISABLE_NATIVE_CRYPTO: '1', + DDISABLE_NATIVE_CRYPTO: options.crypto ? '0' : '1', /** A consistent name for the output "library" directory */ DCMAKE_INSTALL_LIBDIR: 'lib', /** No warnings allowed */ @@ -131,12 +136,13 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], { cwd: nodeBuildRoot } ); + await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { cwd: nodeBuildRoot }); } -export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto, fastDownload }) { const downloadURL = ref === 'latest' ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' @@ -164,37 +170,72 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`); - const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`]; + const downloadDestination = crypto ? `${prebuild}` : `${prebuild}/nocrypto`; + const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination]; console.error(`+ tar ${unzipArgs.join(' ')}`); const unzip = child_process.spawn('tar', unzipArgs, { - stdio: ['pipe', 'inherit'], + stdio: ['pipe', 'inherit', 'pipe'], cwd: resolveRoot('.') }); const [response] = await events.once(https.get(downloadURL), 'response'); const start = performance.now(); - await stream.pipeline(response, unzip.stdin); + + let signal; + if (fastDownload) { + /** + * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto) + * For each file it prints, we give it a deadline of 10seconds to print the next one. + * If nothing prints after 10 seconds we exit early. + * This depends on the tar file being in order and un-tar-able in under 10sec. + * + * download time went from 230s to 80s + */ + const controller = new AbortController(); + signal = controller.signal; + let isFirstStdoutDataEv = true; + let timeout; + unzip.stderr.on('data', chunk => { + process.stderr.write(chunk, () => { + if (isFirstStdoutDataEv) { + isFirstStdoutDataEv = false; + timeout = setTimeout(() => controller.abort(), 5_000); + } + timeout?.refresh(); + }); + }); + } + + try { + await stream.pipeline(response, unzip.stdin, { signal }); + } catch { + await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination)); + } const end = performance.now(); console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`); await fs.rm(nodeDepsRoot, { recursive: true, force: true }); - await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true }); - const currentPath = path.join(nodeDepsRoot, 'lib64'); + const source = crypto + ? resolveRoot(destination, prebuild) + : resolveRoot(destination, prebuild, 'nocrypto'); + await fs.cp(source, nodeDepsRoot, { recursive: true }); + const potentialLib64Path = path.join(nodeDepsRoot, 'lib64'); try { - await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib')); + await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib')); } catch (error) { - console.error(`error renaming ${currentPath}: ${error.message}`); + await fs.access(path.join(nodeDepsRoot, 'lib')); // Ensure there is a "lib" directory } } async function main() { - const { libmongocrypt, build, clean, pkg } = await parseArguments(); + const args = await parseArguments(); + console.log(args); const nodeDepsDir = resolveRoot('deps'); - if (build) { + if (args.build) { const libmongocryptCloneDir = resolveRoot('_libmongocrypt'); const currentLibMongoCryptBranch = await fs @@ -202,23 +243,23 @@ async function main() { .catch(() => ''); const isClonedAndCheckedOut = currentLibMongoCryptBranch .trim() - .endsWith(`r-${libmongocrypt.ref}`); + .endsWith(`r-${args.ref}`); - if (clean || !isClonedAndCheckedOut) { - await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt); + if (args.clean || !isClonedAndCheckedOut) { + await cloneLibMongoCrypt(libmongocryptCloneDir, args); } const libmongocryptBuiltVersion = await fs .readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8') .catch(() => ''); - const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; + const isBuilt = libmongocryptBuiltVersion.trim() === args.ref; - if (clean || !isBuilt) { - await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir); + if (args.clean || !isBuilt) { + await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, args); } } else { // Download - await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt); + await downloadLibMongoCrypt(nodeDepsDir, args); } await fs.rm(resolveRoot('build'), { force: true, recursive: true }); @@ -233,8 +274,8 @@ async function main() { if (process.platform === 'darwin') { // The "arm64" build is actually a universal binary await fs.copyFile( - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-arm64.tar.gz`), - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-x64.tar.gz`) + resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-arm64.tar.gz`), + resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-x64.tar.gz`) ); } } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4c3421..d6e73b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - name: Build ${{ matrix.os }} Prebuild - run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }} + run: npm run install:libmongocrypt ${{ runner.os == 'Windows' && '-- --build' || '' }} shell: bash - name: Test ${{ matrix.os }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e1f501a..c44d69d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,7 +46,7 @@ jobs: - if: matrix.build-mode == 'manual' shell: bash - run: node .github/scripts/libmongocrypt.mjs + run: npm run install:libmongocrypt - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 diff --git a/README.md b/README.md index 50f51ca..2f87705 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,32 @@ npm install mongodb-client-encryption #### Setup -Run the following command to build libmongocrypt and setup the node bindings for development: +Run the following command to build libmongocrypt and you are setup to develop the node bindings: ```shell -bash ./etc/build-static.sh +npm run install:libmongocrypt +``` + +#### `libmongocrypt.mjs` + +``` +node libmongocrypt.mjs [--gitURL=string] [--libVersion=string] [--clean] [--build] [--no-crypto] [--fastDownload] + +By default attempts to download and compile the bindings with the crypto prebuilds of libmongocrypt. +Can be configured to clone and build without crypto. + +--gitURL=string A custom remote git repository to clone libmongocrypt from. You must also set --build to use this. +--libVersion=string A custom version reference to either download or checkout after cloning. + You may use "latest" to get current libmongocrypt `HEAD`. +--clean Combined with --build, the script will not skip cloning and rebuilding libmongocrypt. +--build Instead of downloading, clone and build libmongocrypt along with the bindings. +--no-crypto Use libmongocrypt prebuild or build libmongocrypt without OpenSSL symbols. + Requires the bindings to provide cryptoCallbacks. + +Only suitable for local development: + +--fastDownload If you are improving this script or otherwise repeatedly downloading libmongocrypt, + this flag will interrupt the un-tar operation as early as possible. It should work, most of the time. ``` #### Prebuild Platforms diff --git a/addon/mongocrypt.cc b/addon/mongocrypt.cc index 70046ca..d903b2d 100644 --- a/addon/mongocrypt.cc +++ b/addon/mongocrypt.cc @@ -472,7 +472,7 @@ MongoCrypt::MongoCrypt(const CallbackInfo& info) } } - if (options.Has("cryptoCallbacks")) { + if (!mongocrypt_is_crypto_available() && options.Has("cryptoCallbacks")) { Object cryptoCallbacks = options.Get("cryptoCallbacks").ToObject(); SetCallback("aes256CbcEncryptHook", cryptoCallbacks["aes256CbcEncryptHook"]); diff --git a/test/benchmarks/bench.mjs b/test/benchmarks/bench.mjs index 7062a5d..7e87ffb 100644 --- a/test/benchmarks/bench.mjs +++ b/test/benchmarks/bench.mjs @@ -15,8 +15,7 @@ const ERROR = 0; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); -const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '', BENCH_WITH_NATIVE_CRYPTO = '' } = - process.env; +const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '' } = process.env; const warmupSecs = 2; const testInSecs = 57; @@ -121,8 +120,7 @@ function main() { `testInSecs=${testInSecs}` ); - const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders) }; - if (!BENCH_WITH_NATIVE_CRYPTO) mongoCryptOptions.cryptoCallbacks = cryptoCallbacks; + const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders), cryptoCallbacks }; if (cryptSharedLibPath) mongoCryptOptions.cryptSharedLibPath = cryptSharedLibPath; const mongoCrypt = new MongoCrypt(mongoCryptOptions);