From 429ef0df2e09c8c21c8016821ae053362ac094b5 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 14 Jun 2023 11:26:20 -0400 Subject: [PATCH 01/11] fix(NODE-4977): ensure snappy is loaded lazily --- src/cmap/auth/gssapi.ts | 37 +++--- src/cmap/auth/mongodb_aws.ts | 9 +- src/cmap/auth/scram.ts | 5 +- src/cmap/wire_protocol/compression.ts | 63 +++++----- src/deps.ts | 160 ++++++++++++-------------- src/error.ts | 10 ++ test/action/dependency.test.ts | 48 ++++++++ test/unit/deps.test.ts | 1 + 8 files changed, 196 insertions(+), 137 deletions(-) create mode 100644 test/unit/deps.test.ts diff --git a/src/cmap/auth/gssapi.ts b/src/cmap/auth/gssapi.ts index 3f8135a4e4f..26df6ef478d 100644 --- a/src/cmap/auth/gssapi.ts +++ b/src/cmap/auth/gssapi.ts @@ -1,7 +1,11 @@ import * as dns from 'dns'; -import { getKerberos, type Kerberos, type KerberosClient } from '../../deps'; -import { MongoInvalidArgumentError, MongoMissingCredentialsError } from '../../error'; +import { getKerberos, type KerberosClient } from '../../deps'; +import { + MongoInvalidArgumentError, + MongoMissingCredentialsError, + MongoMissingDependencyError +} from '../../error'; import { ns } from '../../utils'; import type { Connection } from '../connection'; import { type AuthContext, AuthProvider } from './auth_provider'; @@ -36,7 +40,17 @@ async function externalCommand( }>; } -let krb: typeof Kerberos; +let kerberos: typeof import('kerberos') | null = null; +async function loadKerberos() { + if (kerberos == null) { + const moduleOrError = await getKerberos(); + if (MongoMissingDependencyError.isMongoMissingDependencyError(moduleOrError)) { + throw moduleOrError; + } + kerberos = moduleOrError; + } + return kerberos; +} export class GSSAPI extends AuthProvider { override async auth(authContext: AuthContext): Promise { @@ -79,11 +93,9 @@ async function makeKerberosClient(authContext: AuthContext): Promise { return host; } } - -/** - * Load the Kerberos library. - */ -function loadKrb() { - if (!krb) { - krb = getKerberos(); - } -} diff --git a/src/cmap/auth/mongodb_aws.ts b/src/cmap/auth/mongodb_aws.ts index 57e3a028ff8..75b7a3cd4e3 100644 --- a/src/cmap/auth/mongodb_aws.ts +++ b/src/cmap/auth/mongodb_aws.ts @@ -8,6 +8,7 @@ import { MongoAWSError, MongoCompatibilityError, MongoMissingCredentialsError, + MongoMissingDependencyError, MongoRuntimeError } from '../../error'; import { ByteUtils, maxWireVersion, ns, request } from '../../utils'; @@ -47,8 +48,8 @@ export class MongoDBAWS extends AuthProvider { throw new MongoMissingCredentialsError('AuthContext must provide credentials.'); } - if ('kModuleError' in aws4) { - throw aws4['kModuleError']; + if (MongoMissingDependencyError.isMongoMissingDependencyError(aws4)) { + throw aws4; } const { sign } = aws4; @@ -182,11 +183,11 @@ async function makeTempCredentials(credentials: MongoCredentials): Promise { - if (key === 'kModuleError') { - return error; - } - throw error; - }, - set: () => { - throw error; - } - }); -} - -export let Kerberos: typeof import('kerberos') | { kModuleError: MongoMissingDependencyError } = - makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `kerberos` not found. Please install it to enable kerberos authentication' - ) - ); - -export function getKerberos(): typeof Kerberos | { kModuleError: MongoMissingDependencyError } { - try { - // Ensure you always wrap an optional require in the try block NODE-3199 - Kerberos = require('kerberos'); - return Kerberos; - } catch { - return Kerberos; - } -} - export interface KerberosClient { step(challenge: string): Promise; step(challenge: string, callback: Callback): void; @@ -47,7 +14,22 @@ export interface KerberosClient { unwrap(challenge: string, callback: Callback): void; } -type ZStandardLib = { +export async function getKerberos(): Promise< + typeof import('kerberos') | MongoMissingDependencyError +> { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + return require('kerberos'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( + 'Optional module `kerberos` not found. Please install it to enable kerberos authentication' + ); + missingDepError.cause = error; + return missingDepError; + } +} + +export type ZSTD = { /** * Compress using zstd. * @param buf - Buffer to be compressed. @@ -60,44 +42,39 @@ type ZStandardLib = { decompress(buf: Buffer): Promise; }; -export let ZStandard: ZStandardLib | { kModuleError: MongoMissingDependencyError } = - makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression' - ) - ); - -export function getZstdLibrary(): typeof ZStandard | { kModuleError: MongoMissingDependencyError } { +export async function getZstd(): Promise { try { - ZStandard = require('@mongodb-js/zstd'); - return ZStandard; - } catch { - return ZStandard; + return require('@mongodb-js/zstd'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( + 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression' + ); + missingDepError.cause = error; + return missingDepError; } } -type CredentialProvider = { +type AWSCredentialProvider = { fromNodeProviderChain(this: void): () => Promise; }; -export function getAwsCredentialProvider(): - | CredentialProvider - | { kModuleError: MongoMissingDependencyError } { +export async function getAwsCredentialProvider(): Promise< + AWSCredentialProvider | MongoMissingDependencyError +> { try { // Ensure you always wrap an optional require in the try block NODE-3199 - const credentialProvider = require('@aws-sdk/credential-providers'); - return credentialProvider; - } catch { - return makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `@aws-sdk/credential-providers` not found.' + - ' Please install it to enable getting aws credentials via the official sdk.' - ) + return require('@aws-sdk/credential-providers'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( + 'Optional module `@aws-sdk/credential-providers` not found.' + + ' Please install it to enable getting aws credentials via the official sdk.' ); + missingDepError.cause = error; + return missingDepError; } } -type SnappyLib = { +export type Snappy = { /** * In order to support both we must check the return value of the function * @param buf - Buffer to be compressed @@ -111,29 +88,34 @@ type SnappyLib = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export let Snappy: SnappyLib | { kModuleError: MongoMissingDependencyError } = makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `snappy` not found. Please install it to enable snappy compression' - ) -); +export async function getSnappy(): Promise { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + return require('snappy'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( + 'Optional module `snappy` not found. Please install it to enable snappy compression' + ); + missingDepError.cause = error; + return missingDepError; + } +} -try { - // Ensure you always wrap an optional require in the try block NODE-3199 - Snappy = require('snappy'); -} catch {} // eslint-disable-line +export const saslprep = getSaslPrep(); -export let saslprep: typeof import('saslprep') | { kModuleError: MongoMissingDependencyError } = - makeErrorModule( - new MongoMissingDependencyError( +function getSaslPrep(): typeof import('saslprep') | MongoMissingDependencyError { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + return require('saslprep'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( 'Optional module `saslprep` not found.' + ' Please install it to enable Stringprep Profile for User Names and Passwords' - ) - ); - -try { - // Ensure you always wrap an optional require in the try block NODE-3199 - saslprep = require('saslprep'); -} catch {} // eslint-disable-line + ); + missingDepError.cause = error; + return missingDepError; + } +} interface AWS4 { /** @@ -176,16 +158,20 @@ interface AWS4 { }; } -export let aws4: AWS4 | { kModuleError: MongoMissingDependencyError } = makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `aws4` not found. Please install it to enable AWS authentication' - ) -); +export const aws4 = getAWS4(); -try { - // Ensure you always wrap an optional require in the try block NODE-3199 - aws4 = require('aws4'); -} catch {} // eslint-disable-line +function getAWS4(): AWS4 | MongoMissingDependencyError { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + return require('aws4'); + } catch (error) { + const missingDepError = new MongoMissingDependencyError( + 'Optional module `aws4` not found. Please install it to enable AWS authentication' + ); + missingDepError.cause = error; + return missingDepError; + } +} /** @public */ export const AutoEncryptionLoggerLevel = Object.freeze({ diff --git a/src/error.ts b/src/error.ts index f839cda2df2..18ada16feaa 100644 --- a/src/error.ts +++ b/src/error.ts @@ -699,6 +699,16 @@ export class MongoMissingDependencyError extends MongoAPIError { override get name(): string { return 'MongoMissingDependencyError'; } + + /** @internal */ + static isMongoMissingDependencyError(error: unknown): error is MongoMissingDependencyError { + return ( + error != null && + typeof error === 'object' && + 'name' in error && + error.name === 'MongoMissingDependencyError' + ); + } } /** * An error signifying a general system issue diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index ffc2a7def11..0171389ca69 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -84,4 +84,52 @@ describe('package.json', function () { }); } }); + + const EXPECTED_IMPORTS = [ + 'bson', + 'saslprep', + 'sparse-bitfield', + 'memory-pager', + 'mongodb-connection-string-url', + 'whatwg-url', + 'webidl-conversions', + 'tr46', + 'socks', + 'ip', + 'smart-buffer' + ]; + + describe('mongodb imports', () => { + let imports: string[]; + beforeEach(async function () { + for (const key of Object.keys(require.cache)) delete require.cache[key]; + require('../../src'); + imports = Array.from( + new Set( + Object.entries(require.cache) + .filter(([modKey]) => modKey.includes('/node_modules/')) + .map(([modKey]) => { + const leadingPkgName = modKey.split('/node_modules/')[1]; + const [orgName, pkgName] = leadingPkgName.split('/'); + if (orgName.startsWith('@')) { + return `${orgName}/${pkgName}`; + } + return orgName; + }) + ) + ); + }); + + context('when importing mongodb', () => { + it('only contains the expected imports', function () { + expect(imports).to.deep.equal(EXPECTED_IMPORTS); + }); + + it('does not import optional dependencies', () => { + for (const peerDependency of EXPECTED_PEER_DEPENDENCIES) { + expect(imports).to.not.include(peerDependency); + } + }); + }); + }); }); diff --git a/test/unit/deps.test.ts b/test/unit/deps.test.ts new file mode 100644 index 00000000000..9e032fd13cd --- /dev/null +++ b/test/unit/deps.test.ts @@ -0,0 +1 @@ +import { expect } from 'chai'; From c0715149ff7260fa093906dd61d04fe159274b83 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 14 Jun 2023 15:24:52 -0400 Subject: [PATCH 02/11] fix: test result structure --- src/cmap/auth/gssapi.ts | 12 +-- src/cmap/auth/mongodb_aws.ts | 22 ++-- src/cmap/auth/scram.ts | 5 +- src/cmap/wire_protocol/compression.ts | 18 ++-- src/deps.ts | 145 +++++++++++++++++--------- src/encrypter.ts | 19 ++-- src/error.ts | 13 +-- src/utils.ts | 37 ------- test/action/dependency.test.ts | 56 +++++++++- test/unit/deps.test.ts | 1 - 10 files changed, 186 insertions(+), 142 deletions(-) delete mode 100644 test/unit/deps.test.ts diff --git a/src/cmap/auth/gssapi.ts b/src/cmap/auth/gssapi.ts index 26df6ef478d..1775efb6d91 100644 --- a/src/cmap/auth/gssapi.ts +++ b/src/cmap/auth/gssapi.ts @@ -1,11 +1,7 @@ import * as dns from 'dns'; import { getKerberos, type KerberosClient } from '../../deps'; -import { - MongoInvalidArgumentError, - MongoMissingCredentialsError, - MongoMissingDependencyError -} from '../../error'; +import { MongoInvalidArgumentError, MongoMissingCredentialsError } from '../../error'; import { ns } from '../../utils'; import type { Connection } from '../connection'; import { type AuthContext, AuthProvider } from './auth_provider'; @@ -44,10 +40,10 @@ let kerberos: typeof import('kerberos') | null = null; async function loadKerberos() { if (kerberos == null) { const moduleOrError = await getKerberos(); - if (MongoMissingDependencyError.isMongoMissingDependencyError(moduleOrError)) { - throw moduleOrError; + if (moduleOrError.status === 'rejected') { + throw moduleOrError.reason; } - kerberos = moduleOrError; + kerberos = moduleOrError.value; } return kerberos; } diff --git a/src/cmap/auth/mongodb_aws.ts b/src/cmap/auth/mongodb_aws.ts index 75b7a3cd4e3..e0a8f5afafc 100644 --- a/src/cmap/auth/mongodb_aws.ts +++ b/src/cmap/auth/mongodb_aws.ts @@ -3,12 +3,16 @@ import { promisify } from 'util'; import type { Binary, BSONSerializeOptions } from '../../bson'; import * as BSON from '../../bson'; -import { aws4, getAwsCredentialProvider } from '../../deps'; +import { + aws4, + type AWSCredentialProvider, + getAwsCredentialProvider, + type ImportResult +} from '../../deps'; import { MongoAWSError, MongoCompatibilityError, MongoMissingCredentialsError, - MongoMissingDependencyError, MongoRuntimeError } from '../../error'; import { ByteUtils, maxWireVersion, ns, request } from '../../utils'; @@ -48,10 +52,10 @@ export class MongoDBAWS extends AuthProvider { throw new MongoMissingCredentialsError('AuthContext must provide credentials.'); } - if (MongoMissingDependencyError.isMongoMissingDependencyError(aws4)) { - throw aws4; + if (aws4.status === 'rejected') { + throw aws4.reason; } - const { sign } = aws4; + const { sign } = aws4.value; if (maxWireVersion(connection) < 9) { throw new MongoCompatibilityError( @@ -166,6 +170,8 @@ export interface AWSCredentials { expiration?: Date; } +let awsCredentialProvider: ImportResult; + async function makeTempCredentials(credentials: MongoCredentials): Promise { function makeMongoCredentialsFromAWSTemp(creds: AWSTempCredentials) { if (!creds.AccessKeyId || !creds.SecretAccessKey || !creds.Token) { @@ -183,11 +189,11 @@ async function makeTempCredentials(credentials: MongoCredentials): Promise = + | { status: 'fulfilled'; value: T } + | { status: 'rejected'; reason: MongoMissingDependencyError }; export interface KerberosClient { step(challenge: string): Promise; step(challenge: string, callback: Callback): void; @@ -14,18 +20,17 @@ export interface KerberosClient { unwrap(challenge: string, callback: Callback): void; } -export async function getKerberos(): Promise< - typeof import('kerberos') | MongoMissingDependencyError -> { +export async function getKerberos(): Promise> { try { // Ensure you always wrap an optional require in the try block NODE-3199 - return require('kerberos'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( - 'Optional module `kerberos` not found. Please install it to enable kerberos authentication' + const value = require('kerberos'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Optional module `kerberos` not found. Please install it to enable kerberos authentication', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } @@ -42,35 +47,35 @@ export type ZSTD = { decompress(buf: Buffer): Promise; }; -export async function getZstd(): Promise { +export async function getZstd(): Promise> { try { - return require('@mongodb-js/zstd'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( - 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression' + const value = require('@mongodb-js/zstd'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } -type AWSCredentialProvider = { +export type AWSCredentialProvider = { fromNodeProviderChain(this: void): () => Promise; }; -export async function getAwsCredentialProvider(): Promise< - AWSCredentialProvider | MongoMissingDependencyError -> { +export async function getAwsCredentialProvider(): Promise> { try { // Ensure you always wrap an optional require in the try block NODE-3199 - return require('@aws-sdk/credential-providers'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( + const value = require('@aws-sdk/credential-providers'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( 'Optional module `@aws-sdk/credential-providers` not found.' + - ' Please install it to enable getting aws credentials via the official sdk.' + ' Please install it to enable getting aws credentials via the official sdk.', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } @@ -88,32 +93,34 @@ export type Snappy = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export async function getSnappy(): Promise { +export async function getSnappy(): Promise> { try { // Ensure you always wrap an optional require in the try block NODE-3199 - return require('snappy'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( - 'Optional module `snappy` not found. Please install it to enable snappy compression' + const value = require('snappy'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Optional module `snappy` not found. Please install it to enable snappy compression', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } export const saslprep = getSaslPrep(); -function getSaslPrep(): typeof import('saslprep') | MongoMissingDependencyError { +function getSaslPrep(): ImportResult { try { // Ensure you always wrap an optional require in the try block NODE-3199 - return require('saslprep'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( + const value = require('saslprep'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( 'Optional module `saslprep` not found.' + - ' Please install it to enable Stringprep Profile for User Names and Passwords' + ' Please install it to enable Stringprep Profile for User Names and Passwords', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } @@ -160,16 +167,17 @@ interface AWS4 { export const aws4 = getAWS4(); -function getAWS4(): AWS4 | MongoMissingDependencyError { +function getAWS4(): ImportResult { try { // Ensure you always wrap an optional require in the try block NODE-3199 - return require('aws4'); - } catch (error) { - const missingDepError = new MongoMissingDependencyError( - 'Optional module `aws4` not found. Please install it to enable AWS authentication' + const value = require('aws4'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Optional module `aws4` not found. Please install it to enable AWS authentication', + { cause } ); - missingDepError.cause = error; - return missingDepError; + return { status: 'rejected', reason }; } } @@ -389,3 +397,46 @@ export interface AutoEncrypter { /** @experimental */ readonly cryptSharedLibVersionInfo: { version: bigint; versionStr: string } | null; } + +/** A utility function to get the instance of mongodb-client-encryption, if it exists. */ +export function getMongoDBClientEncryption(): ImportResult<{ + extension: (mdb: unknown) => { + AutoEncrypter: any; + ClientEncryption: any; + }; +}> { + const { MONGODB_CLIENT_ENCRYPTION_OVERRIDE = '' } = process.env; + // NOTE(NODE-4254): This is to get around the circular dependency between + // mongodb-client-encryption and the driver in the test scenarios. + if (MONGODB_CLIENT_ENCRYPTION_OVERRIDE.length > 0) { + try { + // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block + // Cannot be moved to helper utility function, bundlers search and replace the actual require call + // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed + const value = require(MONGODB_CLIENT_ENCRYPTION_OVERRIDE); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Auto-encryption requested, but the module is not installed. ' + + `tried importing: MONGODB_CLIENT_ENCRYPTION_OVERRIDE=${MONGODB_CLIENT_ENCRYPTION_OVERRIDE}`, + { cause } + ); + return { status: 'rejected', reason }; + } + } else { + try { + // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block + // Cannot be moved to helper utility function, bundlers search and replace the actual require call + // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed + const value = require('mongodb-client-encryption'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Auto-encryption requested, but the module is not installed. ' + + 'Please add `mongodb-client-encryption` as a dependency of your project', + { cause } + ); + return { status: 'rejected', reason }; + } + } +} diff --git a/src/encrypter.ts b/src/encrypter.ts index 8ef92321c59..e7ac0bc8f43 100644 --- a/src/encrypter.ts +++ b/src/encrypter.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import { MONGO_CLIENT_EVENTS } from './constants'; -import type { AutoEncrypter, AutoEncryptionOptions } from './deps'; -import { MongoInvalidArgumentError, MongoMissingDependencyError } from './error'; +import { type AutoEncrypter, type AutoEncryptionOptions, getMongoDBClientEncryption } from './deps'; +import { MongoInvalidArgumentError } from './error'; import { MongoClient, type MongoClientOptions } from './mongo_client'; -import { type Callback, getMongoDBClientEncryption } from './utils'; +import { type Callback } from './utils'; let AutoEncrypterClass: { new (...args: ConstructorParameters): AutoEncrypter }; @@ -119,13 +119,12 @@ export class Encrypter { } static checkForMongoCrypt(): void { - const mongodbClientEncryption = getMongoDBClientEncryption(); - if (mongodbClientEncryption == null) { - throw new MongoMissingDependencyError( - 'Auto-encryption requested, but the module is not installed. ' + - 'Please add `mongodb-client-encryption` as a dependency of your project' - ); + if (AutoEncrypterClass == null) { + const moduleOrError = getMongoDBClientEncryption(); + if (moduleOrError.status === 'rejected') { + throw moduleOrError.reason; + } + AutoEncrypterClass = moduleOrError.value.extension(require('../lib/index')).AutoEncrypter; } - AutoEncrypterClass = mongodbClientEncryption.extension(require('../lib/index')).AutoEncrypter; } } diff --git a/src/error.ts b/src/error.ts index 18ada16feaa..8c9f495626d 100644 --- a/src/error.ts +++ b/src/error.ts @@ -692,23 +692,14 @@ export class MongoMissingCredentialsError extends MongoAPIError { * @category Error */ export class MongoMissingDependencyError extends MongoAPIError { - constructor(message: string) { + constructor(message: string, { cause }: { cause?: Error } = {}) { super(message); + if (cause) this.cause = cause; } override get name(): string { return 'MongoMissingDependencyError'; } - - /** @internal */ - static isMongoMissingDependencyError(error: unknown): error is MongoMissingDependencyError { - return ( - error != null && - typeof error === 'object' && - 'name' in error && - error.name === 'MongoMissingDependencyError' - ); - } } /** * An error signifying a general system issue diff --git a/src/utils.ts b/src/utils.ts index 505f3bfd1d5..6dd512d1cf2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1172,43 +1172,6 @@ export function commandSupportsReadConcern(command: Document, options?: Document return false; } -/** A utility function to get the instance of mongodb-client-encryption, if it exists. */ -export function getMongoDBClientEncryption(): { - extension: (mdb: unknown) => { - AutoEncrypter: any; - ClientEncryption: any; - }; -} | null { - let mongodbClientEncryption = null; - - // NOTE(NODE-4254): This is to get around the circular dependency between - // mongodb-client-encryption and the driver in the test scenarios. - if ( - typeof process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE === 'string' && - process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE.length > 0 - ) { - try { - // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block - // Cannot be moved to helper utility function, bundlers search and replace the actual require call - // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed - mongodbClientEncryption = require(process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE); - } catch { - // ignore - } - } else { - try { - // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block - // Cannot be moved to helper utility function, bundlers search and replace the actual require call - // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed - mongodbClientEncryption = require('mongodb-client-encryption'); - } catch { - // ignore - } - } - - return mongodbClientEncryption; -} - /** * Compare objectIds. `null` is always less * - `+1 = oid1 is greater than oid2` diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index 0171389ca69..5f2cdcedd93 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -1,10 +1,12 @@ import { execSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import * as timers from 'node:timers/promises'; import { expect } from 'chai'; import { dependencies, peerDependencies, peerDependenciesMeta } from '../../package.json'; +import * as mongodb from '../mongodb'; const EXPECTED_DEPENDENCIES = ['bson', 'mongodb-connection-string-url', 'socks']; const EXPECTED_PEER_DEPENDENCIES = [ @@ -15,6 +17,24 @@ const EXPECTED_PEER_DEPENDENCIES = [ 'mongodb-client-encryption' ]; +const resolvable = depName => { + try { + require.resolve(depName); + return true; + } catch { + return false; + } +}; + +const importerMap = new Map() + .set('@aws-sdk/credential-providers', 'getAwsCredentialProvider') + .set('@mongodb-js/zstd', 'getZstd') + .set('kerberos', 'getKerberos') + .set('snappy', 'getSnappy') + .set('saslprep', 'getSaslPrep') + .set('aws4', 'getAWS4') + .set('mongodb-client-encryption', 'getMongoDBClientEncryption'); + describe('package.json', function () { describe('dependencies', function () { it('only contains the expected dependencies', function () { @@ -54,33 +74,59 @@ describe('package.json', function () { context(`when ${depName} is NOT installed`, () => { beforeEach(async () => { fs.rmSync(path.join(repoRoot, 'node_modules', depName), { recursive: true, force: true }); + + while (resolvable(depName)) { + await timers.setTimeout(100); + } + + for (const key of Object.keys(require.cache)) delete require.cache[key]; }); it(`driver is importable`, () => { - expect(fs.existsSync(path.join(repoRoot, 'node_modules', depName))).to.be.false; - const result = execSync(`./node_modules/.bin/ts-node -e "${testScript}"`, { encoding: 'utf8' }); expect(result).to.include('import success!'); }); + + it.skip('importer helper returns rejected', async () => { + const importer = importerMap.get(depName); + expect(mongodb).to.have.property(importer).that.is.a('function'); + const importResult = await mongodb[importer](); + + expect(importResult).to.have.property('status', 'rejected'); + expect(importResult) + .to.have.property('reason') + .to.be.instanceOf(mongodb.MongoMissingDependencyError); + }); }); context(`when ${depName} is installed`, () => { beforeEach(async () => { - execSync(`npm install --no-save "${depName}"@${depMajor}`); + execSync(`npm install --no-save "${depName}"@"${depMajor}"`); + while (!resolvable(depName)) { + await timers.setTimeout(100); + } + for (const key of Object.keys(require.cache)) delete require.cache[key]; }); it(`driver is importable`, () => { - expect(fs.existsSync(path.join(repoRoot, 'node_modules', depName))).to.be.true; - const result = execSync(`./node_modules/.bin/ts-node -e "${testScript}"`, { encoding: 'utf8' }); expect(result).to.include('import success!'); }); + + it.skip('importer helper returns fulfilled', async () => { + const importer = importerMap.get(depName); + expect(mongodb).to.have.property(importer).that.is.a('function'); + const importResult = await mongodb[importer](); + + expect(importResult).to.have.property('status', 'fulfilled'); + expect(importResult).to.have.property('value').and.to.exist; + }); }); } }); diff --git a/test/unit/deps.test.ts b/test/unit/deps.test.ts deleted file mode 100644 index 9e032fd13cd..00000000000 --- a/test/unit/deps.test.ts +++ /dev/null @@ -1 +0,0 @@ -import { expect } from 'chai'; From b74bd810fe2f09196d3c95bf77ea794bc94ec1f4 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Fri, 16 Jun 2023 11:27:10 -0400 Subject: [PATCH 03/11] undo --- src/cmap/auth/gssapi.ts | 31 ++-- src/cmap/auth/mongodb_aws.ts | 21 +-- src/cmap/auth/scram.ts | 2 +- src/cmap/wire_protocol/compression.ts | 57 ++++--- src/deps.ts | 211 +++++++++++--------------- src/encrypter.ts | 19 +-- src/error.ts | 3 +- src/operations/create_collection.ts | 6 +- src/utils.ts | 37 +++++ 9 files changed, 191 insertions(+), 196 deletions(-) diff --git a/src/cmap/auth/gssapi.ts b/src/cmap/auth/gssapi.ts index 1775efb6d91..3f8135a4e4f 100644 --- a/src/cmap/auth/gssapi.ts +++ b/src/cmap/auth/gssapi.ts @@ -1,6 +1,6 @@ import * as dns from 'dns'; -import { getKerberos, type KerberosClient } from '../../deps'; +import { getKerberos, type Kerberos, type KerberosClient } from '../../deps'; import { MongoInvalidArgumentError, MongoMissingCredentialsError } from '../../error'; import { ns } from '../../utils'; import type { Connection } from '../connection'; @@ -36,17 +36,7 @@ async function externalCommand( }>; } -let kerberos: typeof import('kerberos') | null = null; -async function loadKerberos() { - if (kerberos == null) { - const moduleOrError = await getKerberos(); - if (moduleOrError.status === 'rejected') { - throw moduleOrError.reason; - } - kerberos = moduleOrError.value; - } - return kerberos; -} +let krb: typeof Kerberos; export class GSSAPI extends AuthProvider { override async auth(authContext: AuthContext): Promise { @@ -89,9 +79,11 @@ async function makeKerberosClient(authContext: AuthContext): Promise { return host; } } + +/** + * Load the Kerberos library. + */ +function loadKrb() { + if (!krb) { + krb = getKerberos(); + } +} diff --git a/src/cmap/auth/mongodb_aws.ts b/src/cmap/auth/mongodb_aws.ts index e0a8f5afafc..57e3a028ff8 100644 --- a/src/cmap/auth/mongodb_aws.ts +++ b/src/cmap/auth/mongodb_aws.ts @@ -3,12 +3,7 @@ import { promisify } from 'util'; import type { Binary, BSONSerializeOptions } from '../../bson'; import * as BSON from '../../bson'; -import { - aws4, - type AWSCredentialProvider, - getAwsCredentialProvider, - type ImportResult -} from '../../deps'; +import { aws4, getAwsCredentialProvider } from '../../deps'; import { MongoAWSError, MongoCompatibilityError, @@ -52,10 +47,10 @@ export class MongoDBAWS extends AuthProvider { throw new MongoMissingCredentialsError('AuthContext must provide credentials.'); } - if (aws4.status === 'rejected') { - throw aws4.reason; + if ('kModuleError' in aws4) { + throw aws4['kModuleError']; } - const { sign } = aws4.value; + const { sign } = aws4; if (maxWireVersion(connection) < 9) { throw new MongoCompatibilityError( @@ -170,8 +165,6 @@ export interface AWSCredentials { expiration?: Date; } -let awsCredentialProvider: ImportResult; - async function makeTempCredentials(credentials: MongoCredentials): Promise { function makeMongoCredentialsFromAWSTemp(creds: AWSTempCredentials) { if (!creds.AccessKeyId || !creds.SecretAccessKey || !creds.Token) { @@ -189,11 +182,11 @@ async function makeTempCredentials(credentials: MongoCredentials): Promise = - | { status: 'fulfilled'; value: T } - | { status: 'rejected'; reason: MongoMissingDependencyError }; +function makeErrorModule(error: any) { + const props = error ? { kModuleError: error } : {}; + return new Proxy(props, { + get: (_: any, key: any) => { + if (key === 'kModuleError') { + return error; + } + throw error; + }, + set: () => { + throw error; + } + }); +} + +export let Kerberos: typeof import('kerberos') | { kModuleError: MongoMissingDependencyError } = + makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `kerberos` not found. Please install it to enable kerberos authentication' + ) + ); + +export function getKerberos(): typeof Kerberos | { kModuleError: MongoMissingDependencyError } { + try { + // Ensure you always wrap an optional require in the try block NODE-3199 + Kerberos = require('kerberos'); + return Kerberos; + } catch { + return Kerberos; + } +} + export interface KerberosClient { step(challenge: string): Promise; step(challenge: string, callback: Callback): void; @@ -20,21 +47,7 @@ export interface KerberosClient { unwrap(challenge: string, callback: Callback): void; } -export async function getKerberos(): Promise> { - try { - // Ensure you always wrap an optional require in the try block NODE-3199 - const value = require('kerberos'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Optional module `kerberos` not found. Please install it to enable kerberos authentication', - { cause } - ); - return { status: 'rejected', reason }; - } -} - -export type ZSTD = { +type ZStandardLib = { /** * Compress using zstd. * @param buf - Buffer to be compressed. @@ -47,39 +60,44 @@ export type ZSTD = { decompress(buf: Buffer): Promise; }; -export async function getZstd(): Promise> { +export let ZStandard: ZStandardLib | { kModuleError: MongoMissingDependencyError } = + makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression' + ) + ); + +export function getZstdLibrary(): typeof ZStandard | { kModuleError: MongoMissingDependencyError } { try { - const value = require('@mongodb-js/zstd'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression', - { cause } - ); - return { status: 'rejected', reason }; + ZStandard = require('@mongodb-js/zstd'); + return ZStandard; + } catch { + return ZStandard; } } -export type AWSCredentialProvider = { +type CredentialProvider = { fromNodeProviderChain(this: void): () => Promise; }; -export async function getAwsCredentialProvider(): Promise> { +export function getAwsCredentialProvider(): + | CredentialProvider + | { kModuleError: MongoMissingDependencyError } { try { // Ensure you always wrap an optional require in the try block NODE-3199 - const value = require('@aws-sdk/credential-providers'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Optional module `@aws-sdk/credential-providers` not found.' + - ' Please install it to enable getting aws credentials via the official sdk.', - { cause } + const credentialProvider = require('@aws-sdk/credential-providers'); + return credentialProvider; + } catch { + return makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `@aws-sdk/credential-providers` not found.' + + ' Please install it to enable getting aws credentials via the official sdk.' + ) ); - return { status: 'rejected', reason }; } } -export type Snappy = { +type SnappyLib = { /** * In order to support both we must check the return value of the function * @param buf - Buffer to be compressed @@ -93,36 +111,29 @@ export type Snappy = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export async function getSnappy(): Promise> { - try { - // Ensure you always wrap an optional require in the try block NODE-3199 - const value = require('snappy'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Optional module `snappy` not found. Please install it to enable snappy compression', - { cause } - ); - return { status: 'rejected', reason }; - } -} +export let Snappy: SnappyLib | { kModuleError: MongoMissingDependencyError } = makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `snappy` not found. Please install it to enable snappy compression' + ) +); -export const saslprep = getSaslPrep(); +try { + // Ensure you always wrap an optional require in the try block NODE-3199 + Snappy = require('snappy'); +} catch {} // eslint-disable-line -function getSaslPrep(): ImportResult { - try { - // Ensure you always wrap an optional require in the try block NODE-3199 - const value = require('saslprep'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( +export let saslprep: typeof import('saslprep') | { kModuleError: MongoMissingDependencyError } = + makeErrorModule( + new MongoMissingDependencyError( 'Optional module `saslprep` not found.' + - ' Please install it to enable Stringprep Profile for User Names and Passwords', - { cause } - ); - return { status: 'rejected', reason }; - } -} + ' Please install it to enable Stringprep Profile for User Names and Passwords' + ) + ); + +try { + // Ensure you always wrap an optional require in the try block NODE-3199 + saslprep = require('saslprep'); +} catch {} // eslint-disable-line interface AWS4 { /** @@ -165,21 +176,16 @@ interface AWS4 { }; } -export const aws4 = getAWS4(); +export let aws4: AWS4 | { kModuleError: MongoMissingDependencyError } = makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `aws4` not found. Please install it to enable AWS authentication' + ) +); -function getAWS4(): ImportResult { - try { - // Ensure you always wrap an optional require in the try block NODE-3199 - const value = require('aws4'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Optional module `aws4` not found. Please install it to enable AWS authentication', - { cause } - ); - return { status: 'rejected', reason }; - } -} +try { + // Ensure you always wrap an optional require in the try block NODE-3199 + aws4 = require('aws4'); +} catch {} // eslint-disable-line /** @public */ export const AutoEncryptionLoggerLevel = Object.freeze({ @@ -397,46 +403,3 @@ export interface AutoEncrypter { /** @experimental */ readonly cryptSharedLibVersionInfo: { version: bigint; versionStr: string } | null; } - -/** A utility function to get the instance of mongodb-client-encryption, if it exists. */ -export function getMongoDBClientEncryption(): ImportResult<{ - extension: (mdb: unknown) => { - AutoEncrypter: any; - ClientEncryption: any; - }; -}> { - const { MONGODB_CLIENT_ENCRYPTION_OVERRIDE = '' } = process.env; - // NOTE(NODE-4254): This is to get around the circular dependency between - // mongodb-client-encryption and the driver in the test scenarios. - if (MONGODB_CLIENT_ENCRYPTION_OVERRIDE.length > 0) { - try { - // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block - // Cannot be moved to helper utility function, bundlers search and replace the actual require call - // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed - const value = require(MONGODB_CLIENT_ENCRYPTION_OVERRIDE); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Auto-encryption requested, but the module is not installed. ' + - `tried importing: MONGODB_CLIENT_ENCRYPTION_OVERRIDE=${MONGODB_CLIENT_ENCRYPTION_OVERRIDE}`, - { cause } - ); - return { status: 'rejected', reason }; - } - } else { - try { - // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block - // Cannot be moved to helper utility function, bundlers search and replace the actual require call - // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed - const value = require('mongodb-client-encryption'); - return { status: 'fulfilled', value }; - } catch (cause) { - const reason = new MongoMissingDependencyError( - 'Auto-encryption requested, but the module is not installed. ' + - 'Please add `mongodb-client-encryption` as a dependency of your project', - { cause } - ); - return { status: 'rejected', reason }; - } - } -} diff --git a/src/encrypter.ts b/src/encrypter.ts index e7ac0bc8f43..8ef92321c59 100644 --- a/src/encrypter.ts +++ b/src/encrypter.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import { MONGO_CLIENT_EVENTS } from './constants'; -import { type AutoEncrypter, type AutoEncryptionOptions, getMongoDBClientEncryption } from './deps'; -import { MongoInvalidArgumentError } from './error'; +import type { AutoEncrypter, AutoEncryptionOptions } from './deps'; +import { MongoInvalidArgumentError, MongoMissingDependencyError } from './error'; import { MongoClient, type MongoClientOptions } from './mongo_client'; -import { type Callback } from './utils'; +import { type Callback, getMongoDBClientEncryption } from './utils'; let AutoEncrypterClass: { new (...args: ConstructorParameters): AutoEncrypter }; @@ -119,12 +119,13 @@ export class Encrypter { } static checkForMongoCrypt(): void { - if (AutoEncrypterClass == null) { - const moduleOrError = getMongoDBClientEncryption(); - if (moduleOrError.status === 'rejected') { - throw moduleOrError.reason; - } - AutoEncrypterClass = moduleOrError.value.extension(require('../lib/index')).AutoEncrypter; + const mongodbClientEncryption = getMongoDBClientEncryption(); + if (mongodbClientEncryption == null) { + throw new MongoMissingDependencyError( + 'Auto-encryption requested, but the module is not installed. ' + + 'Please add `mongodb-client-encryption` as a dependency of your project' + ); } + AutoEncrypterClass = mongodbClientEncryption.extension(require('../lib/index')).AutoEncrypter; } } diff --git a/src/error.ts b/src/error.ts index 8c9f495626d..f839cda2df2 100644 --- a/src/error.ts +++ b/src/error.ts @@ -692,9 +692,8 @@ export class MongoMissingCredentialsError extends MongoAPIError { * @category Error */ export class MongoMissingDependencyError extends MongoAPIError { - constructor(message: string, { cause }: { cause?: Error } = {}) { + constructor(message: string) { super(message); - if (cause) this.cause = cause; } override get name(): string { diff --git a/src/operations/create_collection.ts b/src/operations/create_collection.ts index 0f0ae9fb471..da245a79ce5 100644 --- a/src/operations/create_collection.ts +++ b/src/operations/create_collection.ts @@ -137,7 +137,11 @@ export class CreateCollectionOperation extends CommandOperation { if (encryptedFields) { // Creating a QE collection required min server of 7.0.0 - if (server.description.maxWireVersion < MIN_SUPPORTED_QE_WIRE_VERSION) { + // TODO(NODE-5353): Get wire version information from connection. + if ( + !server.loadBalanced && + server.description.maxWireVersion < MIN_SUPPORTED_QE_WIRE_VERSION + ) { throw new MongoCompatibilityError( `${INVALID_QE_VERSION} The minimum server version required is ${MIN_SUPPORTED_QE_SERVER_VERSION}` ); diff --git a/src/utils.ts b/src/utils.ts index 6dd512d1cf2..505f3bfd1d5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1172,6 +1172,43 @@ export function commandSupportsReadConcern(command: Document, options?: Document return false; } +/** A utility function to get the instance of mongodb-client-encryption, if it exists. */ +export function getMongoDBClientEncryption(): { + extension: (mdb: unknown) => { + AutoEncrypter: any; + ClientEncryption: any; + }; +} | null { + let mongodbClientEncryption = null; + + // NOTE(NODE-4254): This is to get around the circular dependency between + // mongodb-client-encryption and the driver in the test scenarios. + if ( + typeof process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE === 'string' && + process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE.length > 0 + ) { + try { + // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block + // Cannot be moved to helper utility function, bundlers search and replace the actual require call + // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed + mongodbClientEncryption = require(process.env.MONGODB_CLIENT_ENCRYPTION_OVERRIDE); + } catch { + // ignore + } + } else { + try { + // NOTE(NODE-3199): Ensure you always wrap an optional require literally in the try block + // Cannot be moved to helper utility function, bundlers search and replace the actual require call + // in a way that makes this line throw at bundle time, not runtime, catching here will make bundling succeed + mongodbClientEncryption = require('mongodb-client-encryption'); + } catch { + // ignore + } + } + + return mongodbClientEncryption; +} + /** * Compare objectIds. `null` is always less * - `+1 = oid1 is greater than oid2` From 4487070ecf84da58a954ac8516cedaffabe9d87f Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Fri, 16 Jun 2023 11:39:34 -0400 Subject: [PATCH 04/11] fix: make snappy lazy --- src/cmap/wire_protocol/compression.ts | 21 ++++++++++++------- src/deps.ts | 30 +++++++++++++++++---------- src/error.ts | 3 ++- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/cmap/wire_protocol/compression.ts b/src/cmap/wire_protocol/compression.ts index 7701b0d4033..2ff043ff70d 100644 --- a/src/cmap/wire_protocol/compression.ts +++ b/src/cmap/wire_protocol/compression.ts @@ -2,7 +2,7 @@ import { promisify } from 'util'; import * as zlib from 'zlib'; import { LEGACY_HELLO_COMMAND } from '../../constants'; -import { getZstdLibrary, Snappy, type ZStandard } from '../../deps'; +import { getSnappy, getZstdLibrary, type SnappyLib, type ZStandard } from '../../deps'; import { MongoDecompressionError, MongoInvalidArgumentError } from '../../error'; /** @public */ @@ -38,6 +38,17 @@ const zlibInflate = promisify(zlib.inflate.bind(zlib)); const zlibDeflate = promisify(zlib.deflate.bind(zlib)); let zstd: typeof ZStandard; +let Snappy: SnappyLib | null = null; +async function loadSnappy() { + if (Snappy == null) { + const snappyImport = await getSnappy(); + if (snappyImport.status === 'rejected') { + throw snappyImport.reason; + } + Snappy = snappyImport.value; + } + return Snappy; +} // Facilitate compressing a message using an agreed compressor export async function compress( @@ -47,9 +58,7 @@ export async function compress( const zlibOptions = {} as zlib.ZlibOptions; switch (options.agreedCompressor) { case 'snappy': { - if ('kModuleError' in Snappy) { - throw Snappy['kModuleError']; - } + Snappy ??= await loadSnappy(); return Snappy.compress(dataToBeCompressed); } case 'zstd': { @@ -88,9 +97,7 @@ export async function decompress(compressorID: number, compressedData: Buffer): switch (compressorID) { case Compressor.snappy: { - if ('kModuleError' in Snappy) { - throw Snappy['kModuleError']; - } + Snappy ??= await loadSnappy(); return Snappy.uncompress(compressedData, { asBuffer: true }); } case Compressor.zstd: { diff --git a/src/deps.ts b/src/deps.ts index 008a2757960..18e91d16475 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -6,6 +6,11 @@ import { MongoMissingDependencyError } from './error'; import type { MongoClient } from './mongo_client'; import type { Callback } from './utils'; +/** @internal */ +export type ImportResult = + | { status: 'fulfilled'; value: Module } + | { status: 'rejected'; reason: MongoMissingDependencyError }; + function makeErrorModule(error: any) { const props = error ? { kModuleError: error } : {}; return new Proxy(props, { @@ -97,7 +102,8 @@ export function getAwsCredentialProvider(): } } -type SnappyLib = { +/** @internal */ +export type SnappyLib = { /** * In order to support both we must check the return value of the function * @param buf - Buffer to be compressed @@ -111,16 +117,18 @@ type SnappyLib = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export let Snappy: SnappyLib | { kModuleError: MongoMissingDependencyError } = makeErrorModule( - new MongoMissingDependencyError( - 'Optional module `snappy` not found. Please install it to enable snappy compression' - ) -); - -try { - // Ensure you always wrap an optional require in the try block NODE-3199 - Snappy = require('snappy'); -} catch {} // eslint-disable-line +export async function getSnappy(): Promise> { + try { + const value = require('snappy'); + return { status: 'fulfilled', value }; + } catch (cause) { + const reason = new MongoMissingDependencyError( + 'Optional module `snappy` not found. Please install it to enable snappy compression', + { cause } + ); + return { status: 'rejected', reason }; + } +} export let saslprep: typeof import('saslprep') | { kModuleError: MongoMissingDependencyError } = makeErrorModule( diff --git a/src/error.ts b/src/error.ts index f839cda2df2..8c9f495626d 100644 --- a/src/error.ts +++ b/src/error.ts @@ -692,8 +692,9 @@ export class MongoMissingCredentialsError extends MongoAPIError { * @category Error */ export class MongoMissingDependencyError extends MongoAPIError { - constructor(message: string) { + constructor(message: string, { cause }: { cause?: Error } = {}) { super(message); + if (cause) this.cause = cause; } override get name(): string { From 5565230ec06cf29215154c0ee8c6487517341530 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Fri, 16 Jun 2023 12:55:21 -0400 Subject: [PATCH 05/11] test: snappy get --- test/action/dependency.test.ts | 76 +++++++++++++--------------------- test/tools/utils.ts | 60 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 48 deletions(-) diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index 5f2cdcedd93..322163038e7 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -1,12 +1,11 @@ import { execSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import * as timers from 'node:timers/promises'; import { expect } from 'chai'; import { dependencies, peerDependencies, peerDependenciesMeta } from '../../package.json'; -import * as mongodb from '../mongodb'; +import { itInNodeProcess } from '../tools/utils'; const EXPECTED_DEPENDENCIES = ['bson', 'mongodb-connection-string-url', 'socks']; const EXPECTED_PEER_DEPENDENCIES = [ @@ -17,24 +16,6 @@ const EXPECTED_PEER_DEPENDENCIES = [ 'mongodb-client-encryption' ]; -const resolvable = depName => { - try { - require.resolve(depName); - return true; - } catch { - return false; - } -}; - -const importerMap = new Map() - .set('@aws-sdk/credential-providers', 'getAwsCredentialProvider') - .set('@mongodb-js/zstd', 'getZstd') - .set('kerberos', 'getKerberos') - .set('snappy', 'getSnappy') - .set('saslprep', 'getSaslPrep') - .set('aws4', 'getAWS4') - .set('mongodb-client-encryption', 'getMongoDBClientEncryption'); - describe('package.json', function () { describe('dependencies', function () { it('only contains the expected dependencies', function () { @@ -74,15 +55,11 @@ describe('package.json', function () { context(`when ${depName} is NOT installed`, () => { beforeEach(async () => { fs.rmSync(path.join(repoRoot, 'node_modules', depName), { recursive: true, force: true }); - - while (resolvable(depName)) { - await timers.setTimeout(100); - } - - for (const key of Object.keys(require.cache)) delete require.cache[key]; }); it(`driver is importable`, () => { + expect(fs.existsSync(path.join(repoRoot, 'node_modules', depName))).to.be.false; + const result = execSync(`./node_modules/.bin/ts-node -e "${testScript}"`, { encoding: 'utf8' }); @@ -90,28 +67,29 @@ describe('package.json', function () { expect(result).to.include('import success!'); }); - it.skip('importer helper returns rejected', async () => { - const importer = importerMap.get(depName); - expect(mongodb).to.have.property(importer).that.is.a('function'); - const importResult = await mongodb[importer](); - - expect(importResult).to.have.property('status', 'rejected'); - expect(importResult) - .to.have.property('reason') - .to.be.instanceOf(mongodb.MongoMissingDependencyError); - }); + if (depName === 'snappy') { + itInNodeProcess( + 'getSnappy returns rejected import', + async function ({ expect, mongodb }) { + const snappyImport = await mongodb.getSnappy(); + expect(snappyImport).to.have.property('status', 'rejected'); + expect(snappyImport).to.have.nested.property( + 'reason.name', + 'MongoMissingDependencyError' + ); + } + ); + } }); context(`when ${depName} is installed`, () => { beforeEach(async () => { execSync(`npm install --no-save "${depName}"@"${depMajor}"`); - while (!resolvable(depName)) { - await timers.setTimeout(100); - } - for (const key of Object.keys(require.cache)) delete require.cache[key]; }); it(`driver is importable`, () => { + expect(fs.existsSync(path.join(repoRoot, 'node_modules', depName))).to.be.true; + const result = execSync(`./node_modules/.bin/ts-node -e "${testScript}"`, { encoding: 'utf8' }); @@ -119,14 +97,16 @@ describe('package.json', function () { expect(result).to.include('import success!'); }); - it.skip('importer helper returns fulfilled', async () => { - const importer = importerMap.get(depName); - expect(mongodb).to.have.property(importer).that.is.a('function'); - const importResult = await mongodb[importer](); - - expect(importResult).to.have.property('status', 'fulfilled'); - expect(importResult).to.have.property('value').and.to.exist; - }); + if (depName === 'snappy') { + itInNodeProcess( + 'getSnappy returns fulfilled import', + async function ({ expect, mongodb }) { + const snappyImport = await mongodb.getSnappy(); + expect(snappyImport).to.have.property('status', 'fulfilled'); + expect(snappyImport).to.have.property('value'); + } + ); + } }); } }); diff --git a/test/tools/utils.ts b/test/tools/utils.ts index 19544897921..75df28d784d 100644 --- a/test/tools/utils.ts +++ b/test/tools/utils.ts @@ -1,3 +1,8 @@ +import * as child_process from 'node:child_process'; +import { once } from 'node:events'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + import { EJSON } from 'bson'; import * as BSON from 'bson'; import { expect } from 'chai'; @@ -499,3 +504,58 @@ export function topologyWithPlaceholderClient( options as TopologyOptions ); } + +export async function itInNodeProcess( + title: string, + fn: (d: { expect: typeof import('chai').expect; mongodb: typeof import('../mongodb') }) => void +) { + it(title, async () => { + const script = ` + import { expect } from 'chai'; + import * as mongodb from './test/mongodb'; + const run = ${fn}; + run({ expect, mongodb }).then( + () => process.exit(0), + error => { + console.error(error) + process.exit(1) + } + );\n`; + + const scriptName = `./testing_${title.split(/\s/).join('_')}_script.cts`; + const cwd = path.resolve(__dirname, '..', '..'); + const tsNode = path.resolve(__dirname, '..', '..', 'node_modules', '.bin', 'ts-node'); + try { + await fs.writeFile(scriptName, script, { encoding: 'utf8' }); + const scriptInstance = child_process.fork(scriptName, { + signal: AbortSignal.timeout(50_000), + cwd, + stdio: 'pipe', + execArgv: [tsNode] + }); + + scriptInstance.stdout?.setEncoding('utf8'); + scriptInstance.stderr?.setEncoding('utf8'); + + let stdout = ''; + scriptInstance.stdout?.addListener('data', data => { + stdout += data; + }); + + let stderr = ''; + scriptInstance.stderr?.addListener('data', (data: string) => { + stderr += data + .split('\n') + .filter(line => !line.startsWith('Debugger') && !line.startsWith('For help')) + .join('\n'); + }); + + const [exitCode] = await once(scriptInstance, 'close'); + + if (stderr.length) console.log(stderr); + expect({ exitCode, stdout, stderr }).to.deep.equal({ exitCode: 0, stdout: '', stderr: '' }); + } finally { + await fs.unlink(scriptName); + } + }); +} From f6e5cc8629a051dc3ff16cc334920f1fcacf327b Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 20 Jun 2023 13:13:24 -0400 Subject: [PATCH 06/11] fix: exitCode and filtering after chunks --- test/tools/utils.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/tools/utils.ts b/test/tools/utils.ts index 75df28d784d..6139ac0a139 100644 --- a/test/tools/utils.ts +++ b/test/tools/utils.ts @@ -515,10 +515,12 @@ export async function itInNodeProcess( import * as mongodb from './test/mongodb'; const run = ${fn}; run({ expect, mongodb }).then( - () => process.exit(0), + () => { + process.exitCode = 0; + }, error => { console.error(error) - process.exit(1) + process.exitCode = 1; } );\n`; @@ -544,12 +546,15 @@ export async function itInNodeProcess( let stderr = ''; scriptInstance.stderr?.addListener('data', (data: string) => { - stderr += data - .split('\n') - .filter(line => !line.startsWith('Debugger') && !line.startsWith('For help')) - .join('\n'); + stderr += data; }); + // do not fail the test if the debugger is running + stderr = stderr + .split('\n') + .filter(line => !line.startsWith('Debugger') && !line.startsWith('For help')) + .join('\n'); + const [exitCode] = await once(scriptInstance, 'close'); if (stderr.length) console.log(stderr); From 79b095ac28d33d1a3ae84c88c0e2bdf758654b9e Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 20 Jun 2023 16:12:17 -0400 Subject: [PATCH 07/11] fix: remove ImportResult --- src/cmap/wire_protocol/compression.ts | 6 +++--- src/deps.ts | 16 +++++++--------- test/action/dependency.test.ts | 7 +++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/cmap/wire_protocol/compression.ts b/src/cmap/wire_protocol/compression.ts index 2ff043ff70d..85a29262280 100644 --- a/src/cmap/wire_protocol/compression.ts +++ b/src/cmap/wire_protocol/compression.ts @@ -42,10 +42,10 @@ let Snappy: SnappyLib | null = null; async function loadSnappy() { if (Snappy == null) { const snappyImport = await getSnappy(); - if (snappyImport.status === 'rejected') { - throw snappyImport.reason; + if ('kModuleError' in snappyImport) { + throw snappyImport.kModuleError; } - Snappy = snappyImport.value; + Snappy = snappyImport; } return Snappy; } diff --git a/src/deps.ts b/src/deps.ts index 18e91d16475..ce82cc49f8d 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -6,11 +6,6 @@ import { MongoMissingDependencyError } from './error'; import type { MongoClient } from './mongo_client'; import type { Callback } from './utils'; -/** @internal */ -export type ImportResult = - | { status: 'fulfilled'; value: Module } - | { status: 'rejected'; reason: MongoMissingDependencyError }; - function makeErrorModule(error: any) { const props = error ? { kModuleError: error } : {}; return new Proxy(props, { @@ -117,16 +112,19 @@ export type SnappyLib = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export async function getSnappy(): Promise> { +export async function getSnappy(): Promise< + SnappyLib | { kModuleError: MongoMissingDependencyError } +> { try { + // Ensure you always wrap an optional require in the try block NODE-3199 const value = require('snappy'); - return { status: 'fulfilled', value }; + return value; } catch (cause) { - const reason = new MongoMissingDependencyError( + const kModuleError = new MongoMissingDependencyError( 'Optional module `snappy` not found. Please install it to enable snappy compression', { cause } ); - return { status: 'rejected', reason }; + return { kModuleError }; } } diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index 322163038e7..ad8ffb59288 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -72,9 +72,8 @@ describe('package.json', function () { 'getSnappy returns rejected import', async function ({ expect, mongodb }) { const snappyImport = await mongodb.getSnappy(); - expect(snappyImport).to.have.property('status', 'rejected'); expect(snappyImport).to.have.nested.property( - 'reason.name', + 'kModuleError.name', 'MongoMissingDependencyError' ); } @@ -102,8 +101,8 @@ describe('package.json', function () { 'getSnappy returns fulfilled import', async function ({ expect, mongodb }) { const snappyImport = await mongodb.getSnappy(); - expect(snappyImport).to.have.property('status', 'fulfilled'); - expect(snappyImport).to.have.property('value'); + expect(snappyImport).to.have.property('compress').that.is.a('function'); + expect(snappyImport).to.have.property('decompress').that.is.a('function'); } ); } From 3bb9f4b8144ba0516315f0285b905b48a4cadc88 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 20 Jun 2023 17:02:37 -0400 Subject: [PATCH 08/11] fix: assertion --- test/action/dependency.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index ad8ffb59288..807a4963db2 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -102,7 +102,7 @@ describe('package.json', function () { async function ({ expect, mongodb }) { const snappyImport = await mongodb.getSnappy(); expect(snappyImport).to.have.property('compress').that.is.a('function'); - expect(snappyImport).to.have.property('decompress').that.is.a('function'); + expect(snappyImport).to.have.property('uncompress').that.is.a('function'); } ); } From 71a72af1e088467edde841aff1ac957a244babd7 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 29 Jun 2023 13:34:49 -0400 Subject: [PATCH 09/11] test: add unit test for cause --- test/unit/error.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/unit/error.test.ts b/test/unit/error.test.ts index 7a6551e3368..cfeb854b6e5 100644 --- a/test/unit/error.test.ts +++ b/test/unit/error.test.ts @@ -16,6 +16,7 @@ import { MONGODB_ERROR_CODES, MongoError, MongoErrorLabel, + MongoMissingDependencyError, MongoNetworkError, MongoNetworkTimeoutError, MongoParseError, @@ -155,6 +156,22 @@ describe('MongoErrors', () => { }); }); + describe('MongoMissingDependencyError#constructor', () => { + context('when options.cause is set', () => { + it('attaches the cause property to the instance', () => { + const error = new MongoMissingDependencyError('missing!', { cause: new Error('hello') }); + expect(error).to.have.property('cause'); + }); + }); + + context('when options.cause is not set', () => { + it('attaches the cause property to the instance', () => { + const error = new MongoMissingDependencyError('missing!', { cause: undefined }); + expect(error).to.not.have.property('cause'); + }); + }); + }); + describe('#isSDAMUnrecoverableError', function () { context('when the error is a MongoParseError', function () { it('returns true', function () { From fa53dde0742183ef900d765eea5ceb3a4853d796 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Fri, 30 Jun 2023 10:23:58 -0400 Subject: [PATCH 10/11] fix --- src/cmap/wire_protocol/compression.ts | 8 ++++---- src/deps.ts | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cmap/wire_protocol/compression.ts b/src/cmap/wire_protocol/compression.ts index 85a29262280..6e55268c54a 100644 --- a/src/cmap/wire_protocol/compression.ts +++ b/src/cmap/wire_protocol/compression.ts @@ -39,9 +39,9 @@ const zlibDeflate = promisify(zlib.deflate.bind(zlib)); let zstd: typeof ZStandard; let Snappy: SnappyLib | null = null; -async function loadSnappy() { +function loadSnappy() { if (Snappy == null) { - const snappyImport = await getSnappy(); + const snappyImport = getSnappy(); if ('kModuleError' in snappyImport) { throw snappyImport.kModuleError; } @@ -58,7 +58,7 @@ export async function compress( const zlibOptions = {} as zlib.ZlibOptions; switch (options.agreedCompressor) { case 'snappy': { - Snappy ??= await loadSnappy(); + Snappy ??= loadSnappy(); return Snappy.compress(dataToBeCompressed); } case 'zstd': { @@ -97,7 +97,7 @@ export async function decompress(compressorID: number, compressedData: Buffer): switch (compressorID) { case Compressor.snappy: { - Snappy ??= await loadSnappy(); + Snappy ??= loadSnappy(); return Snappy.uncompress(compressedData, { asBuffer: true }); } case Compressor.zstd: { diff --git a/src/deps.ts b/src/deps.ts index ce82cc49f8d..df2efada116 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -112,9 +112,7 @@ export type SnappyLib = { uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; }; -export async function getSnappy(): Promise< - SnappyLib | { kModuleError: MongoMissingDependencyError } -> { +export function getSnappy(): SnappyLib | { kModuleError: MongoMissingDependencyError } { try { // Ensure you always wrap an optional require in the try block NODE-3199 const value = require('snappy'); From 97f6043e425ba631de6a064f3c25d4ddc40dbe25 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Fri, 30 Jun 2023 10:29:25 -0400 Subject: [PATCH 11/11] fix test --- test/action/dependency.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/action/dependency.test.ts b/test/action/dependency.test.ts index 807a4963db2..7035124fb5a 100644 --- a/test/action/dependency.test.ts +++ b/test/action/dependency.test.ts @@ -71,7 +71,7 @@ describe('package.json', function () { itInNodeProcess( 'getSnappy returns rejected import', async function ({ expect, mongodb }) { - const snappyImport = await mongodb.getSnappy(); + const snappyImport = mongodb.getSnappy(); expect(snappyImport).to.have.nested.property( 'kModuleError.name', 'MongoMissingDependencyError' @@ -100,7 +100,7 @@ describe('package.json', function () { itInNodeProcess( 'getSnappy returns fulfilled import', async function ({ expect, mongodb }) { - const snappyImport = await mongodb.getSnappy(); + const snappyImport = mongodb.getSnappy(); expect(snappyImport).to.have.property('compress').that.is.a('function'); expect(snappyImport).to.have.property('uncompress').that.is.a('function'); }