refactor(NODE-5419): move csfle source and tests into the driver

.eslintignore

@@ -2,3 +2,6 @@
 lib
 test/disabled
 !etc/docs
+
+src/client-side-encryption
+test/unit/client-side-encryption

.evergreen/config.in.yml

@@ -196,6 +196,8 @@ functions:
             export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
             export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
             export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+            export AWS_REGION='${AWS_REGION}'
+            export AWS_CMK_ID='${AWS_CMK_ID}'
             export AWS_DEFAULT_REGION='us-east-1'
             export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
             export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -250,6 +252,8 @@ functions:
             export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
             export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
             export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+            export AWS_REGION='${AWS_REGION}'
+            export AWS_CMK_ID='${AWS_CMK_ID}'
             export AWS_DEFAULT_REGION='us-east-1'
             export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
             export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -557,6 +561,8 @@ functions:
           cat <<EOT > prepare_client_encryption.sh
           export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}'
           export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_REGION='${AWS_REGION}'
+          export AWS_CMK_ID='${AWS_CMK_ID}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
           export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
@@ -974,6 +980,8 @@ functions:
           export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+          export AWS_REGION='${AWS_REGION}'
+          export AWS_CMK_ID='${AWS_CMK_ID}'
           export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
           export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT

.evergreen/config.yml

@@ -165,6 +165,8 @@ functions:
             export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
             export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
             export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+            export AWS_REGION='${AWS_REGION}'
+            export AWS_CMK_ID='${AWS_CMK_ID}'
             export AWS_DEFAULT_REGION='us-east-1'
             export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
             export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -218,6 +220,8 @@ functions:
             export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
             export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
             export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+            export AWS_REGION='${AWS_REGION}'
+            export AWS_CMK_ID='${AWS_CMK_ID}'
             export AWS_DEFAULT_REGION='us-east-1'
             export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
             export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -501,6 +505,8 @@ functions:
           cat <<EOT > prepare_client_encryption.sh
           export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}'
           export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_REGION='${AWS_REGION}'
+          export AWS_CMK_ID='${AWS_CMK_ID}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
           export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
@@ -921,6 +927,8 @@ functions:
           export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
           export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
           export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
+          export AWS_REGION='${AWS_REGION}'
+          export AWS_CMK_ID='${AWS_CMK_ID}'
           export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
           export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
           EOT
@@ -2668,7 +2676,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
+          CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
   - name: run-custom-csfle-tests-5.0-master
     tags:
       - run-custom-dependency-tests
@@ -2698,7 +2706,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
+          CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
   - name: run-custom-csfle-tests-rapid-master
     tags:
       - run-custom-dependency-tests
@@ -2728,7 +2736,7 @@ tasks:
       - func: bootstrap kms servers
       - func: run custom csfle tests
         vars:
-          CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
+          CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
   - name: run-custom-csfle-tests-latest-master
     tags:
       - run-custom-dependency-tests
@@ -3646,7 +3654,6 @@ buildvariants:
       - test-auth-ldap
       - test-auth-oidc
       - test-socks5
-      - test-socks5-csfle
       - test-socks5-tls
       - test-tls-support-latest
       - test-tls-support-6.0
@@ -3697,7 +3704,6 @@ buildvariants:
       - test-auth-ldap
       - test-auth-oidc
       - test-socks5
-      - test-socks5-csfle
       - test-socks5-tls
       - test-tls-support-latest
       - test-tls-support-6.0

.evergreen/generate_evergreen_tasks.js

@@ -643,7 +643,7 @@ BUILD_VARIANTS.push({
 
 const oneOffFuncAsTasks = [];
 
-const FLE_PINNED_COMMIT = 'c56c70340093070b1ef5c8a28190187eea21a6e9';
+const FLE_PINNED_COMMIT = '974a4614f8c1c3786e5e39fa63568d83f4f69ebd';
 
 for (const version of ['5.0', 'rapid', 'latest']) {
   for (const ref of [FLE_PINNED_COMMIT, 'master']) {
@@ -807,6 +807,13 @@ for (const variant of BUILD_VARIANTS.filter(
   variant.tasks = variant.tasks.filter(name => !['test-socks5'].includes(name));
 }
 
+// TODO(NODE-5283): fix socks5 fle tests on node 20+
+for (const variant of BUILD_VARIANTS.filter(
+  variant => variant.expansions && [20].includes(variant.expansions.NODE_LTS_VERSION)
+) ) {
+  variant.tasks = variant.tasks.filter(name => !['test-socks5-csfle'].includes(name));
+}
+
 const fileData = yaml.load(fs.readFileSync(`${__dirname}/config.in.yml`, 'utf8'));
 fileData.tasks = (fileData.tasks || [])
   .concat(BASE_TASKS)

.evergreen/run-azure-kms-tests.sh

@@ -9,7 +9,7 @@ source ".evergreen/init-node-and-npm-env.sh"
 
 set -o xtrace
 
-npm install mongodb-client-encryption
+npm install mongodb-client-encryption@alpha --force
 
 export MONGODB_URI="mongodb://localhost:27017"
 

.evergreen/run-custom-csfle-tests.sh

@@ -55,54 +55,20 @@ pushd libmongocrypt/bindings/node
 npm install --production --ignore-scripts
 bash ./etc/build-static.sh
 
+npm run rebuild # just in case this is necessary?
+
+ls
+ls lib
+BINDINGS_DIR=$(pwd)
 popd # libmongocrypt/bindings/node
 popd # ../csfle-deps-tmp
 
 # copy mongodb-client-encryption into driver's node_modules
-cp -R ../csfle-deps-tmp/libmongocrypt/bindings/node node_modules/mongodb-client-encryption
+npm link $BINDINGS_DIR
 
 export MONGODB_URI=${MONGODB_URI}
 export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
 export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
 export TEST_CSFLE=true
 
-set +o errexit # We want to run both test suites even if the first fails
 npm run check:csfle
-DRIVER_CSFLE_TEST_RESULT=$?
-set -o errexit
-
-# Great! our drivers tests ran
-# there are tests inside the bindings repo that we also want to check
-
-pushd ../csfle-deps-tmp/libmongocrypt/bindings/node
-
-# a mongocryptd was certainly started by the driver tests,
-# let us let the bindings tests start their own
-killall mongocryptd || true
-
-# only prod deps were installed earlier, install devDependencies here (except for mongodb!)
-npm install --ignore-scripts
-
-# copy mongodb into CSFLE's node_modules
-rm -rf node_modules/mongodb
-cp -R "$ABS_PATH_TO_PATCH" node_modules/mongodb
-pushd node_modules/mongodb
-# lets be sure we have compiled TS since driver tests don't need to compile
-npm run build:ts
-popd # node_modules/mongodb
-
-# this variable needs to be empty
-export MONGODB_NODE_SKIP_LIVE_TESTS=""
-# all of the below must be defined (as well as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
-export AWS_REGION="us-east-1"
-export AWS_CMK_ID="arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0"
-
-npm test -- --colors
-
-popd # ../csfle-deps-tmp/libmongocrypt/bindings/node
-
-# Exit the script in a way that will show evergreen a pass or fail
-if [ $DRIVER_CSFLE_TEST_RESULT -ne 0 ]; then
-  echo "Driver tests failed, look above for results"
-  exit 1
-fi

.evergreen/run-gcp-kms-tests.sh

@@ -9,7 +9,7 @@ source ".evergreen/init-node-and-npm-env.sh"
 
 set -o xtrace
 
-npm install mongodb-client-encryption
+npm install mongodb-client-encryption@alpha --force
 npm install gcp-metadata
 
 export MONGODB_URI="mongodb://localhost:27017"

.evergreen/run-serverless-tests.sh

@@ -10,7 +10,7 @@ if [ -z ${MONGODB_URI+omitted} ]; then echo "MONGODB_URI is unset" && exit 1; fi
 if [ -z ${SERVERLESS_ATLAS_USER+omitted} ]; then echo "SERVERLESS_ATLAS_USER is unset" && exit 1; fi
 if [ -z ${SERVERLESS_ATLAS_PASSWORD+omitted} ]; then echo "SERVERLESS_ATLAS_PASSWORD is unset" && exit 1; fi
 
-npm install mongodb-client-encryption
+npm install mongodb-client-encryption@alpha --force
 
 npx mocha \
   --config test/mocha_mongodb.json \

.evergreen/run-socks5-tests.sh

@@ -5,6 +5,27 @@ source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
 set -o errexit  # Exit the script with error if any of the commands fail
 set -o xtrace  # For debuggability, no external credentials are used here
 
+function setup_fle() {
+  export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+  export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+  export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS}
+  export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
+  echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH"
+
+  set -o xtrace   # Write all commands first to stderr
+  set -o errexit  # Exit the script with error if any of the commands fail
+
+  # Get access to the AWS temporary credentials:
+  echo "adding temporary AWS credentials to environment"
+  # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
+  . "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
+
+  npm i --force mongodb-client-encryption@alpha
+  export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
+  export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
+  export TEST_CSFLE=true
+}
+
 node -v
 
 PYTHON_BINARY=${PYTHON_BINARY:-python3}
@@ -32,9 +53,10 @@ fi
 "$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "127.0.0.1:12345 to $FIRST_HOST" &
 SOCKS5_PROXY_PID=$!
 if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
+  setup_fle
   [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
   env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyUsername=username&proxyPassword=p4ssw0rd' \
-  bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
+  npm run check:csfle
 else
   env SOCKS5_CONFIG='["127.0.0.1",1080,"username","p4ssw0rd"]' npm run check:socks5
 fi
@@ -44,9 +66,10 @@ kill $SOCKS5_PROXY_PID
 "$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "127.0.0.1:12345 to $FIRST_HOST" &
 SOCKS5_PROXY_PID=$!
 if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
+  setup_fle
   [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
     env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyPort=1081' \
-    bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
+    npm run check:csfle
 else
   env SOCKS5_CONFIG='["127.0.0.1",1081]' npm run check:socks5
 fi

.evergreen/run-tests.sh

@@ -52,7 +52,7 @@ else
   source "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
 fi
 
-npm install mongodb-client-encryption
+npm install mongodb-client-encryption@alpha --force
 npm install @mongodb-js/zstd
 npm install snappy
 

.evergreen/run-unit-tests.sh

@@ -4,4 +4,6 @@ set -o errexit # Exit the script with error if any of the commands fail
 source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
 set -o xtrace
 
+npm i --force mongodb-client-encryption@alpha
+
 npx nyc npm run check:unit

.gitignore

@@ -95,7 +95,3 @@ node-artifacts
 # AWS SAM generated
 test/lambda/.aws-sam
 test/lambda/env.json
-
-!encryption/lib
-!encryption/test
-!encryption/test/types