feat: add support for QE range indexes MONGOSH-1371

config/eslintrc.base.js

@@ -38,7 +38,8 @@ module.exports = {
     'no-console': [1, { allow: ['warn', 'error', 'info'] }],
     'no-shadow': 0,
     'no-use-before-define': 0,
-    'no-cond-assign': [2, 'except-parens']
+    'no-cond-assign': [2, 'except-parens'],
+    'no-multi-str': 0
   },
   overrides: [{
     files: ['**/*.js'],

package.json

@@ -122,7 +122,7 @@
     "lerna": "^4.0.0",
     "mocha": "^7.1.2",
     "mongodb": "^4.13.0",
-    "mongodb-download-url": "^1.2.2",
+    "mongodb-download-url": "^1.3.0",
     "mongodb-js-precommit": "^2.0.0",
     "nock": "^13.0.11",
     "node-codesign": "^0.3.3",

packages/build/src/packaging/download-crypt-library.ts

@@ -13,8 +13,11 @@ export async function downloadCryptLibrary(variant: PackageVariant | 'host'): Pr
   console.info('mongosh: downloading latest crypt shared library for inclusion in package:', JSON.stringify(opts));
 
   const cryptTmpTargetDir = path.resolve(__dirname, '..', '..', '..', '..', 'tmp', 'crypt-store', variant);
-  // Download mongodb for latest server version.
-  const libdir = await downloadMongoDb(cryptTmpTargetDir, 'stable', opts);
+  // Download mongodb for latest server version, including rapid releases
+  // (for the platforms that they exist for, i.e. for ppc64le/s390x only pick stable releases).
+  const versionSpec = (opts.arch || process.arch).match(/ppc64|s390x/)
+    ? 'stable' : 'continuous';
+  const libdir = await downloadMongoDb(cryptTmpTargetDir, versionSpec, opts);
   const cryptLibrary = path.join(
     libdir,
     (await fs.readdir(libdir)).find(filename => filename.match(/^mongo_crypt_v1\.(so|dylib|dll)$/)) as string

packages/cli-repl/test/e2e-fle.spec.ts

@@ -15,6 +15,13 @@ import { inspect } from 'util';
 import path from 'path';
 import os from 'os';
 
+function isMacosTooOldForQE() {
+  // Indexed search is not supported on macOS 10.14 (which in turn is
+  // not supported by 6.0+ servers anyway).
+  // See e.g. https://jira.mongodb.org/browse/MONGOCRYPT-440
+  return os.type() === 'Darwin' && +os.release().split('.')[0] < 20;
+}
+
 describe('FLE tests', () => {
   const testServer = startTestServer('not-shared', '--replicaset', '--nodes', '1');
   skipIfServerVersion(testServer, '< 4.2'); // FLE only available on 4.2+
@@ -293,13 +300,10 @@ describe('FLE tests', () => {
   });
 
   context('6.0+', () => {
-    skipIfServerVersion(testServer, '< 6.0'); // FLE2 only available on 6.0+
+    skipIfServerVersion(testServer, '< 6.0'); // Queryable Encryption only available on 6.0+
 
     it('allows explicit encryption with bypassQueryAnalysis', async function() {
-      if (os.type() === 'Darwin' && +os.release().split('.')[0] < 20) {
-        // Indexed search is not supported on macOS 10.14 (which in turn is
-        // not supported by 6.0+ servers anyway).
-        // See e.g. https://jira.mongodb.org/browse/MONGOCRYPT-440
+      if (isMacosTooOldForQE()) {
         return this.skip();
       }
 
@@ -322,10 +326,8 @@ describe('FLE tests', () => {
         // Create necessary data key
         dataKey = keyVault.createKey('local');
 
-        // (re-)create collection -- this needs to be done
-        // with the plain mongo client until MONGOCRYPT-435 is done
         coll = client.getDB('${dbname}').encryptiontest;
-        Mongo(${uri}).getDB('${dbname}').createCollection('encryptiontest', {
+        client.getDB('${dbname}').createCollection('encryptiontest', {
           encryptedFields: {
             fields: [{
               keyId: dataKey,
@@ -465,6 +467,145 @@ describe('FLE tests', () => {
     });
   });
 
+  context('6.2+', () => {
+    skipIfServerVersion(testServer, '< 6.2'); // Range QE only available on 6.2+
+
+    it('allows explicit range encryption with bypassQueryAnalysis', async function() {
+      if (isMacosTooOldForQE()) {
+        return this.skip();
+      }
+
+      // No --cryptSharedLibPath since bypassQueryAnalysis is also a community edition feature
+      const shell = TestShell.start({ args: ['--nodb'] });
+      const uri = JSON.stringify(await testServer.connectionString());
+
+      await shell.waitForPrompt();
+
+      await shell.executeLine(`{
+        client = Mongo(${uri}, {
+          keyVaultNamespace: '${dbname}.keyVault',
+          kmsProviders: { local: { key: 'A'.repeat(128) } },
+          bypassQueryAnalysis: true
+        });
+
+        keyVault = client.getKeyVault();
+        clientEncryption = client.getClientEncryption();
+
+        // Create necessary data key
+        dataKey = keyVault.createKey('local');
+
+        rangeOptions = {
+          sparsity: Long(1),
+          min: new Date('1970'),
+          max: new Date('2100')
+        };
+        coll = client.getDB('${dbname}').encryptiontest;
+        client.getDB('${dbname}').createCollection('encryptiontest', {
+          encryptedFields: {
+            fields: [{
+              keyId: dataKey,
+              path: 'v',
+              bsonType: 'date',
+              queries: [{
+                queryType: 'rangePreview',
+                contention: 4,
+                ...rangeOptions
+              }]
+            }]
+          }
+        });
+
+        // Encrypt and insert data encrypted with specified data key
+        for (let year = 1990; year < 2010; year++) {
+          const insertPayload = clientEncryption.encrypt(
+            dataKey,
+            new Date(year + '-02-02T12:45:16.277Z'),
+            {
+              algorithm: 'RangePreview',
+              contentionFactor: 4,
+              rangeOptions
+            });
+          coll.insertOne({ v: insertPayload, year });
+        }
+      }`
+      );
+      expect(await shell.executeLine('({ count: coll.countDocuments() })')).to.include('{ count: 20 }');
+
+      await shell.executeLine(`
+      const findPayload = clientEncryption.encryptExpression(dataKey, {
+        $and: [ { v: {$gt: new Date('1992')} }, { v: {$lt: new Date('1999')} } ]
+      }, {
+        algorithm: 'RangePreview',
+        queryType: 'rangePreview',
+        contentionFactor: 4,
+        rangeOptions
+      });
+      `);
+
+      // Make sure the find payload allows searching for the encrypted values
+      const out = await shell.executeLine('\
+        coll.find(findPayload) \
+          .toArray() \
+          .map(d => d.year) \
+          .sort() \
+          .join(\',\')');
+      expect(out).to.include('1992,1993,1994,1995,1996,1997,1998');
+    });
+
+    it('allows automatic range encryption', async function() {
+      if (isMacosTooOldForQE()) {
+        return this.skip();
+      }
+
+      const shell = TestShell.start({ args: ['--nodb', `--cryptSharedLibPath=${cryptLibrary}`] });
+      const uri = JSON.stringify(await testServer.connectionString());
+
+      await shell.waitForPrompt();
+
+      await shell.executeLine(`{
+        client = Mongo(${uri}, {
+          keyVaultNamespace: '${dbname}.keyVault',
+          kmsProviders: { local: { key: 'A'.repeat(128) } }
+        });
+
+        dataKey = client.getKeyVault().createKey('local');
+
+        coll = client.getDB('${dbname}').encryptiontest;
+        client.getDB('${dbname}').createCollection('encryptiontest', {
+          encryptedFields: {
+            fields: [{
+              keyId: dataKey,
+              path: 'v',
+              bsonType: 'date',
+              queries: [{
+                queryType: 'rangePreview',
+                contention: 4,
+                sparsity: 1,
+                min: new Date('1970'),
+                max: new Date('2100')
+              }]
+            }]
+          }
+        });
+
+        for (let year = 1990; year < 2010; year++) {
+          coll.insertOne({ v: new Date(year + '-02-02T12:45:16.277Z'), year })
+        }
+      }`
+      );
+      expect(await shell.executeLine('({ count: coll.countDocuments() })')).to.include('{ count: 20 }');
+
+      // Make sure the find payload allows searching for the encrypted values
+      const out = await shell.executeLine('\
+        coll.find({ v: {$gt: new Date(\'1992\'), $lt: new Date(\'1999\') } }) \
+          .toArray() \
+          .map(d => d.year) \
+          .sort() \
+          .join(\',\')');
+      expect(out).to.include('1992,1993,1994,1995,1996,1997,1998');
+    });
+  });
+
   context('pre-6.0', () => {
     skipIfServerVersion(testServer, '>= 6.0'); // FLE2 available on 6.0+
 

packages/i18n/src/locales/en_US.ts

@@ -2195,6 +2195,10 @@ const translations: Catalog = {
               link: 'https://docs.mongodb.com/manual/reference/method/ClientEncryption.encrypt/#ClientEncryption.encrypt',
               description: 'Encrypts the value using the specified encryptionKeyId and encryptionAlgorithm. encrypt supports explicit (manual) encryption of field values.'
             },
+            encryptExpression: {
+              link: 'https://docs.mongodb.com/manual/reference/method/ClientEncryption.encrypt/#ClientEncryption.encryptExpression',
+              description: 'Encrypts an MQL expression using the specified encryptionKeyId and encryptionAlgorithm.'
+            },
             decrypt: {
               link: 'https://docs.mongodb.com/manual/reference/method/ClientEncryption.decrypt/#ClientEncryption.decrypt',
               description: 'decrypts the encryptionValue if the current database connection was configured with access to the Key Management Service (KMS) and key vault used to encrypt encryptionValue.'

packages/shell-api/src/field-level-encryption.spec.ts

@@ -204,6 +204,41 @@ describe('Field Level Encryption', () => {
         expect(caughtError).to.equal(expectedError);
       });
     });
+    describe('encryptExpression', () => {
+      const expression = {
+        $and: [{ someField: { $gt: 1 } }]
+      };
+
+      const options = {
+        algorithm: 'RangePreview',
+        queryType: 'rangePreview',
+        contentionFactor: 0,
+        rangeOptions: {
+          sparsity: new bson.Long(1)
+        }
+      } as const;
+
+      it('calls encryptExpression with algorithm on libmongoc', async() => {
+        libmongoc.encryptExpression.resolves();
+        await clientEncryption.encryptExpression(KEY_ID, expression, options);
+        expect(libmongoc.encryptExpression).calledOnceWithExactly(expression, { keyId: KEY_ID, ...options });
+      });
+      it('calls encryptExpression with algorithm, contentionFactor, and queryType on libmongoc', async() => {
+        const expression = {
+          $and: [{ someField: { $gt: 1 } }]
+        };
+        libmongoc.encryptExpression.resolves();
+        await clientEncryption.encryptExpression(KEY_ID, expression, options);
+        expect(libmongoc.encryptExpression).calledOnceWithExactly(expression, { keyId: KEY_ID, ...options });
+      });
+      it('throw if failed', async() => {
+        const expectedError = new Error();
+        libmongoc.encryptExpression.rejects(expectedError);
+        const caughtError = await clientEncryption.encryptExpression(KEY_ID, expression, options)
+          .catch(e => e);
+        expect(caughtError).to.equal(expectedError);
+      });
+    });
     describe('createKey', () => {
       it('calls createDataKey on libmongoc with no key for local', async() => {
         const raw = { result: 1 };

packages/shell-api/src/field-level-encryption.ts

@@ -168,23 +168,23 @@ export class ClientEncryption extends ShellApiWithMongoClass {
 
   @returnsPromise
   async encrypt(
-    encryptionId: BinaryType,
+    keyId: BinaryType,
     value: any,
     algorithmOrEncryptionOptions: ClientEncryptionEncryptOptions['algorithm'] | ClientEncryptionEncryptOptions
   ): Promise<BinaryType> {
     let encryptionOptions: ClientEncryptionEncryptOptions;
     if (typeof algorithmOrEncryptionOptions === 'object') {
       encryptionOptions = {
-        keyId: encryptionId,
+        keyId,
         ...algorithmOrEncryptionOptions
       };
     } else {
       encryptionOptions = {
-        keyId: encryptionId,
+        keyId,
         algorithm: algorithmOrEncryptionOptions
       };
     }
-    assertArgsDefinedType([encryptionId, value, encryptionOptions], [true, true, true], 'ClientEncryption.encrypt');
+    assertArgsDefinedType([keyId, value, encryptionOptions], [true, true, true], 'ClientEncryption.encrypt');
     return await this._libmongocrypt.encrypt(
       value,
       encryptionOptions
@@ -198,6 +198,20 @@ export class ClientEncryption extends ShellApiWithMongoClass {
     assertArgsDefinedType([encryptedValue], [true], 'ClientEncryption.decrypt');
     return await this._libmongocrypt.decrypt(encryptedValue);
   }
+
+  @returnsPromise
+  async encryptExpression(
+    keyId: BinaryType,
+    value: Document,
+    options: ClientEncryptionEncryptOptions
+  ) {
+    assertArgsDefinedType([keyId, value, options], [true, true, true], 'ClientEncryption.encryptExpression');
+    return await this._libmongocrypt.encryptExpression(
+      value,
+      // @ts-expect-error libmongocrypt typings contain a typo, see NODE-5023
+      { keyId, ...options }
+    );
+  }
 }
 
 @shellApiClassDefault