fix(import-export): promoteValues: false for export json COMPASS-8099 (…

…#6052)
mongodb-js · Jul 22, 2024 · 48ad532 · 48ad532
1 parent 8ea741e
commit 48ad532
Show file tree

Hide file tree

Showing 9 changed files with 85 additions and 41 deletions.
diff --git a/packages/compass-import-export/src/export/export-csv.ts b/packages/compass-import-export/src/export/export-csv.ts
@@ -7,7 +7,6 @@ import type { Readable, Writable } from 'stream';
 import toNS from 'mongodb-ns';
 import type { DataService } from 'mongodb-data-service';
 import type { PreferencesAccess } from 'compass-preferences-model/provider';
-import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider';
 import Parser from 'stream-json/Parser';
 import StreamValues from 'stream-json/streamers/StreamValues';
 import path from 'path';
@@ -29,6 +28,7 @@ import { formatCSVHeaderName } from '../csv/csv-utils';
 import type { Delimiter, Linebreak, PathPart } from '../csv/csv-types';
 import { createDebug } from '../utils/logger';
 import type { AggregationCursor, FindCursor } from 'mongodb';
+import { createAggregationCursor, createFindCursor } from './export-cursor';
 
 const debug = createDebug('export-csv');
 
@@ -274,18 +274,12 @@ export async function exportCSVFromAggregation({
 }) {
   debug('exportCSVFromAggregation()', { ns: toNS(ns), aggregation });
 
-  const { stages, options: aggregationOptions = {} } = aggregation;
-  aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit(
-    preferences,
-    aggregationOptions.maxTimeMS
-  );
-  aggregationOptions.promoteValues = false;
-  aggregationOptions.bsonRegExp = true;
-  const aggregationCursor = dataService.aggregateCursor(
+  const aggregationCursor = createAggregationCursor({
     ns,
-    stages,
-    aggregationOptions
-  );
+    aggregation,
+    dataService,
+    preferences,
+  });
 
   let filename, input, columns;
   try {
@@ -335,14 +329,10 @@ export async function exportCSVFromQuery({
 }) {
   debug('exportCSVFromQuery()', { ns: toNS(ns), query });
 
-  const findCursor = dataService.findCursor(ns, query.filter ?? {}, {
-    projection: query.projection,
-    sort: query.sort,
-    limit: query.limit,
-    skip: query.skip,
-    collation: query.collation,
-    promoteValues: false,
-    bsonRegExp: true,
+  const findCursor = createFindCursor({
+    ns,
+    query,
+    dataService,
   });
 
   let filename, input, columns;

diff --git a/packages/compass-import-export/src/export/export-cursor.ts b/packages/compass-import-export/src/export/export-cursor.ts
@@ -0,0 +1,46 @@
+import type { DataService } from 'mongodb-data-service';
+import type { PreferencesAccess } from 'compass-preferences-model/provider';
+import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider';
+
+import type { ExportAggregation, ExportQuery } from './export-types';
+
+export function createAggregationCursor({
+  ns,
+  aggregation,
+  dataService,
+  preferences,
+}: {
+  ns: string;
+  dataService: Pick<DataService, 'aggregateCursor'>;
+  preferences: PreferencesAccess;
+  aggregation: ExportAggregation;
+}) {
+  const { stages, options: aggregationOptions = {} } = aggregation;
+  aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit(
+    preferences,
+    aggregationOptions.maxTimeMS
+  );
+  aggregationOptions.promoteValues = false;
+  aggregationOptions.bsonRegExp = true;
+  return dataService.aggregateCursor(ns, stages, aggregationOptions);
+}
+
+export function createFindCursor({
+  ns,
+  query,
+  dataService,
+}: {
+  ns: string;
+  dataService: Pick<DataService, 'findCursor'>;
+  query: ExportQuery;
+}) {
+  return dataService.findCursor(ns, query.filter ?? {}, {
+    projection: query.projection,
+    sort: query.sort,
+    limit: query.limit,
+    skip: query.skip,
+    collation: query.collation,
+    promoteValues: false,
+    bsonRegExp: true,
+  });
+}
diff --git a/packages/compass-import-export/src/export/export-json.ts b/packages/compass-import-export/src/export/export-json.ts
@@ -5,7 +5,6 @@ import type { Writable } from 'stream';
 import toNS from 'mongodb-ns';
 import type { DataService } from 'mongodb-data-service';
 import type { PreferencesAccess } from 'compass-preferences-model/provider';
-import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider';
 import type { AggregationCursor, FindCursor } from 'mongodb';
 import { objectToIdiomaticEJSON } from 'hadron-document';
 
@@ -15,6 +14,7 @@ import type {
   ExportResult,
 } from './export-types';
 import { createDebug } from '../utils/logger';
+import { createAggregationCursor, createFindCursor } from './export-cursor';
 
 const debug = createDebug('export-json');
 
@@ -126,16 +126,12 @@ export async function exportJSONFromAggregation({
 }) {
   debug('exportJSONFromAggregation()', { ns: toNS(ns), aggregation });
 
-  const { stages, options: aggregationOptions = {} } = aggregation;
-  aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit(
-    preferences,
-    aggregationOptions.maxTimeMS
-  );
-  const aggregationCursor = dataService.aggregateCursor(
+  const aggregationCursor = createAggregationCursor({
     ns,
-    stages,
-    aggregationOptions
-  );
+    aggregation,
+    dataService,
+    preferences,
+  });
 
   return await exportJSON({
     ...exportOptions,
@@ -155,12 +151,10 @@ export async function exportJSONFromQuery({
 }) {
   debug('exportJSONFromQuery()', { ns: toNS(ns), query });
 
-  const findCursor = dataService.findCursor(ns, query.filter ?? {}, {
-    projection: query.projection,
-    sort: query.sort,
-    limit: query.limit,
-    skip: query.skip,
-    collation: query.collation,
+  const findCursor = createFindCursor({
+    ns,
+    query,
+    dataService,
   });
 
   return await exportJSON({

diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson
@@ -5,6 +5,9 @@
   "double": {
     "$numberDouble": "1.2"
   },
+  "doubleThatIsAlsoAnInteger": {
+    "$numberDouble": "1.0"
+  },
   "string": "Hello, world!",
   "object": {
     "key": "value"
@@ -45,7 +48,9 @@
   "javascript": {
     "$code": "function() {}"
   },
-  "symbol": "symbol",
+  "symbol": {
+    "$symbol": "symbol"
+  },
   "javascriptWithScope": {
     "$code": "function() {}",
     "$scope": {

diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.csv b/packages/compass-import-export/test/docs/all-bson-types.exported.csv
@@ -1,2 +1,2 @@
-_id,double,string,object.key,array[0],array[1],array[2],binData,objectId,boolean,date,null,regex,javascript,symbol,javascriptWithScope,int,timestamp,long,decimal,minKey,maxKey,binaries.generic,binaries.functionData,binaries.binaryOld,binaries.uuidOld,binaries.uuid,binaries.md5,binaries.encrypted,binaries.compressedTimeSeries,binaries.custom,dbRef
-1,1.2,"Hello, world!",value,1,2,3,AQID,642d766c7300158b1f22e975,true,2023-04-05T13:25:08.445Z,,/pattern/i,"{""$code"":""function() {}""}","{""$symbol"":""symbol""}","{""$code"":""function() {}"",""$scope"":{""foo"":{""$numberInt"":""1""},""bar"":""a""}}",12345,7218556297505931265,123456789123456784,5.477284286264328586719275128128001E-4088,$MinKey,$MaxKey,AQID,Ly84PQ==,Ly84PQ==,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==,Ly84PQ==,"{""$ref"":""namespace"",""$id"":{""$oid"":""642d76b4b7ebfab15d3c4a78""}}"
+_id,double,doubleThatIsAlsoAnInteger,string,object.key,array[0],array[1],array[2],binData,objectId,boolean,date,null,regex,javascript,symbol,javascriptWithScope,int,timestamp,long,decimal,minKey,maxKey,binaries.generic,binaries.functionData,binaries.binaryOld,binaries.uuidOld,binaries.uuid,binaries.md5,binaries.encrypted,binaries.compressedTimeSeries,binaries.custom,dbRef
+1,1.2,1,"Hello, world!",value,1,2,3,AQID,642d766c7300158b1f22e975,true,2023-04-05T13:25:08.445Z,,/pattern/i,"{""$code"":""function() {}""}","{""$symbol"":""symbol""}","{""$code"":""function() {}"",""$scope"":{""foo"":{""$numberInt"":""1""},""bar"":""a""}}",12345,7218556297505931265,123456789123456784,5.477284286264328586719275128128001E-4088,$MinKey,$MaxKey,AQID,Ly84PQ==,Ly84PQ==,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==,Ly84PQ==,"{""$ref"":""namespace"",""$id"":{""$oid"":""642d76b4b7ebfab15d3c4a78""}}"
diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson
@@ -3,6 +3,7 @@
     "$oid": "123456789012345678901234"
   },
   "double": 1.2,
+  "doubleThatIsAlsoAnInteger": 1,
   "string": "Hello, world!",
   "object": {
     "key": "value"
@@ -35,7 +36,9 @@
   "javascript": {
     "$code": "function() {}"
   },
-  "symbol": "symbol",
+  "symbol": {
+    "$symbol": "symbol"
+  },
   "javascriptWithScope": {
     "$code": "function() {}",
     "$scope": {

diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson
@@ -3,6 +3,7 @@
     "$oid": "123456789012345678901234"
   },
   "double": 1.2,
+  "doubleThatIsAlsoAnInteger": 1,
   "string": "Hello, world!",
   "object": {
     "key": "value"
@@ -35,7 +36,9 @@
   "javascript": {
     "$code": "function() {}"
   },
-  "symbol": "symbol",
+  "symbol": {
+    "$symbol": "symbol"
+  },
   "javascriptWithScope": {
     "$code": "function() {}",
     "$scope": {

diff --git a/packages/compass-import-export/test/docs/all-bson-types.gathered.json b/packages/compass-import-export/test/docs/all-bson-types.gathered.json
@@ -18,6 +18,7 @@
     ["dbRef"],
     ["decimal"],
     ["double"],
+    ["doubleThatIsAlsoAnInteger"],
     ["int"],
     ["javascript"],
     ["javascriptWithScope"],
@@ -45,6 +46,7 @@
     "dbRef": 1,
     "decimal": 1,
     "double": 1,
+    "doubleThatIsAlsoAnInteger": 1,
     "int": 1,
     "javascript": 1,
     "javascriptWithScope": 1,

diff --git a/packages/compass-import-export/test/docs/all-bson-types.js b/packages/compass-import-export/test/docs/all-bson-types.js
@@ -19,6 +19,7 @@ export default [
   {
     _id: new ObjectId('642d766b7300158b1f22e972'),
     double: new Double(1.2), // Double, 1, double
+    doubleThatIsAlsoAnInteger: new Double(1), // Double, 1, double
     string: 'Hello, world!', // String, 2, string
     object: { key: 'value' }, // Object, 3, object
     array: [1, 2, 3], // Array, 4, array