diff --git a/packages/compass-import-export/src/export/export-csv.ts b/packages/compass-import-export/src/export/export-csv.ts index 296e0e5b1d3..ed1c8307fce 100644 --- a/packages/compass-import-export/src/export/export-csv.ts +++ b/packages/compass-import-export/src/export/export-csv.ts @@ -7,7 +7,6 @@ import type { Readable, Writable } from 'stream'; import toNS from 'mongodb-ns'; import type { DataService } from 'mongodb-data-service'; import type { PreferencesAccess } from 'compass-preferences-model/provider'; -import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider'; import Parser from 'stream-json/Parser'; import StreamValues from 'stream-json/streamers/StreamValues'; import path from 'path'; @@ -29,6 +28,7 @@ import { formatCSVHeaderName } from '../csv/csv-utils'; import type { Delimiter, Linebreak, PathPart } from '../csv/csv-types'; import { createDebug } from '../utils/logger'; import type { AggregationCursor, FindCursor } from 'mongodb'; +import { createAggregationCursor, createFindCursor } from './export-cursor'; const debug = createDebug('export-csv'); @@ -274,18 +274,12 @@ export async function exportCSVFromAggregation({ }) { debug('exportCSVFromAggregation()', { ns: toNS(ns), aggregation }); - const { stages, options: aggregationOptions = {} } = aggregation; - aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit( - preferences, - aggregationOptions.maxTimeMS - ); - aggregationOptions.promoteValues = false; - aggregationOptions.bsonRegExp = true; - const aggregationCursor = dataService.aggregateCursor( + const aggregationCursor = createAggregationCursor({ ns, - stages, - aggregationOptions - ); + aggregation, + dataService, + preferences, + }); let filename, input, columns; try { @@ -335,14 +329,10 @@ export async function exportCSVFromQuery({ }) { debug('exportCSVFromQuery()', { ns: toNS(ns), query }); - const findCursor = dataService.findCursor(ns, query.filter ?? {}, { - projection: query.projection, - sort: query.sort, - limit: query.limit, - skip: query.skip, - collation: query.collation, - promoteValues: false, - bsonRegExp: true, + const findCursor = createFindCursor({ + ns, + query, + dataService, }); let filename, input, columns; diff --git a/packages/compass-import-export/src/export/export-cursor.ts b/packages/compass-import-export/src/export/export-cursor.ts new file mode 100644 index 00000000000..8981aca0113 --- /dev/null +++ b/packages/compass-import-export/src/export/export-cursor.ts @@ -0,0 +1,46 @@ +import type { DataService } from 'mongodb-data-service'; +import type { PreferencesAccess } from 'compass-preferences-model/provider'; +import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider'; + +import type { ExportAggregation, ExportQuery } from './export-types'; + +export function createAggregationCursor({ + ns, + aggregation, + dataService, + preferences, +}: { + ns: string; + dataService: Pick; + preferences: PreferencesAccess; + aggregation: ExportAggregation; +}) { + const { stages, options: aggregationOptions = {} } = aggregation; + aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit( + preferences, + aggregationOptions.maxTimeMS + ); + aggregationOptions.promoteValues = false; + aggregationOptions.bsonRegExp = true; + return dataService.aggregateCursor(ns, stages, aggregationOptions); +} + +export function createFindCursor({ + ns, + query, + dataService, +}: { + ns: string; + dataService: Pick; + query: ExportQuery; +}) { + return dataService.findCursor(ns, query.filter ?? {}, { + projection: query.projection, + sort: query.sort, + limit: query.limit, + skip: query.skip, + collation: query.collation, + promoteValues: false, + bsonRegExp: true, + }); +} diff --git a/packages/compass-import-export/src/export/export-json.ts b/packages/compass-import-export/src/export/export-json.ts index cadc9d49959..d91757209b7 100644 --- a/packages/compass-import-export/src/export/export-json.ts +++ b/packages/compass-import-export/src/export/export-json.ts @@ -5,7 +5,6 @@ import type { Writable } from 'stream'; import toNS from 'mongodb-ns'; import type { DataService } from 'mongodb-data-service'; import type { PreferencesAccess } from 'compass-preferences-model/provider'; -import { capMaxTimeMSAtPreferenceLimit } from 'compass-preferences-model/provider'; import type { AggregationCursor, FindCursor } from 'mongodb'; import { objectToIdiomaticEJSON } from 'hadron-document'; @@ -15,6 +14,7 @@ import type { ExportResult, } from './export-types'; import { createDebug } from '../utils/logger'; +import { createAggregationCursor, createFindCursor } from './export-cursor'; const debug = createDebug('export-json'); @@ -126,16 +126,12 @@ export async function exportJSONFromAggregation({ }) { debug('exportJSONFromAggregation()', { ns: toNS(ns), aggregation }); - const { stages, options: aggregationOptions = {} } = aggregation; - aggregationOptions.maxTimeMS = capMaxTimeMSAtPreferenceLimit( - preferences, - aggregationOptions.maxTimeMS - ); - const aggregationCursor = dataService.aggregateCursor( + const aggregationCursor = createAggregationCursor({ ns, - stages, - aggregationOptions - ); + aggregation, + dataService, + preferences, + }); return await exportJSON({ ...exportOptions, @@ -155,12 +151,10 @@ export async function exportJSONFromQuery({ }) { debug('exportJSONFromQuery()', { ns: toNS(ns), query }); - const findCursor = dataService.findCursor(ns, query.filter ?? {}, { - projection: query.projection, - sort: query.sort, - limit: query.limit, - skip: query.skip, - collation: query.collation, + const findCursor = createFindCursor({ + ns, + query, + dataService, }); return await exportJSON({ diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson index 855094ded6a..e2b531cbcf2 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson +++ b/packages/compass-import-export/test/docs/all-bson-types.exported.canonical.ejson @@ -5,6 +5,9 @@ "double": { "$numberDouble": "1.2" }, + "doubleThatIsAlsoAnInteger": { + "$numberDouble": "1.0" + }, "string": "Hello, world!", "object": { "key": "value" @@ -45,7 +48,9 @@ "javascript": { "$code": "function() {}" }, - "symbol": "symbol", + "symbol": { + "$symbol": "symbol" + }, "javascriptWithScope": { "$code": "function() {}", "$scope": { diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.csv b/packages/compass-import-export/test/docs/all-bson-types.exported.csv index f3f44b790d7..93cdceb26f3 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.exported.csv +++ b/packages/compass-import-export/test/docs/all-bson-types.exported.csv @@ -1,2 +1,2 @@ -_id,double,string,object.key,array[0],array[1],array[2],binData,objectId,boolean,date,null,regex,javascript,symbol,javascriptWithScope,int,timestamp,long,decimal,minKey,maxKey,binaries.generic,binaries.functionData,binaries.binaryOld,binaries.uuidOld,binaries.uuid,binaries.md5,binaries.encrypted,binaries.compressedTimeSeries,binaries.custom,dbRef -1,1.2,"Hello, world!",value,1,2,3,AQID,642d766c7300158b1f22e975,true,2023-04-05T13:25:08.445Z,,/pattern/i,"{""$code"":""function() {}""}","{""$symbol"":""symbol""}","{""$code"":""function() {}"",""$scope"":{""foo"":{""$numberInt"":""1""},""bar"":""a""}}",12345,7218556297505931265,123456789123456784,5.477284286264328586719275128128001E-4088,$MinKey,$MaxKey,AQID,Ly84PQ==,Ly84PQ==,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==,Ly84PQ==,"{""$ref"":""namespace"",""$id"":{""$oid"":""642d76b4b7ebfab15d3c4a78""}}" +_id,double,doubleThatIsAlsoAnInteger,string,object.key,array[0],array[1],array[2],binData,objectId,boolean,date,null,regex,javascript,symbol,javascriptWithScope,int,timestamp,long,decimal,minKey,maxKey,binaries.generic,binaries.functionData,binaries.binaryOld,binaries.uuidOld,binaries.uuid,binaries.md5,binaries.encrypted,binaries.compressedTimeSeries,binaries.custom,dbRef +1,1.2,1,"Hello, world!",value,1,2,3,AQID,642d766c7300158b1f22e975,true,2023-04-05T13:25:08.445Z,,/pattern/i,"{""$code"":""function() {}""}","{""$symbol"":""symbol""}","{""$code"":""function() {}"",""$scope"":{""foo"":{""$numberInt"":""1""},""bar"":""a""}}",12345,7218556297505931265,123456789123456784,5.477284286264328586719275128128001E-4088,$MinKey,$MaxKey,AQID,Ly84PQ==,Ly84PQ==,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,Yy8vU1pFU3pUR21RNk9mUjM4QTExQT09,CQCKW/8XjAEAAIfx//////////H/////////AQAAAAAAAABfAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAHAAAAAAAAAA4AAAAAAAAAAA==,Ly84PQ==,"{""$ref"":""namespace"",""$id"":{""$oid"":""642d76b4b7ebfab15d3c4a78""}}" diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson index de134967514..bfce47d06c8 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson +++ b/packages/compass-import-export/test/docs/all-bson-types.exported.default.ejson @@ -3,6 +3,7 @@ "$oid": "123456789012345678901234" }, "double": 1.2, + "doubleThatIsAlsoAnInteger": 1, "string": "Hello, world!", "object": { "key": "value" @@ -35,7 +36,9 @@ "javascript": { "$code": "function() {}" }, - "symbol": "symbol", + "symbol": { + "$symbol": "symbol" + }, "javascriptWithScope": { "$code": "function() {}", "$scope": { diff --git a/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson b/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson index f0e2c247b69..4fd9f3bf4d3 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson +++ b/packages/compass-import-export/test/docs/all-bson-types.exported.relaxed.ejson @@ -3,6 +3,7 @@ "$oid": "123456789012345678901234" }, "double": 1.2, + "doubleThatIsAlsoAnInteger": 1, "string": "Hello, world!", "object": { "key": "value" @@ -35,7 +36,9 @@ "javascript": { "$code": "function() {}" }, - "symbol": "symbol", + "symbol": { + "$symbol": "symbol" + }, "javascriptWithScope": { "$code": "function() {}", "$scope": { diff --git a/packages/compass-import-export/test/docs/all-bson-types.gathered.json b/packages/compass-import-export/test/docs/all-bson-types.gathered.json index 7b9592e4e28..eb1ea91b3a0 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.gathered.json +++ b/packages/compass-import-export/test/docs/all-bson-types.gathered.json @@ -18,6 +18,7 @@ ["dbRef"], ["decimal"], ["double"], + ["doubleThatIsAlsoAnInteger"], ["int"], ["javascript"], ["javascriptWithScope"], @@ -45,6 +46,7 @@ "dbRef": 1, "decimal": 1, "double": 1, + "doubleThatIsAlsoAnInteger": 1, "int": 1, "javascript": 1, "javascriptWithScope": 1, diff --git a/packages/compass-import-export/test/docs/all-bson-types.js b/packages/compass-import-export/test/docs/all-bson-types.js index d83e13eac89..dc95634b2f4 100644 --- a/packages/compass-import-export/test/docs/all-bson-types.js +++ b/packages/compass-import-export/test/docs/all-bson-types.js @@ -19,6 +19,7 @@ export default [ { _id: new ObjectId('642d766b7300158b1f22e972'), double: new Double(1.2), // Double, 1, double + doubleThatIsAlsoAnInteger: new Double(1), // Double, 1, double string: 'Hello, world!', // String, 2, string object: { key: 'value' }, // Object, 3, object array: [1, 2, 3], // Array, 4, array