Sm/zip reads readables earlier (#683)

* fix: close open readables sooner * perf: stats from zip writer fix :) * test: skip tests to see windows perf/nuts * chore: auto-update metadata coverage in METADATA_SUPPORT.md * test: record perf [ci skip] * fix: folder types don't zip immediately (thanks, nuts!) * test: skip ut (temporary) * chore: auto-update metadata coverage in METADATA_SUPPORT.md * test: record perf [ci skip] * test: chcnage mocks to keep tests from hanging * test: record perf [ci skip] * ci: don't try to record windows perf * test: record perf [ci skip] * chore: bump jsforce,core * chore: dedupe lockfile * test: record perf [ci skip] * Update src/convert/streams.ts Co-authored-by: Rodrigo Espinosa de los Monteros <[email protected]> * Update .circleci/config.yml Co-authored-by: Rodrigo Espinosa de los Monteros <[email protected]> * ci: fix job naming * test: record perf [ci skip] Co-authored-by: Ubuntu <[email protected]> Co-authored-by: Rodrigo Espinosa de los Monteros <[email protected]>
forcedotcom · Aug 11, 2022 · 602b78d · 602b78d
1 parent ca35eed
commit 602b78d
Show file tree

Hide file tree

Showing 12 changed files with 120 additions and 196 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -172,18 +172,22 @@ workflows:
               - os: windows
                 node_version: maintenance
       - release-management/test-nut:
+          name: nuts-on-linux
           sfdx_version: latest
           node_version: lts
           size: large
           # git config so it can push without --set-upstream
           command: 'git config --global push.default current; git config --global user.email [email protected];  yarn test:nuts:scale:record'
-          matrix:
-            parameters:
-              os:
-                - linux
-                - windows
-          # required because this will commit to github
-          context: salesforce-cli
+          os: linux
+      # I wish we could *also* record the CI perf for windows, but I can't figure out the syntax
+      # until then, run as a separate job to test scaling for larger repos
+      - release-management/test-nut:
+          name: nuts-on-windows
+          sfdx_version: latest
+          node_version: lts
+          size: large
+          command: 'yarn test:nuts:scale'
+          os: windows
           filters:
             branches:
               ignore: main

diff --git a/METADATA_SUPPORT.md b/METADATA_SUPPORT.md
@@ -1,10 +1,10 @@
 # Supported CLI Metadata Types
 
-This list compares metadata types found in Salesforce v55 with the [metadata registry file](./src/registry/metadataRegistry.json) included in this repository.
+This list compares metadata types found in Salesforce v56 with the [metadata registry file](./src/registry/metadataRegistry.json) included in this repository.
 
 This repository is used by both the Salesforce CLIs and Salesforce's VSCode Extensions.
 
-Currently, there are 462/491 supported metadata types.
+Currently, there are 465/510 supported metadata types.
 For status on any existing gaps, please search or file an issue in the [Salesforce CLI issues only repo](https://github.com/forcedotcom/cli/issues).
 To contribute a new metadata type, please see the [Contributing Metadata Types to the Registry](./contributing/metadata.md)
 
@@ -13,11 +13,15 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |AIApplication|✅||
 |AIApplicationConfig|✅||
 |AIReplyRecommendationsSettings|✅||
+|AIUsecaseDefinition|⚠️|Supports deploy/retrieve but not source tracking|
 |AccountForecastSettings|✅||
 |AccountInsightsSettings|✅||
 |AccountIntelligenceSettings|✅||
 |AccountRelationshipShareRule|✅||
 |AccountSettings|✅||
+|AccountingFieldMapping|❌|Not supported, but support could be added|
+|AccountingModelConfig|❌|Not supported, but support could be added|
+|AccountingSettings|✅||
 |AcctMgrTargetSettings|✅||
 |ActionLinkGroupTemplate|✅||
 |ActionPlanTemplate|✅||
@@ -29,7 +33,6 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |AdvAcctForecastDimSource|✅||
 |AdvAcctForecastPeriodGroup|✅||
 |AnalyticSnapshot|✅||
-|AnalyticsDataServicesSettings|✅||
 |AnalyticsSettings|✅||
 |AnimationRule|✅||
 |ApexClass|✅||
@@ -70,6 +73,8 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |BldgEnrgyIntensityCnfg|✅||
 |BlockchainSettings|✅||
 |Bot|✅||
+|BotBlock|❌|Not supported, but support could be added|
+|BotBlockVersion|❌|Not supported, but support could be added|
 |BotSettings|✅||
 |BotTemplate|✅||
 |BotVersion|✅||
@@ -103,6 +108,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |ChatterExtension|✅||
 |ChatterSettings|✅||
 |CleanDataService|✅||
+|CollectionsDashboardSettings|✅||
 |CommandAction|✅||
 |CommerceSettings|✅||
 |CommunitiesSettings|✅||
@@ -113,7 +119,6 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |CompanySettings|✅||
 |ConnectedApp|✅||
 |ConnectedAppSettings|✅||
-|ConnectedSystem|✅||
 |ContentAsset|✅||
 |ContentSettings|✅||
 |ContractSettings|✅||
@@ -141,28 +146,33 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |CustomTab|✅||
 |CustomValue|❌|Not supported, but support could be added|
 |CustomerDataPlatformSettings|✅||
+|CustomizablePropensityScoringSettings|✅||
 |Dashboard|✅||
 |DashboardFolder|✅||
 |DataCategoryGroup|✅||
 |DataConnectorIngestApi|✅||
 |DataConnectorS3|✅||
 |DataDotComSettings|✅||
 |DataImportManagementSettings|✅||
-|DataMapping|✅||
-|DataMappingFieldDefinition|✅||
-|DataMappingObjectDefinition|✅||
-|DataMappingSchema|✅||
+|DataPackageKitDefinition|✅||
+|DataPackageKitObject|✅||
 |DataSource|✅||
+|DataSourceBundleDefinition|✅||
 |DataSourceObject|✅||
 |DataSourceTenant|✅||
+|DataSrcDataModelFieldMap|✅||
 |DataStreamDefinition|✅||
+|DataStreamTemplate|✅||
 |DecisionMatrixDefinition|✅||
 |DecisionMatrixDefinitionVersion|✅||
 |DecisionTable|✅||
 |DecisionTableDatasetLink|✅||
 |DelegateGroup|✅||
 |DeploymentSettings|✅||
 |DevHubSettings|✅||
+|DigitalExperience|❌|Not supported, but support could be added (but not for tracking)|
+|DigitalExperienceBundle|❌|Not supported, but support could be added (but not for tracking)|
+|DigitalExperienceConfig|❌|Not supported, but support could be added (but not for tracking)|
 |DiscoveryAIModel|✅||
 |DiscoveryGoal|✅||
 |DiscoverySettings|✅||
@@ -208,8 +218,10 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |ExperienceBundleSettings|✅||
 |ExplainabilityActionDefinition|✅||
 |ExplainabilityActionVersion|✅||
+|ExplainabilityMsgTemplate|❌|Not supported, but support could be added|
 |ExpressionSetDefinition|✅||
 |ExpressionSetDefinitionVersion|✅||
+|ExpressionSetObjectAlias|❌|Not supported, but support could be added|
 |ExternalAIModel|❌|Not supported, but support could be added|
 |ExternalCredential|❌|Not supported, but support could be added|
 |ExternalDataConnector|✅||
@@ -218,11 +230,9 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |ExternalDataTranField|❌|Not supported, but support could be added|
 |ExternalDataTranObject|❌|Not supported, but support could be added|
 |ExternalServiceRegistration|✅||
-|ExternalServicesSettings|✅||
 |FeatureParameterBoolean|✅||
 |FeatureParameterDate|✅||
 |FeatureParameterInteger|✅||
-|FederationDataMappingUsage|✅||
 |FieldRestrictionRule|✅||
 |FieldServiceMobileExtension|✅||
 |FieldServiceSettings|✅||
@@ -244,6 +254,8 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |ForecastingType|✅||
 |ForecastingTypeSource|✅||
 |FormulaSettings|✅||
+|FuelType|❌|Not supported, but support could be added|
+|FuelTypeSustnUom|❌|Not supported, but support could be added|
 |FunctionReference|⚠️|Supports deploy/retrieve but not source tracking|
 |GatewayProviderPaymentMethodType|✅||
 |GlobalValueSet|✅||
@@ -261,6 +273,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |InboundCertificate|✅||
 |InboundNetworkConnection|✅||
 |IncidentMgmtSettings|✅||
+|IncludeEstTaxInQuoteSettings|✅||
 |Index|⚠️|Supports deploy/retrieve but not source tracking|
 |IndustriesAutomotiveSettings|✅||
 |IndustriesLoyaltySettings|✅||
@@ -310,6 +323,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |MeetingsSettings|✅||
 |MessagingChannel|❌|Not supported, but support could be added (but not for tracking)|
 |MfgProgramTemplate|❌|Not supported, but support could be added|
+|MfgServiceConsoleSettings|✅||
 |MilestoneType|✅||
 |MktCalcInsightObjectDef|✅||
 |MktDataTranObject|✅||
@@ -318,7 +332,6 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |MobileApplicationDetail|✅||
 |MobileSecurityAssignment|✅||
 |MobileSecurityPolicy|✅||
-|MobileSecurityPolicySet|✅||
 |MobileSettings|✅||
 |ModerationRule|✅||
 |MutingPermissionSet|✅||
@@ -332,6 +345,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |NotificationTypeConfig|✅||
 |NotificationsSettings|✅||
 |OauthCustomScope|✅||
+|OauthOidcSettings|✅||
 |ObjectHierarchyRelationship|✅||
 |ObjectLinkingSettings|✅||
 |ObjectSourceTargetMap|✅||
@@ -370,6 +384,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |PlatformEventChannelMember|✅||
 |PlatformEventSubscriberConfig|✅||
 |PlatformSlackSettings|✅||
+|PortalDelegablePermissionSet|❌|Not supported, but support could be added|
 |PortalsSettings|✅||
 |PostTemplate|✅||
 |PredictionBuilderSettings|✅||
@@ -403,6 +418,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |Report|✅||
 |ReportFolder|✅||
 |ReportType|✅||
+|ReportingTypeConfig|❌|Not supported, but support could be added|
 |RestrictionRule|✅||
 |RetailExecutionSettings|✅||
 |Role|✅||
@@ -444,6 +460,8 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |StreamingAppDataConnector|❌|Not supported, but support could be added|
 |SubscriptionManagementSettings|✅||
 |SurveySettings|✅||
+|SustainabilityUom|❌|Not supported, but support could be added|
+|SustnUomConversion|❌|Not supported, but support could be added|
 |SvcCatalogCategory|✅||
 |SvcCatalogFulfillmentFlow|✅||
 |SvcCatalogItemDef|✅||
@@ -464,6 +482,7 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 |TrialOrgSettings|✅||
 |UIObjectRelationConfig|✅||
 |UiPlugin|✅||
+|UserAccessPolicy|❌|Not supported, but support could be added|
 |UserAuthCertificate|✅||
 |UserCriteria|✅||
 |UserEngagementSettings|✅||
@@ -504,39 +523,13 @@ To contribute a new metadata type, please see the [Contributing Metadata Types t
 
 
 
-## Next Release (v56)
-v56 introduces the following new types.  Here's their current level of support
+## Next Release (v57)
+v57 introduces the following new types.  Here's their current level of support
 
 |Metadata Type|Support|Notes|
 |:---|:---|:---|
-|AIUsecaseDefinition|⚠️|Supports deploy/retrieve but not source tracking|
-|AccountingFieldMapping|❌|Not supported, but support could be added|
-|AccountingModelConfig|❌|Not supported, but support could be added|
-|AccountingSettings|✅||
-|BotBlock|❌|Not supported, but support could be added|
-|BotBlockVersion|❌|Not supported, but support could be added|
-|CollectionsDashboardSettings|✅||
-|CustomizablePropensityScoringSettings|✅||
-|DataPackageKitDefinition|✅||
-|DataPackageKitObject|✅||
-|DataSourceBundleDefinition|✅||
-|DataSrcDataModelFieldMap|✅||
-|DataStreamTemplate|✅||
-|DigitalExperience|❌|Not supported, but support could be added (but not for tracking)|
-|DigitalExperienceBundle|❌|Not supported, but support could be added (but not for tracking)|
-|DigitalExperienceConfig|❌|Not supported, but support could be added (but not for tracking)|
-|ExplainabilityMsgTemplate|❌|Not supported, but support could be added|
-|ExpressionSetObjectAlias|❌|Not supported, but support could be added|
-|FuelType|❌|Not supported, but support could be added|
-|FuelTypeSustnUom|❌|Not supported, but support could be added|
-|IncludeEstTaxInQuoteSettings|✅||
-|MfgServiceConsoleSettings|✅||
-|OauthOidcSettings|✅||
-|PortalDelegablePermissionSet|❌|Not supported, but support could be added|
-|ReportingTypeConfig|❌|Not supported, but support could be added|
-|SustainabilityUom|❌|Not supported, but support could be added|
-|SustnUomConversion|❌|Not supported, but support could be added|
-|UserAccessPolicy|❌|Not supported, but support could be added|
+|ExternalServicesSettings|✅||
+|ReferencedDashboard|❌|Not supported, but support could be added|
 
 ## Additional Types
 
@@ -584,6 +577,13 @@ v56 introduces the following new types.  Here's their current level of support
 - DynamicTrigger
 - MktDataTranField
 - ConversationVendorFieldDef
+- DataMappingSchema
+- DataMappingObjectDefinition
+- DataMappingFieldDefinition
+- DataMapping
+- FederationDataMappingUsage
+- ConnectedSystem
 - InternalOrganization
 - UiViewDefinition
+- MobileSecurityPolicySet
 - DataWeaveResource
diff --git a/package.json b/package.json
@@ -25,7 +25,7 @@
     "node": ">=14.0.0"
   },
   "dependencies": {
-    "@salesforce/core": "^3.21.1",
+    "@salesforce/core": "^3.26.1",
     "@salesforce/kit": "^1.5.41",
     "@salesforce/ts-types": "^1.5.20",
     "archiver": "^5.3.0",
@@ -69,7 +69,7 @@
     "eslint-plugin-jsdoc": "^35.1.3",
     "eslint-plugin-prettier": "^4.0.0",
     "husky": "^7.0.4",
-    "jsforce": "2.0.0-beta.10",
+    "jsforce": "2.0.0-beta.18",
     "lint-staged": "^10.2.11",
     "mocha": "^9.1.3",
     "mocha-junit-reporter": "^1.23.3",
@@ -79,7 +79,7 @@
     "shelljs": "0.8.5",
     "shx": "^0.3.2",
     "sinon": "10.0.0",
-    "ts-node": "^10.8.1",
+    "ts-node": "^10.9.1",
     "typescript": "^4.1.3"
   },
   "scripts": {

diff --git a/src/convert/streams.ts b/src/convert/streams.ts
@@ -224,14 +224,22 @@ export class ZipWriter extends ComponentWriter {
     void pipeline(this.zip, this.getOutputStream());
   }
 
-  // required to be async to override Node's Writable class
-  // eslint-disable-next-line @typescript-eslint/require-await
   public async _write(chunk: WriterFormat, encoding: string, callback: (err?: Error) => void): Promise<void> {
     let err: Error;
     try {
-      for (const info of chunk.writeInfos) {
-        this.addToZip(info.source, info.output);
-      }
+      await Promise.all(
+        chunk.writeInfos.map(async (writeInfo) =>
+          this.addToZip(
+            chunk.component.type.folderType ?? chunk.component.type.folderContentType
+              ? // we don't want to prematurely zip folder types when their children might still be not in the zip
+                // those files we'll leave held open as Readable until finalize
+                writeInfo.source
+              : // everything else can be zipped immediately
+                await stream2buffer(writeInfo.source),
+            writeInfo.output
+          )
+        )
+      );
     } catch (e) {
       err = e as Error;
     }

diff --git a/test/convert/streams.test.ts b/test/convert/streams.test.ts
@@ -448,14 +448,23 @@ describe('Streams', () => {
       it('should add entries to zip based on given write infos', async () => {
         writer = new streams.ZipWriter(`${rootDestination}.zip`);
         const appendStub = env.stub(archive, 'append');
+        env.stub(streams, 'stream2buffer').resolves(Buffer.from('hi'));
+
+        await writer._write(chunk, '', (err: Error) => {
+          expect(err).to.be.undefined;
+        });
+        expect(appendStub.firstCall.args[0].toString()).to.equal('hi');
+      });
+
+      it('should add entries to zip based on given write infos when zip is in-memory only', async () => {
+        writer = new streams.ZipWriter();
+        const appendStub = env.stub(archive, 'append');
+        env.stub(streams, 'stream2buffer').resolves(Buffer.from('hi'));
 
         await writer._write(chunk, '', (err: Error) => {
           expect(err).to.be.undefined;
-          expect(appendStub.firstCall.args).to.deep.equal([
-            chunk.writeInfos[0].source,
-            { name: chunk.writeInfos[0].output },
-          ]);
         });
+        expect(appendStub.firstCall.args[0].toString()).to.equal('hi');
       });
 
       it('should finalize zip when stream is finished', async () => {
@@ -479,6 +488,7 @@ describe('Streams', () => {
       it('should pass errors to _write callback', async () => {
         const whoops = new Error('whoops!');
         env.stub(archive, 'append').throws(whoops);
+        env.stub(streams, 'stream2buffer').resolves(Buffer.from('hi'));
 
         await writer._write(chunk, '', (err: Error) => {
           expect(err.message).to.equal(whoops.message);

diff --git a/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json b/test/nuts/perfResults/x64-darwin-16xIntel-Core-i9-9980HK-CPU--2-40GHz/eda.json
@@ -1,18 +1,18 @@
 [
   {
     "name": "componentSetCreate",
-    "duration": 503.34397998452187
+    "duration": 492.98764300346375
   },
   {
     "name": "sourceToMdapi",
-    "duration": 5195.863053023815
+    "duration": 5277.533275008202
   },
   {
     "name": "sourceToZip",
-    "duration": 4191.627218991518
+    "duration": 3976.5643639862537
   },
   {
     "name": "mdapiToSource",
-    "duration": 7129.984247982502
+    "duration": 7064.533327996731
   }
 ]