Merge pull request #262 from amtrack/fix/parse-large-files-from-git

fix: parse large files from git
amtrack · Jul 28, 2020 · 8d4fbc6 · 8d4fbc6
2 parents 5affd0c + 85ec228
commit 8d4fbc6
Show file tree

Hide file tree

Showing 20 changed files with 240,136 additions and 40 deletions.
diff --git a/lib/diff.js b/lib/diff.js
@@ -40,10 +40,10 @@ Diff.prototype.getMetadataContainers = function(opts) {
 				// retrieve file using git
 				var parts = file.index[0].split('..');
 				if (!file.new) {
-					fileFrom.contents = Buffer.from(self.git.show(parts[0]));
+					fileFrom.contents = self.git.show(parts[0]);
 				}
 				if (!file.deleted) {
-					fileTo.contents = Buffer.from(self.git.show(parts[1]));
+					fileTo.contents = self.git.show(parts[1]);
 				}
 			}
 			containerFrom.add(fileFrom, []);
@@ -86,10 +86,10 @@ Diff.stream = function(opts) {
 					// retrieve file using git
 					var parts = file.index[0].split('..');
 					if (!file.new) {
-						fileFrom.contents = Buffer.from(git.show(parts[0]));
+						fileFrom.contents = git.show(parts[0]);
 					}
 					if (!file.deleted) {
-						fileTo.contents = Buffer.from(git.show(parts[1]));
+						fileTo.contents = git.show(parts[1]);
 					}
 				}
 				containerFrom.add(fileFrom, []);

diff --git a/lib/git.js b/lib/git.js
@@ -9,6 +9,7 @@ var Git = module.exports = function(basedir) {
 Git.prototype.show = function(revision) {
 	var self = this;
 	return child.spawnSync('git', ['show', revision], {
-		cwd: self.basedir
+		cwd: self.basedir,
+		maxBuffer: 1024 * 1024 * 100 // 100 MB
 	}).stdout;
 };
diff --git a/lib/metadata-file-container.js b/lib/metadata-file-container.js
@@ -126,7 +126,7 @@ MetadataFileContainer.prototype.parse = function() {
 	if (!self.contents || self.contents.length === 0) {
 		return self;
 	}
-	var parsed = new xmldoc.XmlDocument(self.contents);
+	var parsed = new xmldoc.XmlDocument(self.contents.toString());
 	var type = self.getMetadataType();
 	var objectName = self.getComponent().fullName;
 	if (type.childXmlNames) {
@@ -190,7 +190,7 @@ MetadataFileContainer.prototype.contentsWithoutChilds = function(includedChildCo
 
 	var childComponents = this.getMetadataType().childXmlNames;
 	if (childComponents) {
-		var parsed = new xmldoc.XmlDocument(this.contents);
+		var parsed = new xmldoc.XmlDocument(this.contents.toString());
 		var parentMetadata = '<' + parsed.name + ' xmlns="http://soap.sforce.com/2006/04/metadata">';
 		var tagIncludedChildNames = includedChildComponents ?
 			includedChildComponents.map(function(child) {
@@ -255,7 +255,7 @@ MetadataFileContainer.prototype.toString = function() {
 	var groupedComponents = self.getGroupedAndSortedComponents();
 	_.keys(groupedComponents).sort(MetadataUtils.compareMetadataTypeNames).forEach(function(metadataType) {
 		groupedComponents[metadataType].forEach(function(cmp) {
-			lines.push(cmp.contents);
+			lines.push(cmp.contents.toString());
 		});
 	});
 	lines.push("</" + type.xmlName + ">");
@@ -296,7 +296,7 @@ MetadataFileContainer.diffMaps = function(mapA, mapB) {
 };
 
 // STATIC
-var builtInProps = ['diff', 'getManifest', 'parse', 'addComponent', 'getGroupedAndSortedComponents', 'writeContents', 'toString', 'diffMaps'];
+var builtInProps = ['components', 'diff', 'getManifest', 'parse', 'addComponent', 'getGroupedAndSortedComponents', 'contentsWithoutChilds', 'writeContents', 'updateContents', 'toString', 'diffMaps'];
 
 MetadataFileContainer.isCustomProp = function(name) {
 	return MetadataFile.isCustomProp(name) && builtInProps.indexOf(name) === -1;

diff --git a/lib/metadata-file.js b/lib/metadata-file.js
@@ -147,7 +147,7 @@ MetadataFile.prototype.getComponent = function() {
 };
 
 // STATIC
-var builtInProps = ['basenameDirname', 'parentDirname', 'extnameWithoutDot', 'filename', 'diff', 'getMetadataFilename', 'getMetadataType', 'getComponent'];
+var builtInProps = ['basenameDirname', 'waveTemplateDirname', 'parentDirname', 'extnameWithoutDot', 'filename', 'diff', 'getMetadataFilename', 'getMetadataType', 'getComponent'];
 
 MetadataFile.isCustomProp = function(name) {
 	return File.isCustomProp(name) && builtInProps.indexOf(name) === -1;

diff --git a/scripts/generate-large-customobject.js b/scripts/generate-large-customobject.js
@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+var metadataParts = require("../test/metadata-parts/objects");
+var numberOfComponents = parseInt(process.env.COUNT || "10000");
+
+console.log(metadataParts.header);
+for (var i = 0; i < numberOfComponents; i++) {
+	console.log(
+		metadataParts.fields.textField1.replace(
+			"Test",
+			"Test" + i.toString().padStart(4, "0")
+		)
+	);
+}
+console.log("    <label>LargeCustomObjectTest__c</label>")
+console.log(metadataParts.footer);
diff --git a/...etadata/customField-updated-large-object/expected/objects/LargeCustomObjectTest__c.object b/...etadata/customField-updated-large-object/expected/objects/LargeCustomObjectTest__c.object
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CustomObject xmlns="http://soap.sforce.com/2006/04/metadata">
+    <fields>
+        <fullName>Test0815__c</fullName>
+        <externalId>false</externalId>
+        <label>Test Updated</label>
+        <length>200</length>
+        <required>false</required>
+        <trackFeedHistory>false</trackFeedHistory>
+        <trackHistory>false</trackHistory>
+        <trackTrending>false</trackTrending>
+        <type>Text</type>
+        <unique>false</unique>
+    </fields>
+</CustomObject>
diff --git a/test-functional/data/complex-metadata/customField-updated-large-object/expected/package.xml b/test-functional/data/complex-metadata/customField-updated-large-object/expected/package.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Package xmlns="http://soap.sforce.com/2006/04/metadata">
+    <types>
+        <members>LargeCustomObjectTest__c.Test0815__c</members>
+        <name>CustomField</name>
+    </types>
+    <version>46.0</version>
+</Package>