feat: add heap monitoring (#369)

* feat: add heap monitoring

@W-15760648@ add heap monitoring and class instrumentation

* chore: fix ut

* chore: bump core and jsforce versions

* chore: move log trace calls

* chore: improve instrumentation

* chore: allow only one heap monitor

* chore: refactor interval assignment to c’tor

* chore: bump core

* chore: wip

* fix: use module bfj for json string and perf tuning

@W-15724695@
@W-15760648@

Replace json stringify with bfj module
Add tuning properties to streams
Add env vars to adjust buffer size and json format indent

* chore: apply review comments

* chore: fix windows ut for table

* chore: fix another windows test

* chore: add comments on setImmediate calls

* chore: bump kit

* chore: address review comments

* chore: add limits to new env var values

* chore: bump core to latest

package.json

@@ -10,11 +10,12 @@
     "@salesforce/core": "^7.3.10",
     "@salesforce/kit": "^3.1.2",
     "@types/istanbul-reports": "^3.0.4",
+    "bfj": "^8.0.0",
     "faye": "1.4.0",
-    "glob": "^10.3.10",
+    "glob": "^10.3.16",
     "istanbul-lib-coverage": "^3.2.2",
     "istanbul-lib-report": "^3.0.1",
-    "istanbul-reports": "^3.1.6"
+    "istanbul-reports": "^3.1.7"
   },
   "devDependencies": {
     "@commitlint/config-conventional": "^18.1.0",
@@ -36,13 +37,13 @@
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-header": "^3.0.0",
     "eslint-plugin-jsdoc": "^46.8.2",
-    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-prettier": "^5.1.3",
     "husky": "^8.0.0",
-    "lint-staged": "^15.0.2",
-    "mocha": "^10.2.0",
+    "lint-staged": "^15.2.4",
+    "mocha": "^10.4.0",
     "mocha-junit-reporter": "^2.2.1",
     "nyc": "^15.1.0",
-    "prettier": "^3.0.3",
+    "prettier": "^3.2.5",
     "shx": "^0.3.4",
     "sinon": "^17.0.1",
     "source-map-support": "^0.5.16",
@@ -90,4 +91,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

src/execute/executeService.ts

@@ -21,6 +21,7 @@ import { encodeBody } from './utils';
 import * as readline from 'readline';
 import type { HttpRequest } from '@jsforce/jsforce-node';
 import { elapsedTime } from '../utils/elapsedTime';
+import * as os from 'node:os';
 
 export class ExecuteService {
   public readonly connection: Connection;
@@ -96,7 +97,7 @@ export class ExecuteService {
       let apexCode = '';
       readInterface.on('line', (input: string) => {
         timeout.refresh();
-        apexCode = apexCode + input + '\n';
+        apexCode = apexCode + input + os.EOL;
       });
       readInterface.on('close', () => {
         resolve(apexCode);

src/reporters/coverageReporter.ts

@@ -17,7 +17,8 @@ import * as path from 'path';
 import { glob } from 'glob';
 import * as fs from 'fs';
 import { nls } from '../i18n';
-import { elapsedTime } from '../utils/elapsedTime';
+import { elapsedTime } from '../utils';
+import * as os from 'node:os';
 
 const startOfSource = (source: string): number => {
   if (source) {
@@ -154,7 +155,7 @@ export class CoverageReporter {
         try {
           sourceLines = fs
             .readFileSync(fileCoverageData.path, 'utf8')
-            .split('\n');
+            .split(os.EOL);
         } catch {
           // file not found
         }

src/reporters/humanFormatTransform.ts

@@ -13,11 +13,13 @@ import {
 } from '../tests';
 import { nls } from '../i18n';
 import { Readable, ReadableOptions } from 'node:stream';
-import { elapsedTime } from '../utils';
-import { LoggerLevel } from '@salesforce/core';
+import { elapsedTime, HeapMonitor } from '../utils';
+import { Logger, LoggerLevel } from '@salesforce/core';
 import { EOL } from 'os';
+import * as os from 'node:os';
 
 export class HumanFormatTransform extends Readable {
+  private logger: Logger;
   constructor(
     private readonly testResult: TestResult,
     private readonly detailedCoverage: boolean,
@@ -26,11 +28,19 @@ export class HumanFormatTransform extends Readable {
     super(options);
     this.testResult = testResult;
     this.detailedCoverage ??= false;
+    this.logger = Logger.childFromRoot('HumanFormatTransform');
   }
 
   _read(): void {
-    this.format();
-    this.push(null); // Indicates end of data
+    this.logger.trace('starting _read');
+    HeapMonitor.getInstance().checkHeapSize('HumanFormatTransform._read');
+    try {
+      this.format();
+      this.push(null); // Indicates end of data
+      this.logger.trace('finishing _read');
+    } finally {
+      HeapMonitor.getInstance().checkHeapSize('HumanFormatTransform._read');
+    }
   }
 
   @elapsedTime()
@@ -187,7 +197,7 @@ export class HumanFormatTransform extends Readable {
       }
     });
 
-    this.push('\n\n');
+    this.push(os.EOL.repeat(2));
     tb.createTable(
       testRowArray,
       [
@@ -232,7 +242,7 @@ export class HumanFormatTransform extends Readable {
       }
     );
 
-    this.push('\n\n');
+    this.push(os.EOL.repeat(2));
     tb.createTable(
       codeCovRowArray,
       [

src/reporters/humanReporter.ts

@@ -5,7 +5,7 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-import { elapsedTime, Row, Table } from '../utils';
+import { elapsedTime, HeapMonitor, Row, Table } from '../utils';
 import {
   ApexTestResultData,
   ApexTestResultOutcome,
@@ -14,22 +14,28 @@ import {
 } from '../tests';
 import { nls } from '../i18n';
 import { LoggerLevel } from '@salesforce/core';
+import * as os from 'node:os';
 
 export class HumanReporter {
   @elapsedTime()
   public format(testResult: TestResult, detailedCoverage: boolean): string {
-    let tbResult = this.formatSummary(testResult);
-    if (!testResult.codecoverage || !detailedCoverage) {
-      tbResult += this.formatTestResults(testResult.tests);
-    }
+    HeapMonitor.getInstance().checkHeapSize('HumanReporter.format');
+    try {
+      let tbResult = this.formatSummary(testResult);
+      if (!testResult.codecoverage || !detailedCoverage) {
+        tbResult += this.formatTestResults(testResult.tests);
+      }
 
-    if (testResult.codecoverage) {
-      if (detailedCoverage) {
-        tbResult += this.formatDetailedCov(testResult);
+      if (testResult.codecoverage) {
+        if (detailedCoverage) {
+          tbResult += this.formatDetailedCov(testResult);
+        }
+        tbResult += this.formatCodeCov(testResult.codecoverage);
       }
-      tbResult += this.formatCodeCov(testResult.codecoverage);
+      return tbResult;
+    } finally {
+      HeapMonitor.getInstance().checkHeapSize('HumanReporter.format');
     }
-    return tbResult;
   }
 
   @elapsedTime()
@@ -123,7 +129,7 @@ export class HumanReporter {
       }
     );
 
-    let testResultTable = '\n\n';
+    let testResultTable = os.EOL.repeat(2);
     testResultTable += tb.createTable(
       testRowArray,
       [
@@ -172,7 +178,7 @@ export class HumanReporter {
       }
     });
 
-    let detailedCovTable = '\n\n';
+    let detailedCovTable = os.EOL.repeat(2);
     detailedCovTable += tb.createTable(
       testRowArray,
       [
@@ -218,7 +224,7 @@ export class HumanReporter {
       }
     );
 
-    let codeCovTable = '\n\n';
+    let codeCovTable = os.EOL.repeat(2);
     codeCovTable += tb.createTable(
       codeCovRowArray,
       [

src/reporters/index.ts

@@ -6,7 +6,9 @@
  */
 export { TapReporter } from './tapReporter';
 export { JUnitReporter } from './junitReporter';
+export { JUnitFormatTransformer } from './junitFormatTransformer';
 export { HumanReporter } from './humanReporter';
+export { HumanFormatTransform } from './humanFormatTransform';
 export {
   CoverageReporter,
   CoverageReporterOptions,
@@ -15,4 +17,3 @@ export {
   DefaultWatermarks
 } from './coverageReporter';
 export { TapFormatTransformer } from './tapFormatTransform';
-export { JUnitFormatTransformer } from './junitFormatTransformer';

src/reporters/junitFormatTransformer.ts

@@ -5,9 +5,15 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import { ApexTestResultOutcome, TestResult } from '../tests';
-import { elapsedTime, formatStartTime, msToSecond } from '../utils';
+import {
+  elapsedTime,
+  formatStartTime,
+  HeapMonitor,
+  msToSecond
+} from '../utils';
 import { Readable, ReadableOptions } from 'node:stream';
 import { isEmpty } from '../narrowing';
+import { Logger } from '@salesforce/core';
 
 // cli currently has spaces in multiples of four for junit format
 const tab = '    ';
@@ -21,44 +27,70 @@ const timeProperties = [
 // properties not in cli junit spec
 const skippedProperties = ['skipRate', 'totalLines', 'linesCovered'];
 
+export type JUnitFormatTransformerOptions = ReadableOptions & {
+  bufferSize?: number;
+};
+
 export class JUnitFormatTransformer extends Readable {
+  private logger: Logger;
+  private buffer: string;
+  private bufferSize: number;
+
   constructor(
     private readonly testResult: TestResult,
-    options?: ReadableOptions
+    options?: JUnitFormatTransformerOptions
   ) {
     super(options);
     this.testResult = testResult;
+    this.logger = Logger.childFromRoot('JUnitFormatTransformer');
+    this.buffer = '';
+    this.bufferSize = options?.bufferSize || 256; // Default buffer size is 256
+  }
+
+  private pushToBuffer(chunk: string): void {
+    this.buffer += chunk;
+    if (this.buffer.length >= this.bufferSize) {
+      this.push(this.buffer);
+      this.buffer = '';
+    }
   }
 
   _read(): void {
+    this.logger.trace('starting _read');
+    HeapMonitor.getInstance().checkHeapSize('JUnitFormatTransformer._read');
     this.format();
+    if (this.buffer.length > 0) {
+      this.push(this.buffer);
+    }
     this.push(null); // Signal the end of the stream
+    this.logger.trace('finishing _read');
+    HeapMonitor.getInstance().checkHeapSize('JUnitFormatTransformer._read');
   }
 
   @elapsedTime()
   public format(): void {
     const { summary } = this.testResult;
 
-    this.push(`<?xml version="1.0" encoding="UTF-8"?>\n`);
-    this.push(`<testsuites>\n`);
-    this.push(`${tab}<testsuite name="force.apex" `);
-    this.push(`timestamp="${summary.testStartTime}" `);
-    this.push(`hostname="${summary.hostname}" `);
-    this.push(`tests="${summary.testsRan}" `);
-    this.push(`failures="${summary.failing}"  `);
-    this.push(`errors="0"  `);
-    this.push(`time="${msToSecond(summary.testExecutionTimeInMs)}">\n`);
+    this.pushToBuffer(`<?xml version="1.0" encoding="UTF-8"?>\n`);
+    this.pushToBuffer(`<testsuites>\n`);
+    this.pushToBuffer(`${tab}<testsuite name="force.apex" `);
+    this.pushToBuffer(`timestamp="${summary.testStartTime}" `);
+    this.pushToBuffer(`hostname="${summary.hostname}" `);
+    this.pushToBuffer(`tests="${summary.testsRan}" `);
+    this.pushToBuffer(`failures="${summary.failing}"  `);
+    this.pushToBuffer(`errors="0"  `);
+    this.pushToBuffer(`time="${msToSecond(summary.testExecutionTimeInMs)}">\n`);
 
     this.buildProperties();
     this.buildTestCases();
 
-    this.push(`${tab}</testsuite>\n`);
-    this.push(`</testsuites>\n`);
+    this.pushToBuffer(`${tab}</testsuite>\n`);
+    this.pushToBuffer(`</testsuites>\n`);
   }
 
   @elapsedTime()
   buildProperties(): void {
-    this.push(`${tab}${tab}<properties>\n`);
+    this.pushToBuffer(`${tab}${tab}<properties>\n`);
 
     Object.entries(this.testResult.summary).forEach(([key, value]) => {
       if (isEmpty(value) || skippedProperties.includes(key)) {
@@ -78,12 +110,16 @@ export class JUnitFormatTransformer extends Readable {
         value = formatStartTime(value);
       }
 
-      this.push(
+      this.pushToBuffer(
         `${tab}${tab}${tab}<property name="${key}" value="${value}"/>\n`
       );
+      // this call to setImmediate will schedule the closure on the event loop
+      // this action causing the current code to yield to the event loop
+      // allowing other processes to get time on the event loop
+      setImmediate(() => {});
     });
 
-    this.push(`${tab}${tab}</properties>\n`);
+    this.pushToBuffer(`${tab}${tab}</properties>\n`);
   }
 
   @elapsedTime()
@@ -92,7 +128,7 @@ export class JUnitFormatTransformer extends Readable {
 
     for (const testCase of testCases) {
       const methodName = JUnitFormatTransformer.xmlEscape(testCase.methodName);
-      this.push(
+      this.pushToBuffer(
         `${tab}${tab}<testcase name="${methodName}" classname="${
           testCase.apexClass.fullName
         }" time="${msToSecond(testCase.runTime)}">\n`
@@ -104,14 +140,18 @@ export class JUnitFormatTransformer extends Readable {
       ) {
         let message = isEmpty(testCase.message) ? '' : testCase.message;
         message = JUnitFormatTransformer.xmlEscape(message);
-        this.push(`${tab}${tab}${tab}<failure message="${message}">`);
+        this.pushToBuffer(`${tab}${tab}${tab}<failure message="${message}">`);
         if (testCase.stackTrace) {
-          this.push(`<![CDATA[${testCase.stackTrace}]]>`);
+          this.pushToBuffer(`<![CDATA[${testCase.stackTrace}]]>`);
         }
-        this.push(`</failure>\n`);
+        this.pushToBuffer(`</failure>\n`);
       }
 
-      this.push(`${tab}${tab}</testcase>\n`);
+      this.pushToBuffer(`${tab}${tab}</testcase>\n`);
+      // this call to setImmediate will schedule the closure on the event loop
+      // this action causing the current code to yield to the event loop
+      // allowing other processes to get time on the event loop
+      setImmediate(() => {});
     }
   }