ref: Introduce test runner for node session health tests (#3728)

* ref: Introduce test runner for node session health tests

* Make it work on Node 6

packages/node/package.json

@@ -60,12 +60,13 @@
     "test:watch": "jest --watch",
     "test:express": "node test/manual/express-scope-separation/start.js",
     "test:webpack": "cd test/manual/webpack-domain/ && yarn && node npm-build.js",
-    "test:release-health": "node test/manual/release-health/single-session/healthy-session.js && node test/manual/release-health/single-session/caught-exception-errored-session.js && node test/manual/release-health/single-session/uncaught-exception-crashed-session.js && node test/manual/release-health/single-session/unhandled-rejection-crashed-session.js && node test/manual/release-health/session-aggregates/aggregates-disable-single-session.js && node test/manual/release-health/single-session/errors-in-session-capped-to-one.js & node test/manual/release-health/single-session/terminal-state-sessions-sent-once.js",
+    "test:release-health": "node test/manual/release-health/runner.js",
     "pack": "npm pack",
     "circularDepCheck": "madge --circular src/index.ts"
   },
   "volta": {
-    "extends": "../../package.json"
+    "extends": "../../package.json",
+    "node": "6.17.1"
   },
   "jest": {
     "collectCoverage": true,

packages/node/test/manual/release-health/runner.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+
+const COLOR_RESET = '\x1b[0m';
+const COLORS = {
+  green: '\x1b[32m',
+  red: '\x1b[31m',
+  yellow: '\x1b[33m',
+};
+
+const colorize = (str, color) => {
+  if (!(color in COLORS)) {
+    throw new Error(`Unknown color. Available colors: ${Object.keys(COLORS).join(', ')}`);
+  }
+
+  return `${COLORS[color]}${str}${COLOR_RESET}`;
+};
+
+const scenariosDirs = ['session-aggregates', 'single-session'];
+const scenarios = [];
+
+for (const dir of scenariosDirs) {
+  const scenarioDir = path.resolve(__dirname, dir);
+  const filenames = fs.readdirSync(scenarioDir);
+  const paths = filenames.map(filename => [filename, path.resolve(scenarioDir, filename)]);
+  scenarios.push(...paths);
+}
+
+const processes = scenarios.map(([filename, filepath]) => {
+  return new Promise(resolve => {
+    const scenarioProcess = spawn('/usr/bin/env', ['node', filepath]);
+    const output = [];
+    const errors = [];
+
+    scenarioProcess.stdout.on('data', data => {
+      output.push(data.toString());
+    });
+
+    scenarioProcess.stderr.on('data', data => {
+      errors.push(data.toString());
+    });
+
+    scenarioProcess.on('exit', code => {
+      if (code === 0) {
+        console.log(colorize(`PASSED: ${filename}`, 'green'));
+      } else {
+        console.log(colorize(`FAILED: ${filename}`, 'red'));
+
+        if (output.length) {
+          console.log(colorize(output.join('\n'), 'yellow'));
+        }
+        if (errors.length) {
+          console.log(colorize(errors.join('\n'), 'yellow'));
+        }
+      }
+
+      resolve(code);
+    });
+  });
+});
+
+Promise.all(processes).then(codes => {
+  if (codes.some(code => code !== 0)) {
+    process.exit(1);
+  }
+});

packages/node/test/manual/release-health/session-aggregates/aggregates-disable-single-session.js

@@ -5,7 +5,6 @@ const Sentry = require('../../../../dist');
 const { assertSessions, BaseDummyTransport } = require('../test-utils');
 
 function cleanUpAndExitSuccessfully() {
-  console.log('SUCCESS: Session Aggregates payload is correct!');
   server.close();
   clearInterval(flusher._intervalId);
   process.exit(0);

packages/node/test/manual/release-health/single-session/caught-exception-errored-session.js

@@ -16,21 +16,25 @@ validateSessionCountFunction(sessionCounts);
 class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     sessionCounts.sessionCounter++;
+
     if (sessionCounts.sessionCounter === 1) {
       assertSessions(constructStrippedSessionObject(session), {
         init: true,
         status: 'ok',
         errors: 1,
         release: '1.1',
       });
-    } else if (sessionCounts.sessionCounter === 2) {
+    }
+
+    if (sessionCounts.sessionCounter === 2) {
       assertSessions(constructStrippedSessionObject(session), {
         init: false,
         status: 'exited',
         errors: 1,
         release: '1.1',
       });
     }
+
     return super.sendSession(session);
   }
 }

packages/node/test/manual/release-health/single-session/errors-in-session-capped-to-one.js

@@ -16,21 +16,25 @@ validateSessionCountFunction(sessionCounts);
 class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     sessionCounts.sessionCounter++;
+
     if (sessionCounts.sessionCounter === 1) {
       assertSessions(constructStrippedSessionObject(session), {
         init: true,
         status: 'ok',
         errors: 1,
         release: '1.1',
       });
-    } else if (sessionCounts.sessionCounter === 2) {
+    }
+
+    if (sessionCounts.sessionCounter === 2) {
       assertSessions(constructStrippedSessionObject(session), {
         init: false,
         status: 'exited',
         errors: 1,
         release: '1.1',
       });
     }
+
     return super.sendSession(session);
   }
 }

packages/node/test/manual/release-health/single-session/healthy-session.js

@@ -16,14 +16,14 @@ validateSessionCountFunction(sessionCounts);
 class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     sessionCounts.sessionCounter++;
-    if (sessionCounts.sessionCounter === 1) {
-      assertSessions(constructStrippedSessionObject(session), {
-        init: true,
-        status: 'exited',
-        errors: 0,
-        release: '1.1',
-      });
-    }
+
+    assertSessions(constructStrippedSessionObject(session), {
+      init: true,
+      status: 'exited',
+      errors: 0,
+      release: '1.1',
+    });
+
     return super.sendSession(session);
   }
 }

packages/node/test/manual/release-health/single-session/terminal-state-sessions-sent-once.js

@@ -17,14 +17,13 @@ class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     sessionCounts.sessionCounter++;
 
-    if (sessionCounts.sessionCounter === 1) {
-      assertSessions(constructStrippedSessionObject(session), {
-        init: true,
-        status: 'crashed',
-        errors: 1,
-        release: '1.1',
-      });
-    }
+    assertSessions(constructStrippedSessionObject(session), {
+      init: true,
+      status: 'crashed',
+      errors: 1,
+      release: '1.1',
+    });
+
     return super.sendSession(session);
   }
 }

packages/node/test/manual/release-health/single-session/uncaught-exception-crashed-session.js

@@ -1,12 +1,6 @@
 const Sentry = require('../../../../dist');
 const { assertSessions, constructStrippedSessionObject, BaseDummyTransport } = require('../test-utils');
 
-process.on('exit', () => {
-  if (process.exitCode === 0) {
-    console.log('SUCCESS: All application mode sessions were sent to node transport as expected');
-  }
-});
-
 class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     assertSessions(constructStrippedSessionObject(session), {
@@ -15,6 +9,8 @@ class DummyTransport extends BaseDummyTransport {
       errors: 1,
       release: '1.1',
     });
+
+    // We need to explicitly exit process early here to allow for 0 exit code
     process.exit(0);
   }
 }
@@ -25,6 +21,7 @@ Sentry.init({
   transport: DummyTransport,
   autoSessionTracking: true,
 });
+
 /**
  * The following code snippet will throw an exception of `mechanism.handled` equal to `false`, and so this session
  * is considered a Crashed Session.

packages/node/test/manual/release-health/single-session/unhandled-rejection-crashed-session.js

@@ -17,14 +17,13 @@ class DummyTransport extends BaseDummyTransport {
   sendSession(session) {
     sessionCounts.sessionCounter++;
 
-    if (sessionCounts.sessionCounter === 1) {
-      assertSessions(constructStrippedSessionObject(session), {
-        init: true,
-        status: 'crashed',
-        errors: 1,
-        release: '1.1',
-      });
-    }
+    assertSessions(constructStrippedSessionObject(session), {
+      init: true,
+      status: 'crashed',
+      errors: 1,
+      release: '1.1',
+    });
+
     return super.sendSession(session);
   }
 }

packages/node/test/manual/release-health/test-utils.js

@@ -1,6 +1,8 @@
 function assertSessions(actual, expected) {
-  if (JSON.stringify(actual) !== JSON.stringify(expected)) {
-    console.error('FAILED: Sessions do not match');
+  actual = JSON.stringify(actual);
+  expected = JSON.stringify(expected);
+  if (actual !== expected) {
+    process.stdout.write(`Expected Session:\n  ${expected}\nActual Session:\n  ${actual}`);
     process.exit(1);
   }
 }
@@ -27,15 +29,12 @@ class BaseDummyTransport {
 }
 
 function validateSessionCountFunction(sessionCounts) {
-  process.on('exit', exitCode => {
+  process.on('exit', () => {
     const { sessionCounter, expectedSessions } = sessionCounts;
     if (sessionCounter !== expectedSessions) {
-      console.log(`FAIL: Expected ${expectedSessions} Sessions, Received ${sessionCounter}.`);
+      process.stdout.write(`Expected Session Count: ${expectedSessions}\nActual Session Count:   ${sessionCounter}`);
       process.exitCode = 1;
     }
-    if ((exitCode === 0) & !process.exitCode) {
-      console.log('SUCCESS: All application mode sessions were sent to node transport as expected');
-    }
   });
 }