Feature/SYSTEST-9487 add live session logging (#136)

* update package locks * add documentation * add sessionws and sessionfilestream * add unit tests * update documentation * move ws and filestream into one class * move wsRegex constant * Update documentation
rdkcentral · Aug 11, 2023 · b0ec107 · b0ec107
1 parent 1f4c456
commit b0ec107
Show file tree

Hide file tree

Showing 6 changed files with 401 additions and 60 deletions.
diff --git a/cli/README.md b/cli/README.md
@@ -46,14 +46,19 @@ To change the output format of the session recording use option
 `--sessionOutput log` 
 with start, stop, or call in between starting and stopping.
 Valid output options are:
+
 - log (default): method call request and response objects ordered by the timestamp of the objects.
 - raw: list of method call objects containing both request and response in one object.
-- mock-overrides: a directory of method calls extrapolated from the raw format. This option converts the raw formats into json files or yaml files. A json file would be the response from a method call that took place when mock-firebolt processed it. Yaml files contain a function that returns a json response depending on input params. Yaml files are only generated if there were multiple of the same method call with different params. 
+- mock-overrides: a directory of method calls extrapolated from the raw format. This option converts the raw formats into json files or yaml files. A json file would be the response from a method call that took place when mock-firebolt processed it. Yaml files contain a function that returns a json response depending on input params. Yaml files are only generated if there were multiple of the same method call with different params.
+- live: This format operates similarly to the 'raw' format with an added real-time feature. As each message is received, it gets immediately written to the specified output file. In addition to accepting regular file paths, the 'live' option also supports WebSocket (WS/WSS) URLs. If a WS/WSS URL is designated as the outputPath, a WebSocket connection is established with the specified URL, and the new messages are dispatched to that connection. Please note that specifying an outputPath is essential for the 'live' option. This path is necessary whether you're sending the live log to a WebSocket URL or saving a live copy of the log file to a local directory.
 
-To change the output directory of the session recording use 
-`--sessionOutputPath ./output/examples`
-with start, stop, or call in between starting and stopping.
-This will save the recording to the directory specified. Default for log|raw is server/output/sessions and for mock-overrides is server/output/mocks
+To change the output directory of the session recording, use the --sessionOutputPath option with start, stop, or call. This can be done at any time between starting and stopping the recording. For example, --sessionOutputPath ./output/examples will save the recording to the directory specified. The default paths for log and raw formats are ./output/sessions, and for mock-overrides is ./output/mocks.
+
+Please be aware that while utilizing the "live" option, the system doesn't create a singular encompassing JSON array or object. Instead, it produces individual legal JSON objects for each method call or method response that's observed. Owing to the real-time nature of this process, each message is directly written as it arrives.
+
+For file storage, a newline character is appended after every message to facilitate sequential additions. When it comes to WebSocket, the system simply employs the write() function for every incoming message.
+
+A point to consider is that responses might be written to a file before the corresponding messages are received. Consequently, the file will not correlate the responses with their initiating requests. The responsibility lies with the user to associate request IDs with the relevant response IDs when analyzing the output.
 
 ## Sequence of Events
 

diff --git a/cli/package-lock.json b/cli/package-lock.json
diff --git a/cli/src/usage.mjs b/cli/src/usage.mjs
@@ -42,8 +42,8 @@ const lines = [
   { cmdInfo: "--broadcastEvent ../examples/device-onDeviceNameChanged1.event.json",       comment: "Send BroadcastEvent (method, result keys expected)" },
   { cmdInfo: "--sequence ../examples/events1.sequence.json  ",                            comment: "Send an event sequence (See examples/device-onDeviceNameChanged.sequence.json)" },
   { cmdInfo: "--session start/stop  ",                                                    comment: "Start/Stop Firebolt session recording" },
-  { cmdInfo: "--sessionOutput log|raw|mock-overrides  ",                                  comment: "Set the output format to; log: (paired time sequence of calls, responses)|raw: similiar to log but not paired with request|mock-overrides: a directory of mock overrides" },
-  { cmdInfo: "--sessionOutputPath ../examples/path  ",                                    comment: "Specifiy the session output path. Default for 'log' format will be ./output/sessions and ./output/mocks/<START_TIME> for 'mock-overrides'." },
+  { cmdInfo: "--sessionOutput log|raw|mock-overrides|live  ",                             comment: "Set the output format to; log: (paired time sequence of calls, responses)|raw: similiar to log but not paired with request|mock-overrides: a directory of mock overrides|live: log messages as they are received in real time - can also be a websocket url (live only)" },
+  { cmdInfo: "--sessionOutputPath ../examples/path  ",                                    comment: "Specifiy the session output path. Default for 'log' format will be ./output/sessions and ./output/mocks/<START_TIME> for 'mock-overrides'. Can also be a websocket url" },
   { cmdInfo: "--getStatus ",                                                              comment: "Shows ws connection status of the user"},
   { cmdInfo: "--downloadOverrides https://github.com/myOrg/myRepo.git",                   comment: "Specifies the url of a github repository to clone"},
   { cmdInfo: "--overrideLocation ../externalOverrides",                                   comment: "Specifies a location relative to the current working directory in which to save the cloned github repository's contents"},
@@ -59,4 +59,4 @@ function usage() {
   console.log('  ./mf.sh --help');
 }
 
-export { usage };
+export { usage };
diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/src/sessionManagement.mjs b/server/src/sessionManagement.mjs
@@ -23,6 +23,64 @@
 import { logger } from './logger.mjs';
 import fs from 'fs';
 import yaml from 'js-yaml';
+import WebSocket from 'ws';
+
+const wsRegex = /^(ws(s)?):\/\//i;
+
+class SessionHandler {
+  constructor() {
+    this.mode = null;
+    this.ws = null;
+    this.stream = null;
+  }
+
+  // Determine mode based on directory/url
+  _determineMode(dir) {
+    if (wsRegex.test(dir)) {
+      this.mode = 'websocket';
+    } else {
+      this.mode = 'filestream';
+    }
+  }
+
+  open(dir) {
+    this._determineMode(dir);
+
+    if (this.mode === 'websocket') {
+      this.ws = new WebSocket(dir);
+      this.ws.on('open', () => {
+        logger.info(`Websocket connection opened: ${dir}`);
+      });
+    } else {
+      if (!fs.existsSync(dir)) {
+        logger.info("Directory does not exist for: " + dir);
+        fs.mkdirSync(dir, { recursive: true });
+      }
+
+      this.stream = fs.createWriteStream(`${dir}/FireboltCalls_live.log`, { flags: 'a' });
+    }
+  }
+
+  write(data) {
+    if (this.mode === 'websocket' && this.ws) {
+      this.ws.send(data); 
+    } else if (this.stream) {
+      this.stream.write(`${data}\n`);
+    }
+  }
+
+  close() {
+    if (this.mode === 'websocket' && this.ws) {
+      this.ws.close();
+      this.ws = null;
+    } else if (this.stream) {
+      this.stream.end();
+      this.stream = null;
+    }
+  }
+}
+
+let sessionHandler = new SessionHandler();
 
 class FireboltCall {
     constructor(methodCall, params) {
@@ -59,6 +117,9 @@ class Session {
             // const sessionStartString = sessionStart.toISOString().replace(/T/, '_').replace(/\..+/, '');
             // logger.info(`${sessionStart.toISOString()}`);    
 
+            // Check if the output path is a WebSocket URL
+            this.sessionOutputPath = wsRegex.test(this.sessionOutputPath) ? "./sessions/output" : this.sessionOutputPath;
+
             if (!fs.existsSync(this.sessionOutputPath)) {
                 logger.info("Directory does not exist for: " + this.sessionOutputPath)
                 fs.mkdirSync(this.sessionOutputPath, { recursive: true});
@@ -345,32 +406,38 @@ let sessionRecording = {
     recordedSession : new Session()
 };
 
-function startRecording(){
+function startRecording() {
     logger.info('Starting recording');
     sessionRecording.recording = true;
     sessionRecording.recordedSession = new Session();
 }
 
-function stopRecording(){
-    if (isRecording()) {
-        logger.info('Stopping recording');
-        sessionRecording.recording = false;
-        return sessionRecording.recordedSession.exportSession();
-    } else {
-        logger.warn('Trying to stop recording when not recording');
-        return null;
-    }
+function stopRecording() {
+  if (isRecording()) {
+      logger.info('Stopping recording');
+      sessionRecording.recording = false;
+      const sessionData = sessionRecording.recordedSession.exportSession();
+      sessionHandler.close();
+      return sessionData;
+  } else {
+      logger.warn('Trying to stop recording when not recording');
+      return null;
+  }
 }
 
 function isRecording(){
     return sessionRecording.recording;
 }
 
 function addCall(methodCall, params){
-    if(isRecording()){
+    if (isRecording()) {
         const call = new FireboltCall(methodCall, params);
-        call.sequenceId = sessionRecording.recordedSession.calls.length + 1
+        call.sequenceId = sessionRecording.recordedSession.calls.length + 1;
         sessionRecording.recordedSession.calls.push(call);
+        if (sessionRecording.recordedSession.sessionOutput === "live") {
+            const data = JSON.stringify(call);
+            sessionHandler.write(data);
+        }
     }
 }
 
@@ -383,10 +450,13 @@ function getOutputFormat(){
     return sessionRecording.recordedSession.sessionOutput;
 }
 
-function setOutputDir(dir){
-    sessionRecording.recordedSession.sessionOutputPath = dir;
-    sessionRecording.recordedSession.mockOutputPath = dir;
-    logger.info("Setting output path: " + sessionRecording.recordedSession.mockOutputPath);
+function setOutputDir(dir) {
+  if (sessionRecording.recordedSession.sessionOutput === "live") {
+      sessionHandler.open(dir);
+  }
+  sessionRecording.recordedSession.sessionOutputPath = dir;
+  sessionRecording.recordedSession.mockOutputPath = dir;
+  logger.info("Setting output path: " + sessionRecording.recordedSession.mockOutputPath);
 }
 
 function getSessionOutputDir(){
@@ -398,15 +468,30 @@ function getMockOutputDir(){
 }
 
 function updateCallWithResponse(method, result, key) {
-    if(isRecording()) {
-        const methodCalls = sessionRecording.recordedSession.calls
-        for(let i = 0; i < methodCalls.length; i++) {
-            if(methodCalls[i].methodCall == method) {
-                methodCalls[i].response = {[key]: result, timestamp: Date.now()}
-                sessionRecording.recordedSession.calls.concat(...methodCalls);
-            }
-        }
-    }
+  if (isRecording()) {
+      const methodCalls = sessionRecording.recordedSession.calls;
+      for(let i = 0; i < methodCalls.length; i++) {
+          if(methodCalls[i].methodCall == method) {
+              methodCalls[i].response = {[key]: result, timestamp: Date.now()};
+              sessionRecording.recordedSession.calls.concat(...methodCalls);
+              if (sessionRecording.recordedSession.sessionOutput === "live") {
+                  const data = JSON.stringify(methodCalls[i]);
+                  sessionHandler.write(data);
+              }
+          }
+      }
+  }
+}
+
+// Utility function for unit tests
+const setTestEntity = (mockEntity) => {
+  sessionHandler = mockEntity
+}
+
+export const testExports = {
+  setTestEntity,
+  setOutputDir,
+  SessionHandler
 }
 
-export {Session, FireboltCall, startRecording, stopRecording, addCall, isRecording, updateCallWithResponse, setOutputFormat, getOutputFormat, setOutputDir, getSessionOutputDir, getMockOutputDir};
+export { Session, FireboltCall, startRecording, setOutputDir, stopRecording, addCall, isRecording, updateCallWithResponse, setOutputFormat, getOutputFormat, getSessionOutputDir, getMockOutputDir };