Add python script for running multiple benchmarks

webrtc/.env_sample

@@ -0,0 +1,8 @@
+FISHJAM_ADDRESS="address"
+FISHJAM_TOKEN="token"
+SECURE=true
+DURATION=600
+PEER_DELAY=4.5
+CHROME_EXECUTABLE="/usr/bin/google-chrome"
+USE_SIMULCAST="true"
+CSV_REPORT_DIR="dir_path"

webrtc/grinder_scheduler.py

@@ -0,0 +1,133 @@
+import os
+import sys
+import math
+import json
+import atexit
+import signal
+import subprocess
+
+from dotenv import load_dotenv
+
+
+subprocesses = []
+
+def cleanup():
+    for process in subprocesses:
+        try:
+            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+        except Exception as e:
+            print(f"Error when trying to terminate process {process.pid}: {e}")
+
+def parse_values(values):
+    if isinstance(values, int):
+        return [values]
+    elif isinstance(values, str):
+        try:
+            start, end = map(int, values.split('..'))
+            return list(range(start, end+1))
+        except ValueError:
+            print(f"Invalid range format: {values}")
+            sys.exit(1)
+    else:
+        print(f"Unexpected values type: {type(values)}")
+        sys.exit(1)
+
+def get_configuration():
+    if len(sys.argv) != 2:
+        print("Usage: python grinder_scheduler.py '<configurations_json>'")
+        sys.exit(1)
+
+    configuration_json = sys.argv[1]
+    try:
+        configuration = json.loads(configuration_json)
+        return configuration
+
+    except json.JSONDecodeError:
+        print("Invalid JSON format")
+        sys.exit(1)
+
+def parse_configuration(configuration):
+    benchmarks = []
+
+    for config in configuration: # {peers: int, rooms: int} or {peers: 1..2, rooms: 5..10}
+        peers = config.get("peers")
+        rooms = config.get("rooms")
+
+        if peers is None or rooms is None:
+            print("Each configuration must include 'peers' and 'rooms'")
+            sys.exit(1)
+
+        peers = parse_values(peers)
+        rooms = parse_values(rooms)
+
+        benchmarks += [(peers_no, rooms_no) for rooms_no in rooms for peers_no in peers]
+
+    return benchmarks
+
+
+def run_benchmark(peers, rooms):
+    # Spawn max ten browsers
+    peers_per_browser = math.ceil(peers / 10)
+    csv_report_path = os.path.join(os.getenv("CSV_REPORT_DIR"), f"{peers}peers-{rooms}rooms.csv")
+
+    # Grinder configuration
+    grind_command = [
+        "npm", "run", "grind", "--",
+        f"--secure={os.getenv('SECURE')}",
+        f"--duration={os.getenv('DURATION')}",
+        f"--peer-delay={os.getenv('PEER_DELAY')}",
+        f"--use-simulcast={os.getenv('USE_SIMULCAST')}",
+        f"--fishjam-token={os.getenv('FISHJAM_TOKEN')}",
+        f"--fishjam-address={os.getenv('FISHJAM_ADDRESS')}",
+        f"--chrome-executable={os.getenv('CHROME_EXECUTABLE')}",
+        f"--peers={peers}",
+        f"--peers-per-browser={peers_per_browser}",
+        f"--peers-per-room={math.ceil(peers / rooms)}",
+        f"--csv-report-path={csv_report_path}"
+    ]
+
+    # Uncomment to gather metric on fishjam server side 
+    # Remember to install all necessary tools on fishjam server side
+    # Suggestion: https://github.com/mickel8/system_info
+
+    # ssh_command = [
+    #     "ssh", "-i", ssh_private_key, remote_host,
+    #     f"cd {remote_directory}; {remote_main_script} --file {csv_report_path} --time {remote_script_time} > /dev/null 2>&1"
+    # ]
+
+    grinder_process = subprocess.Popen(grind_command)
+    subprocesses.append(grinder_process)
+
+    # # Wait for grinder to spawn all peers
+    # delay = peers * os.getenv("PEER_DELAY") + 7.5
+    # time.sleep(delay)
+
+    # # Step 3: Start monitoring machine with fishjam
+    # ssh_process = subprocess.Popen(ssh_command)
+    # subprocesses.append(ssh_process)
+
+
+    # Wait for grinder to end
+    subprocess.Popen.wait(grinder_process)
+    subprocesses.remove(grinder_process)
+
+    # subprocess.Popen.wait(ssh_process)
+    # subprocesses.remove(ssh_process)
+
+
+def main(): 
+    configuration = get_configuration()
+    benchmarks = parse_configuration(configuration)
+
+    for (peers, rooms) in benchmarks:
+        run_benchmark(peers, rooms)
+
+if __name__ == "__main__":
+    # Register the cleanup function to be called at exit
+    atexit.register(cleanup)
+
+    # Load .env file with grinder configuration
+    load_dotenv()
+
+    # Run benchmarks
+    main()

webrtc/src/reporter.ts

@@ -2,6 +2,8 @@ import { ConsoleMessage } from "playwright";
 import { PeerToken, RawRtcScores, RawTrackEncodings } from "./types";
 
 const fs = require("fs");
+const path = require('path');
+
 
 let rawTrackEncodings: RawTrackEncodings = new Map();
 let rawTrackScores: RawRtcScores = new Map();
@@ -16,9 +18,15 @@ export const reportToString = (useSimulcast: boolean) => {
   return `Encodings: ${encoding}, RtcScore: ${rtcScore}`;
 };
 
-export const createReportCSV = (path: string) => {
+export const createReportCSV = (filepath: string) => {
   const csv_columns = `timestamp,min,max,q1,q2,q3,low,mid,high\n`;
-  fs.writeFile(path, csv_columns, () => {});
+  const dirpath = path.dirname(filepath);
+
+  fs.mkdir(dirpath, { recursive: true }, (err: any) => {
+    if (err) return console.error('Error creating directory:', err);
+  });
+
+  fs.writeFile(filepath, csv_columns, () => { });
 };
 
 export const appendReportCSV = (
@@ -38,11 +46,10 @@ export const appendReportCSV = (
     2
   )},${report.quartiles.Q1.toFixed(2)},${report.quartiles.Q2.toFixed(
     2
-  )},${report.quartiles.Q3.toFixed(2)},${report.encoding.l},${
-    report.encoding.m
-  },${report.encoding.h}\n`;
+  )},${report.quartiles.Q3.toFixed(2)},${report.encoding.l},${report.encoding.m
+    },${report.encoding.h}\n`;
 
-  fs.appendFile(path, newLine, (err: any) => {});
+  fs.appendFile(path, newLine, (err: any) => { });
 };
 
 export const onConsoleMessage = (msg: ConsoleMessage, peerToken: PeerToken) => {
@@ -104,7 +111,7 @@ export class RtcScoreReport {
       .flat()
       .sort((a, b) => a - b);
 
-    if (values.length < 4) {
+    if (values.length === 0) {
       this.report = null;
       return;
     }
@@ -141,6 +148,8 @@ export class RtcScoreReport {
   };
 
   private quartiles = (values: Array<number>) => {
+    if (values.length === 1) return { Q1: values[0], Q2: values[0], Q3: values[0] }
+
     const lowerHalf = values.slice(0, Math.floor(values.length / 2));
     const upperHalf = values.slice(Math.ceil(values.length / 2));