tests: save smokehouse failures, improve bundle runner logging (#15235)

GoogleChrome · Jul 10, 2023 · fbee662 · fbee662
1 parent 4d3ee72
commit fbee662
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 24 deletions.
diff --git a/.github/workflows/devtools.yml b/.github/workflows/devtools.yml
@@ -176,4 +176,4 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (devtools smoke)
-        path: ${{ github.workspace }}/lighthouse/.tmp/smokehouse-ci-failures/
+        path: ${{ github.workspace }}/lighthouse/.tmp/smokehouse-failures/
diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml
@@ -75,7 +75,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (ubuntu; chrome ${{ matrix.chrome-channel }})
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-windows:
     strategy:
@@ -118,7 +118,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (windows)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-legacy:
     strategy:
@@ -166,7 +166,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (legacy)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-bundle:
     strategy:
@@ -214,7 +214,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (bundled)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-bundle-legacy:
     strategy:
@@ -262,4 +262,4 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (bundled)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
diff --git a/cli/test/smokehouse/frontends/smokehouse-bin.js b/cli/test/smokehouse/frontends/smokehouse-bin.js
@@ -25,6 +25,8 @@ import {runSmokehouse, getShardedDefinitions} from '../smokehouse.js';
 import {updateTestDefnFormat} from './back-compat-util.js';
 import {LH_ROOT} from '../../../../root.js';
 import exclusions from '../config/exclusions.js';
+import {saveArtifacts} from '../../../../core/lib/asset-saver.js';
+import {saveLhr} from '../../../../core/lib/asset-saver.js';
 
 const coreTestDefnsPath =
   path.join(LH_ROOT, 'cli/test/smokehouse/core-tests.js');
@@ -230,19 +232,23 @@ async function begin() {
   if (!smokehouseResult.success) {
     const failedTestResults = smokehouseResult.testResults.filter(r => r.failed);
 
-    // For CI, save failed runs to directory to be uploaded.
-    if (process.env.CI) {
-      const failuresDir = `${LH_ROOT}/.tmp/smokehouse-ci-failures`;
-      fs.mkdirSync(failuresDir, {recursive: true});
-
-      for (const testResult of failedTestResults) {
-        for (let i = 0; i < testResult.runs.length; i++) {
-          const run = testResult.runs[i];
-          fs.writeFileSync(`${failuresDir}/${testResult.id}-${i}.json`, JSON.stringify({
-            ...run,
-            lighthouseLog: run.lighthouseLog.split('\n'),
-            assertionLog: run.assertionLog.split('\n'),
-          }, null, 2));
+    // Save failed runs to directory. In CI, this is uploaded as an artifact.
+    const failuresDir = `${LH_ROOT}/.tmp/smokehouse-failures`;
+    fs.rmSync(failuresDir, {recursive: true, force: true});
+    fs.mkdirSync(failuresDir);
+
+    for (const testResult of failedTestResults) {
+      for (let i = 0; i < testResult.runs.length; i++) {
+        const runDir = `${failuresDir}/${i}/${testResult.id}`;
+        fs.mkdirSync(runDir, {recursive: true});
+
+        const run = testResult.runs[i];
+        await saveArtifacts(run.artifacts, runDir);
+        await saveLhr(run.lhr, runDir);
+        fs.writeFileSync(`${runDir}/assertionLog.txt`, run.assertionLog);
+        fs.writeFileSync(`${runDir}/lighthouseLog.txt`, run.lighthouseLog);
+        if (run.networkRequests) {
+          fs.writeFileSync(`${runDir}/networkRequests.txt`, run.networkRequests.join('\n'));
         }
       }
     }

diff --git a/cli/test/smokehouse/lighthouse-runners/bundle.js b/cli/test/smokehouse/lighthouse-runners/bundle.js
@@ -82,7 +82,7 @@ async function runBundledLighthouse(url, config, testRunnerOptions) {
 
   // Run Lighthouse.
   try {
-    const logLevel = testRunnerOptions.isDebug ? 'info' : undefined;
+    const logLevel = testRunnerOptions.isDebug ? 'verbose' : 'info';
     let runnerResult;
     if (testRunnerOptions.useLegacyNavigation) {
       const connection = new CriConnection(port);
@@ -124,12 +124,10 @@ async function runLighthouse(url, config, testRunnerOptions = {}) {
   worker.stdout.setEncoding('utf8');
   worker.stderr.setEncoding('utf8');
   worker.stdout.addListener('data', (data) => {
-    process.stdout.write(data);
-    logs.push(`STDOUT: ${data}`);
+    logs.push(`[STDOUT] ${data}`);
   });
   worker.stderr.addListener('data', (data) => {
-    process.stderr.write(data);
-    logs.push(`STDERR: ${data}`);
+    logs.push(`[STDERR] ${data}`);
   });
   const [workerResponse] = await once(worker, 'message');
   const log = logs.join('') + '\n';