fix(testing): fix tests for Windows (#165)

In this PR, I identified and fixed various issue while running tests in the Windows environment: - shell needs to be specified to run programs in command shell on Windows, as described [here](https://stackoverflow.com/questions/60386867/node-spawn-child-process-not-working-in-windows) - path notation on Windows is different - JSON formatting is different for UNIX and Windows - writing into files replaced by echo command which works seamlessly on Windows - glob patterns require extra quotation marks for argument expansion on Linux - recreating tmp folder caused issues for me so I partially fixed it via adding the recursive option Note: This is the same scope as #131. Unfortunately, merging changes from the test refactor did not go well so I started off a clean branch and re-applied the changes.
code-pushup · Oct 27, 2023 · b80255b · b80255b
1 parent 7c81f81
commit b80255b
Show file tree

Hide file tree

Showing 29 changed files with 200 additions and 1,160 deletions.
diff --git a/e2e/cli-e2e/tests/__snapshots__/collect.spec.ts.snap b/e2e/cli-e2e/tests/__snapshots__/collect.spec.ts.snap
diff --git a/e2e/cli-e2e/tests/collect.spec.ts b/e2e/cli-e2e/tests/collect.spec.ts
@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { afterEach, beforeEach, vi } from 'vitest';
 import { PluginReport, Report, reportSchema } from '@code-pushup/models';
 import { executeProcess, readJsonFile, readTextFile } from '@code-pushup/utils';
@@ -21,18 +22,21 @@ describe('CLI collect', () => {
       plugins: report.plugins.map(omitVariableData) as PluginReport[],
     });
 
+  const cliPath = join('..', '..', 'dist', 'packages', 'cli');
+
   beforeEach(async () => {
     vi.clearAllMocks();
     cleanFolderPutGitKeep();
   });
+
   afterEach(() => {
     cleanFolderPutGitKeep();
   });
 
   it('should run ESLint plugin and create report.json', async () => {
     const { code, stderr } = await executeProcess({
       command: 'npx',
-      args: ['../../dist/packages/cli', 'collect'],
+      args: [cliPath, 'collect'],
       cwd: 'examples/react-todos-app',
     });
 
@@ -48,7 +52,7 @@ describe('CLI collect', () => {
   it('should create report.md', async () => {
     const { code, stderr } = await executeProcess({
       command: 'npx',
-      args: ['../../dist/packages/cli', 'collect', '--persist.format=md'],
+      args: [cliPath, 'collect', '--persist.format=md'],
       cwd: 'examples/react-todos-app',
     });
 
@@ -65,12 +69,7 @@ describe('CLI collect', () => {
   it('should print report summary to stdout', async () => {
     const { code, stdout, stderr } = await executeProcess({
       command: 'npx',
-      args: [
-        '../../dist/packages/cli',
-        'collect',
-        '--verbose',
-        '--persist.format=stdout',
-      ],
+      args: [cliPath, 'collect', '--verbose', '--persist.format=stdout'],
       cwd: 'examples/react-todos-app',
     });
 

diff --git a/packages/cli/code-pushup.config.ts → packages/cli/code-pushup.config.mock.ts b/packages/cli/code-pushup.config.ts → packages/cli/code-pushup.config.mock.ts
@@ -1,3 +1,6 @@
+import { join } from 'path';
+import { echoRunnerConfig } from '../models/test/fixtures/echo-runner-config.mock';
+
 const outputDir = 'tmp';
 
 export default {
@@ -37,35 +40,26 @@ export default {
           docsUrl: 'http://www.my-docs.dev?slug=dummy-audit-3',
         },
       ],
-      runner: {
-        command: 'node',
-        args: [
-          '-e',
-          `require('fs').writeFileSync('${outputDir}/dummy-plugin-output.json', '${JSON.stringify(
-            [
-              {
-                title: 'Dummy Audit 1',
-                slug: 'dummy-audit-1',
-                value: 420,
-                score: 0.42,
-              },
-              {
-                title: 'Dummy Audit 2',
-                slug: 'dummy-audit-2',
-                value: 80,
-                score: 0,
-              },
-              {
-                title: 'Dummy Audit 3',
-                slug: 'dummy-audit-3',
-                value: 12,
-                score: 0.12,
-              },
-            ],
-          )}')`,
+      runner: echoRunnerConfig(
+        [
+          {
+            slug: 'dummy-audit-1',
+            value: 420,
+            score: 0.42,
+          },
+          {
+            slug: 'dummy-audit-2',
+            value: 80,
+            score: 0,
+          },
+          {
+            slug: 'dummy-audit-3',
+            value: 12,
+            score: 0.12,
+          },
         ],
-        outputFile: `${outputDir}/dummy-plugin-output.json`,
-      },
+        join(outputDir, 'dummy-plugin-output.json'),
+      ),
     },
   ],
   categories: [

diff --git a/packages/cli/src/lib/implementation/config-middleware.spec.ts b/packages/cli/src/lib/implementation/config-middleware.spec.ts
@@ -42,7 +42,7 @@ describe('applyConfigMiddleware', () => {
   });
 
   it('should throw with invalid config', async () => {
-    const invalidConfig = 'wrong/path/to/config';
+    const invalidConfig = join('wrong', 'path', 'to', 'config');
     let error: Error = new Error();
     await configMiddleware({ config: invalidConfig }).catch(e => (error = e));
     expect(error?.message).toContain(invalidConfig);

diff --git a/packages/cli/test/fs.mock.ts b/packages/cli/test/fs.mock.ts
@@ -6,7 +6,7 @@ export function cleanFolder<T extends object>(
   content?: { [key in keyof T]: string },
 ) {
   rmSync(dirName, { recursive: true, force: true });
-  mkdirSync(dirName);
+  mkdirSync(dirName, { recursive: true });
   if (content) {
     for (const fileName in content) {
       writeFileSync(join(dirName, fileName), content[fileName]);
@@ -19,7 +19,7 @@ export function cleanFolderPutGitKeep<T extends object>(
   content?: { [key in keyof T]: string },
 ) {
   rmSync(dirName, { recursive: true, force: true });
-  mkdirSync(dirName);
+  mkdirSync(dirName, { recursive: true });
   writeFileSync(join(dirName, '.gitkeep'), '');
   if (content) {
     for (const fileName in content) {

diff --git a/packages/core/src/lib/implementation/execute-plugin.spec.ts b/packages/core/src/lib/implementation/execute-plugin.spec.ts
@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { beforeEach, describe, expect, it } from 'vitest';
 import {
   AuditReport,
@@ -6,8 +7,8 @@ import {
 } from '@code-pushup/models';
 import {
   auditReport,
+  echoRunnerConfig,
   pluginConfig,
-  runnerConfig,
 } from '@code-pushup/models/testing';
 import { cleanFolder } from '../../../test';
 import { executePlugin, executePlugins } from './execute-plugin';
@@ -45,7 +46,7 @@ describe('executePlugin', () => {
       { p: 42 } as unknown as AuditReport,
     ];
     const pluginCfg = pluginConfig([auditReport()], {
-      runner: runnerConfig(invalidAuditOutputs),
+      runner: echoRunnerConfig(invalidAuditOutputs, join('tmp', 'out.json')),
     });
     await expect(() => executePlugin(pluginCfg)).rejects.toThrowError(
       'Plugin output of plugin with slug mock-plugin-slug',

diff --git a/packages/core/test/fs.mock.ts b/packages/core/test/fs.mock.ts
@@ -6,7 +6,7 @@ export function cleanFolder<T extends object>(
   content?: { [key in keyof T]: string },
 ) {
   rmSync(dirName, { recursive: true, force: true });
-  mkdirSync(dirName);
+  mkdirSync(dirName, { recursive: true });
   if (content) {
     for (const fileName in content) {
       writeFileSync(join(dirName, fileName), content[fileName]);
@@ -19,7 +19,7 @@ export function cleanFolderPutGitKeep<T extends object>(
   content?: { [key in keyof T]: string },
 ) {
   rmSync(dirName, { recursive: true, force: true });
-  mkdirSync(dirName);
+  mkdirSync(dirName, { recursive: true });
   writeFileSync(join(dirName, '.gitkeep'), '');
   if (content) {
     for (const fileName in content) {

diff --git a/packages/models/test/fixtures/code-pushup.config.mock.cjs b/packages/models/test/fixtures/code-pushup.config.mock.cjs
@@ -1,3 +1,6 @@
+import { join } from 'path';
+import { echoRunnerConfig } from './echo-runner-config.mock';
+
 const outputDir = 'tmp';
 module.exports = {
   upload: {
@@ -17,23 +20,16 @@ module.exports = {
           docsUrl: 'http://www.my-docs.dev',
         },
       ],
-      runner: {
-        command: 'node',
-        args: [
-          '-e',
-          `require('fs').writeFileSync('${outputDir}/out.json', '${JSON.stringify(
-            [
-              {
-                title: 'dummy-title',
-                slug: 'command-object-audit-slug',
-                value: 0,
-                score: 0,
-              },
-            ],
-          )}')`,
+      runner: echoRunnerConfig(
+        [
+          {
+            slug: 'command-object-audit-slug',
+            value: 0,
+            score: 0,
+          },
         ],
-        outputFile: `${outputDir}/out.json`,
-      },
+        join(outputDir, 'out.json'),
+      ),
       groups: [],
       slug: 'command-object-plugin',
       title: 'command-object plugin',

diff --git a/packages/models/test/fixtures/code-pushup.config.mock.js b/packages/models/test/fixtures/code-pushup.config.mock.js
@@ -1,3 +1,6 @@
+import { join } from 'path';
+import { echoRunnerConfig } from './echo-runner-config.mock';
+
 const outputDir = 'tmp';
 export default {
   upload: {
@@ -17,23 +20,16 @@ export default {
           docsUrl: 'http://www.my-docs.dev',
         },
       ],
-      runner: {
-        command: 'node',
-        args: [
-          '-e',
-          `require('fs').writeFileSync('${outputDir}/out.json', '${JSON.stringify(
-            [
-              {
-                title: 'dummy-title',
-                slug: 'command-object-audit-slug',
-                value: 0,
-                score: 0,
-              },
-            ],
-          )}')`,
+      runner: echoRunnerConfig(
+        [
+          {
+            slug: 'command-object-audit-slug',
+            value: 0,
+            score: 0,
+          },
         ],
-        outputFile: `${outputDir}/out.json`,
-      },
+        join(outputDir, 'out.json'),
+      ),
       groups: [],
       slug: 'command-object-plugin',
       title: 'command-object plugin',

diff --git a/packages/models/test/fixtures/code-pushup.config.mock.mjs b/packages/models/test/fixtures/code-pushup.config.mock.mjs
@@ -1,3 +1,6 @@
+import { join } from 'path';
+import { echoRunnerConfig } from './echo-runner-config.mock';
+
 const outputDir = 'tmp';
 export default {
   upload: {
@@ -17,23 +20,16 @@ export default {
           docsUrl: 'http://www.my-docs.dev',
         },
       ],
-      runner: {
-        command: 'node',
-        args: [
-          '-e',
-          `require('fs').writeFileSync('${outputDir}/out.json', '${JSON.stringify(
-            [
-              {
-                title: 'dummy-title',
-                slug: 'command-object-audit-slug',
-                value: 0,
-                score: 0,
-              },
-            ],
-          )}')`,
+      runner: echoRunnerConfig(
+        [
+          {
+            slug: 'command-object-audit-slug',
+            value: 0,
+            score: 0,
+          },
         ],
-        outputFile: `${outputDir}/out.json`,
-      },
+        join(outputDir, 'out.json'),
+      ),
       groups: [],
       slug: 'command-object-plugin',
       title: 'command-object plugin',

diff --git a/packages/models/test/fixtures/code-pushup.config.mock.ts b/packages/models/test/fixtures/code-pushup.config.mock.ts
@@ -1,6 +1,7 @@
+import { join } from 'path';
 import { CoreConfig } from '../../src';
+import { echoRunnerConfig } from './echo-runner-config.mock';
 import { auditReport } from './plugin-config.mock';
-import { runnerConfig } from './runner.mock';
 
 const outputDir = 'tmp';
 export default {
@@ -21,7 +22,7 @@ export default {
           docsUrl: 'http://www.my-docs.dev',
         },
       ],
-      runner: runnerConfig([auditReport()], `${outputDir}/out.json`),
+      runner: echoRunnerConfig([auditReport()], join(outputDir, 'out.json')),
       groups: [],
       slug: 'command-object-plugin',
       title: 'command-object plugin',

diff --git a/packages/models/test/fixtures/echo-runner-config.mock.ts b/packages/models/test/fixtures/echo-runner-config.mock.ts
@@ -0,0 +1,17 @@
+import { platform } from 'os';
+import { AuditOutput, RunnerConfig } from '../../src';
+
+export function echoRunnerConfig(
+  output: AuditOutput[],
+  outputFile: string,
+): RunnerConfig {
+  const auditOutput =
+    platform() === 'win32'
+      ? JSON.stringify(output)
+      : "'" + JSON.stringify(output) + "'";
+  return {
+    command: 'echo',
+    args: [auditOutput, '>', outputFile],
+    outputFile,
+  };
+}
diff --git a/packages/models/test/fixtures/eslint-plugin.mock.ts b/packages/models/test/fixtures/eslint-plugin.mock.ts
@@ -5,8 +5,8 @@ import type {
   PluginConfig,
   PluginReport,
 } from '../../src';
+import { echoRunnerConfig } from './echo-runner-config.mock';
 import { ESLINT_AUDITS_MAP } from './eslint-audits.mock';
-import { runnerConfig } from './runner.mock';
 
 const eslintMeta = {
   slug: 'eslint',
@@ -29,7 +29,7 @@ export function eslintPluginConfig(outputDir = 'tmp'): PluginConfig {
   );
   return {
     ...eslintMeta,
-    runner: runnerConfig(
+    runner: echoRunnerConfig(
       Object.values(ESLINT_AUDITS_MAP),
       join(outputDir, 'eslint-out.json'),
     ),

diff --git a/packages/models/test/fixtures/lighthouse-plugin.mock.ts b/packages/models/test/fixtures/lighthouse-plugin.mock.ts
@@ -1,7 +1,8 @@
+import { join } from 'path';
 import type { AuditGroup, PluginReport } from '../../src';
 import { Audit, PluginConfig } from '../../src';
+import { echoRunnerConfig } from './echo-runner-config.mock';
 import { LIGHTHOUSE_AUDIT_REPORTS_MAP } from './lighthouse-audits.mock';
-import { runnerConfig } from './runner.mock';
 
 const PLUGIN_GROUP_PERFORMANCE: AuditGroup = {
   slug: 'performance',
@@ -50,9 +51,9 @@ export function lighthousePluginConfig(outputDir = 'tmp'): PluginConfig {
   );
   return {
     ...lighthousePluginMeta,
-    runner: runnerConfig(
+    runner: echoRunnerConfig(
       Object.values(LIGHTHOUSE_AUDIT_REPORTS_MAP),
-      `${outputDir}/lighthouse-out.json`,
+      join(outputDir, 'lighthouse-out.json'),
     ),
     audits,
     groups: [PLUGIN_GROUP_PERFORMANCE],

diff --git a/packages/models/test/fixtures/plugin-config.mock.ts b/packages/models/test/fixtures/plugin-config.mock.ts
@@ -1,6 +1,6 @@
 import { join } from 'node:path';
 import { Audit, AuditReport, PluginConfig } from '../../src';
-import { runnerConfig } from './runner.mock';
+import { echoRunnerConfig } from './echo-runner-config.mock';
 
 export function pluginConfig(
   auditOutputs: AuditReport[],
@@ -15,7 +15,7 @@ export function pluginConfig(
     description: 'Plugin description',
     docsUrl: 'https://my-plugin.docs.dev?1',
     audits: auditOutputs.map(auditOutput => auditConfig(auditOutput)),
-    runner: runnerConfig(auditOutputs, pluginOutputPath),
+    runner: echoRunnerConfig(auditOutputs, pluginOutputPath),
     ...(opt || {}),
   } satisfies PluginConfig;
 }