diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index a40b4232dfce1..4b726a85e3c06 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -69,8 +69,9 @@ jobs: shell: bash - name: Run tests - run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --shortOutput --concurrency=16 --compare=test-workflows/snapshots + run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --githubWorkflow --shortOutput --concurrency=16 --compare=test-workflows/snapshots shell: bash + id: tests env: N8N_ENCRYPTION_KEY: ${{secrets.ENCRYPTION_KEY}} SKIP_STATISTICS_EVENTS: true @@ -98,4 +99,7 @@ jobs: status: ${{ job.status }} channel: '#updates-build-alerts' webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - message: Test workflows failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + message: | + 🛑 Workflow test failed 🛑: + ${{ steps.tests.outputs.slackMessage}} + Sent by *Github Action*: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}| Test workflow> diff --git a/packages/cli/src/commands/Interfaces.d.ts b/packages/cli/src/commands/Interfaces.d.ts index e06bb83abd495..765b16d7aabf6 100644 --- a/packages/cli/src/commands/Interfaces.d.ts +++ b/packages/cli/src/commands/Interfaces.d.ts @@ -1,5 +1,6 @@ interface IResult { totalWorkflows: number; + slackMessage: string; summary: { failedExecutions: number; successfulExecutions: number; diff --git a/packages/cli/src/commands/executeBatch.ts b/packages/cli/src/commands/executeBatch.ts index f0c60bf20f749..413a4894da03e 100644 --- a/packages/cli/src/commands/executeBatch.ts +++ b/packages/cli/src/commands/executeBatch.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-loop-func */ import fs from 'fs'; +import os from 'os'; import { flags } from '@oclif/command'; import type { ITaskData } from 'n8n-workflow'; import { sleep } from 'n8n-workflow'; @@ -35,6 +36,8 @@ export class ExecuteBatch extends BaseCommand { static concurrency = 1; + static githubWorkflow = false; + static debug = false; static executionTimeout = 3 * 60 * 1000; @@ -80,6 +83,12 @@ export class ExecuteBatch extends BaseCommand { description: 'Compares only if attributes output from node are the same, with no regards to nested JSON objects.', }), + + githubWorkflow: flags.boolean({ + description: + 'Enables more lenient comparison for GitHub workflows. This is useful for reducing false positives when comparing Test workflows.', + }), + skipList: flags.string({ description: 'File containing a comma separated list of workflow IDs to skip.', }), @@ -175,7 +184,6 @@ export class ExecuteBatch extends BaseCommand { async run() { // eslint-disable-next-line @typescript-eslint/no-shadow const { flags } = this.parse(ExecuteBatch); - ExecuteBatch.debug = flags.debug; ExecuteBatch.concurrency = flags.concurrency || 1; @@ -251,6 +259,10 @@ export class ExecuteBatch extends BaseCommand { ExecuteBatch.shallow = true; } + if (flags.githubWorkflow) { + ExecuteBatch.githubWorkflow = true; + } + ExecuteBatch.instanceOwner = await getInstanceOwner(); const query = Db.collections.Workflow.createQueryBuilder('workflows'); @@ -369,6 +381,7 @@ export class ExecuteBatch extends BaseCommand { private async runTests(allWorkflows: IWorkflowDb[]): Promise { const result: IResult = { totalWorkflows: allWorkflows.length, + slackMessage: '', summary: { failedExecutions: 0, warningExecutions: 0, @@ -472,11 +485,30 @@ export class ExecuteBatch extends BaseCommand { } await Promise.allSettled(promisesArray); - + if (ExecuteBatch.githubWorkflow) { + if (result.summary.errors.length < 6) { + const errorMessage = result.summary.errors.map((error) => { + return `*${error.workflowId}*: ${error.error}`; + }); + result.slackMessage = `*${ + result.summary.errors.length + } Executions errors*. Workflows failing: ${errorMessage.join(' ')} `; + } else { + result.slackMessage = `*${result.summary.errors.length} Executions errors*`; + } + this.setOutput('slackMessage', JSON.stringify(result.slackMessage)); + } res(result); }); } + setOutput(key: string, value: any) { + // Temporary hack until we move to the new action. + const output = process.env.GITHUB_OUTPUT; + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + fs.appendFileSync(output as unknown as fs.PathOrFileDescriptor, `${key}=${value}${os.EOL}`); + } + updateStatus() { if (ExecuteBatch.cancelled) { return; @@ -756,8 +788,13 @@ export class ExecuteBatch extends BaseCommand { // and search for the `__deleted` string const changesJson = JSON.stringify(changes); if (changesJson.includes('__deleted')) { - // we have structural changes. Report them. - executionResult.error = 'Workflow may contain breaking changes'; + if (ExecuteBatch.githubWorkflow) { + const deletedChanges = changesJson.match(/__deleted/g) ?? []; + // we have structural changes. Report them. + executionResult.error = `Workflow contains ${deletedChanges.length} deleted data.`; + } else { + executionResult.error = 'Workflow may contain breaking changes'; + } executionResult.changes = changes; executionResult.executionStatus = 'error'; } else {