feat(core): scoring data customizable

docs/guides/2-cli.md

@@ -39,6 +39,7 @@ Other options include:
       --stdin-filepath           path to a file to pretend that stdin comes from                                [string]
       --resolver                 path to custom json-ref-resolver instance                                      [string]
   -r, --ruleset                  path/URL to a ruleset file                                                     [string]
+      --scoring-config           path/URL to a scoring config file                                              [string]
   -F, --fail-severity            results of this level or above will trigger a failure exit code
                                                   [string] [choices: "error", "warn", "info", "hint"] [default: "error"]
   -D, --display-only-failures    only output results equal to or greater than --fail-severity [boolean] [default: false]
@@ -60,6 +61,99 @@ Here you can build a [custom ruleset](../getting-started/3-rulesets.md), or exte
 - [OpenAPI ruleset](../reference/openapi-rules.md)
 - [AsyncAPI ruleset](../reference/asyncapi-rules.md)
 
+## Scoring the API
+
+Scoring an API definition is a way to understand at a high level how compliant the API definition is with the rulesets provided. This helps teams to understand the quality of the APIs regarding the definition.
+
+The scoring is produced in two different metrics:
+
+- A number scoring: Calculated by subtracting any error or warning from 100%.
+- A letter scoring, which groups numeric scoring in letters from A to Z, with A being the best score.
+
+Also it introduces a quality gate, were an API scoring below the specific threshold will fail in a pipeline.
+
+Enabling scoring is done using a new parameter called --scoring-config and the scoring configuration file, where you can define how an error or a warning affects to the scoring
+
+Usage:
+
+```bash
+ spectral lint ./reference/**/*.oas*.{json,yml,yaml} --ruleset mycustomruleset.js --scoring-config ./scoringFile.json
+```
+
+Heres an example of this scoringFile config file:
+
+```
+    {
+      "scoringSubtract":
+        {
+          "error":
+           {
+             1:55,
+             2:65,
+             3:75,
+             6:85,
+             10:95
+           }
+          "warn":
+           {
+             1:3,
+             2:7,
+             3:10,
+             6:15,
+             10:18
+           }
+        },
+      "scoringLetter":
+        {
+          "A":75,
+          "B":65,
+          "C":55,
+          "D":45,
+          "E":0
+        },
+      "threshold":50,
+      "onlySubtractHigherSeverityLevel": true,
+      "uniqueErrors": false
+    }
+```
+
+Where:
+
+- scoringSubtract : An object with key/value pair objects for every result level we want to subtract percentage, with the percentage to subtract from number of results on every result type
+- scoringLetter : An object with key/value pairs with scoring letter and scoring percentage, that the result must be greater, for this letter
+- threshold : A number with minimum percentage value to provide valid the file we are checking. Any scoring below this thresold will mark the API as a failure in the scoring.
+- onlySubtractHigherSeverityLevel : A boolean to decide if only the higher severity level who appears in the results for the API to analize, are subtracted from scoring or every severity level are subtracted from scoring.
+
+See sample:
+
+    true
+
+    API with Errors and Warnings, only Errors substract from scoring
+    API with Warnings, Warnings substract from scoring
+
+    false
+
+    API with Errors and Warnings, Errors and Warnings substracts from scoring
+    API with Warnings, Warnings substract from scoring
+
+- uniqueErrors : A boolean to count unique errors or all errors. An error is considered unique if its code and message have not been seen yet
+
+Example:
+
+    With previous scoring config file, if we have:
+
+    1 error, the scoring is 45% and D
+    2 errors, the scoring is 35% and E
+    3 errors, the scoring is 25% and E
+    4 errors, the scoring is 25% and E
+    and so on
+
+Output:
+
+    Below your output log you can see the scoring, like:
+
+    ✖ SCORING: A (93%)
+
 ## Error Results
 
 Spectral has a few different error severities: `error`, `warn`, `info`, and `hint`, and they're in order from highest to lowest. By default, all results are shown regardless of severity, but since v5.0, only the presence of errors causes a failure status code of 1. Seeing results and getting a failure code for it are now two different things.

packages/cli/src/commands/__tests__/lint.test.ts

@@ -146,6 +146,22 @@ describe('lint', () => {
     );
   });
 
+  it('calls lint with document, ruleset and scoring config file', async () => {
+    const doc = './__fixtures__/empty-oas2-document.json';
+    const ruleset = 'custom-ruleset.json';
+    const configFile = 'scoring-config.json';
+    await run(`lint -r ${ruleset} --scoring-config ${configFile} ${doc}`);
+    expect(lint).toBeCalledWith([doc], {
+      encoding: 'utf8',
+      format: ['stylish'],
+      output: { stylish: '<stdout>' },
+      ruleset: 'custom-ruleset.json',
+      stdinFilepath: undefined,
+      ignoreUnknownFormat: false,
+      failOnUnmatchedGlobs: false,
+    });
+  });
+
   it.each(['json', 'stylish'])('calls formatOutput with %s format', async format => {
     await run(`lint -f ${format} ./__fixtures__/empty-oas2-document.json`);
     expect(formatOutput).toBeCalledWith(results, format, { failSeverity: DiagnosticSeverity.Error });
@@ -244,13 +260,13 @@ describe('lint', () => {
     expect(process.stderr.write).nthCalledWith(2, `Error #1: ${chalk.red('some unhandled exception')}\n`);
     expect(process.stderr.write).nthCalledWith(
       3,
-      expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:236`),
+      expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:252`),
     );
 
     expect(process.stderr.write).nthCalledWith(4, `Error #2: ${chalk.red('another one')}\n`);
     expect(process.stderr.write).nthCalledWith(
       5,
-      expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:237`),
+      expect.stringContaining(`packages/cli/src/commands/__tests__/lint.test.ts:253`),
     );
 
     expect(process.stderr.write).nthCalledWith(6, `Error #3: ${chalk.red('original exception')}\n`);

packages/cli/src/commands/lint.ts

@@ -14,6 +14,14 @@ import { formatOutput, writeOutput } from '../services/output';
 import { FailSeverity, ILintConfig, OutputFormat } from '../services/config';
 
 import { CLIError } from '../errors';
+import { ScoringConfig } from './../formatters/types';
+import {
+  getScoringConfig,
+  getScoringLevel,
+  groupBySource,
+  getCountsBySeverity,
+  getUniqueErrors,
+} from '../formatters//utils';
 
 const formatOptions = Object.values(OutputFormat);
 
@@ -127,6 +135,10 @@ const lintCommand: CommandModule = {
           description: 'path/URL to a ruleset file',
           type: 'string',
         },
+        'scoring-config': {
+          description: 'path/URL to a scoring config file',
+          type: 'string',
+        },
         'fail-severity': {
           alias: 'F',
           description: 'results of this level or above will trigger a failure exit code',
@@ -168,6 +180,7 @@ const lintCommand: CommandModule = {
       failSeverity,
       displayOnlyFailures,
       ruleset,
+      scoringConfig,
       stdinFilepath,
       format,
       output,
@@ -197,20 +210,30 @@ const lintCommand: CommandModule = {
         results = filterResultsBySeverity(results, failSeverity);
       }
 
+      const scoringConfigData = await getScoringConfig(scoringConfig);
+
       await Promise.all(
         format.map(f => {
-          const formattedOutput = formatOutput(results, f, { failSeverity: getDiagnosticSeverity(failSeverity) });
+          const formattedOutput = formatOutput(results, f, {
+            failSeverity: getDiagnosticSeverity(failSeverity),
+            scoringConfig: scoringConfigData,
+          });
           return writeOutput(formattedOutput, output?.[f] ?? '<stdout>');
         }),
       );
 
       if (results.length > 0) {
-        process.exit(severeEnoughToFail(results, failSeverity) ? 1 : 0);
+        process.exit(
+          scoringThresholdNotEnough(results, scoringConfigData) ? 1 : severeEnoughToFail(results, failSeverity) ? 1 : 0,
+        );
       } else if (config.quiet !== true) {
         const isErrorSeverity = getDiagnosticSeverity(failSeverity) === DiagnosticSeverity.Error;
         process.stdout.write(
           `No results with a severity of '${failSeverity}' ${isErrorSeverity ? '' : 'or higher '}found!\n`,
         );
+        if (scoringConfig !== void 0) {
+          process.stdout.write(`SCORING: (100%)\nPASSED!`);
+        }
       }
     } catch (ex) {
       fail(isError(ex) ? ex : new Error(String(ex)), config.verbose === true);
@@ -273,6 +296,25 @@ const filterResultsBySeverity = (results: IRuleResult[], failSeverity: FailSever
   return results.filter(r => r.severity <= diagnosticSeverity);
 };
 
+const scoringThresholdNotEnough = (results: IRuleResult[], scoringConfig: ScoringConfig | undefined): boolean => {
+  if (scoringConfig !== void 0) {
+    const groupedResults = groupBySource(results);
+    let groupedUniqueResults = { ...groupedResults };
+    if (scoringConfig.uniqueErrors) {
+      groupedUniqueResults = { ...groupBySource(getUniqueErrors(results)) };
+    }
+    return (
+      scoringConfig.threshold >
+      getScoringLevel(
+        getCountsBySeverity(groupedUniqueResults),
+        scoringConfig.scoringSubtract,
+        scoringConfig.onlySubtractHigherSeverityLevel,
+      )
+    );
+  }
+  return false;
+};
+
 export const severeEnoughToFail = (results: IRuleResult[], failSeverity: FailSeverity): boolean => {
   const diagnosticSeverity = getDiagnosticSeverity(failSeverity);
   return results.some(r => r.severity <= diagnosticSeverity);

packages/cli/src/formatters/json.ts

@@ -1,6 +1,15 @@
-import { Formatter } from './types';
+import { ISpectralDiagnostic } from '@stoplight/spectral-core';
+import { Formatter, FormatterOptions } from './types';
 
-export const json: Formatter = results => {
+import { groupBySource, getUniqueErrors, getCountsBySeverity, getScoringText } from './utils';
+
+export const json: Formatter = (results: ISpectralDiagnostic[], options: FormatterOptions) => {
+  let groupedResults;
+  let scoringText = '';
+  if (options.scoringConfig !== void 0) {
+    groupedResults = groupBySource(getUniqueErrors(results));
+    scoringText = getScoringText(getCountsBySeverity(groupedResults), options.scoringConfig);
+  }
   const outputJson = results.map(result => {
     return {
       code: result.code,
@@ -11,5 +20,16 @@ export const json: Formatter = results => {
       source: result.source,
     };
   });
-  return JSON.stringify(outputJson, null, '\t');
+  let objectOutput;
+  if (options.scoringConfig !== void 0) {
+    const scoring = +(scoringText !== null ? scoringText.replace('%', '').split(/[()]+/)[1] : 0);
+    objectOutput = {
+      scoring: scoringText.replace('SCORING:', '').trim(),
+      passed: scoring >= options.scoringConfig.threshold,
+      results: outputJson,
+    };
+  } else {
+    objectOutput = outputJson;
+  }
+  return JSON.stringify(objectOutput, null, '\t');
 };

packages/cli/src/formatters/pretty.ts

@@ -24,22 +24,33 @@
  * @author Ava Thorn
  */
 
+import { ISpectralDiagnostic } from '@stoplight/spectral-core';
 import { printPath, PrintStyle } from '@stoplight/spectral-runtime';
-import { IDiagnostic, IRange } from '@stoplight/types';
+import { IDiagnostic, IRange, DiagnosticSeverity } from '@stoplight/types';
 import chalk from 'chalk';
 
-import { Formatter } from './types';
-import { getColorForSeverity, getHighestSeverity, getSummary, getSeverityName, groupBySource } from './utils';
+import { Formatter, FormatterOptions } from './types';
+import {
+  getColorForSeverity,
+  getHighestSeverity,
+  getSummary,
+  getSeverityName,
+  groupBySource,
+  getScoringText,
+  getCountsBySeverity,
+  getUniqueErrors,
+} from './utils';
 
 function formatRange(range?: IRange): string {
   if (range === void 0) return '';
 
   return ` ${range.start.line + 1}:${range.start.character + 1}`;
 }
 
-export const pretty: Formatter = results => {
+export const pretty: Formatter = (results: ISpectralDiagnostic[], options: FormatterOptions) => {
   const cliui = require('cliui');
   let output = '\n';
+
   const DEFAULT_TOTAL_WIDTH = process.stdout.columns;
   const COLUMNS = [10, 13, 25, 20, 20];
   const variableColumns = DEFAULT_TOTAL_WIDTH - COLUMNS.reduce((a, b) => a + b);
@@ -50,10 +61,23 @@ export const pretty: Formatter = results => {
   const PAD_TOP1_LEFT0 = [1, 0, 0, 0];
   const ui = cliui({ width: DEFAULT_TOTAL_WIDTH, wrap: true });
 
+  const uniqueResults = getUniqueErrors(results);
   const groupedResults = groupBySource(results);
-  const summaryColor = getColorForSeverity(getHighestSeverity(results));
+  const summaryColor = getColorForSeverity(getHighestSeverity(uniqueResults));
   const summaryText = getSummary(groupedResults);
 
+  let groupedUniqueResults = { ...groupedResults };
+  let scoringColor = '';
+  let scoringText = null;
+
+  if (options.scoringConfig !== void 0) {
+    if (options.scoringConfig.uniqueErrors) {
+      groupedUniqueResults = { ...groupBySource(uniqueResults) };
+    }
+    scoringColor = getColorForSeverity(DiagnosticSeverity.Information);
+    scoringText = getScoringText(getCountsBySeverity(groupedUniqueResults), options.scoringConfig);
+  }
+
   const uniqueIssues: IDiagnostic['code'][] = [];
   Object.keys(groupedResults).forEach(i => {
     const pathResults = groupedResults[i];
@@ -83,6 +107,15 @@ export const pretty: Formatter = results => {
   output += ui.toString();
   output += chalk[summaryColor].bold(`${uniqueIssues.length} Unique Issue(s)\n`);
   output += chalk[summaryColor].bold(`\u2716${summaryText !== null ? ` ${summaryText}` : ''}\n`);
+  if (options.scoringConfig !== void 0) {
+    output += chalk[scoringColor].bold(`\u2716${scoringText !== null ? ` ${scoringText}` : ''}\n`);
+    const scoring = +(scoringText !== null ? scoringText.replace('%', '').split(/[()]+/)[1] : 0);
+    if (scoring >= options.scoringConfig.threshold) {
+      output += chalk['green'].bold(`\u2716 PASSED!\n`);
+    } else {
+      output += chalk['red'].bold(`\u2716 FAILED!\n`);
+    }
+  }
 
   return output;
 };