generate test cases from our styleguide's README.md

package.json

@@ -4,7 +4,8 @@
   "description": "A mostly reasonable approach to JavaScript.",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "publish-all": "npm publish && cd ./packages/eslint-config-airbnb && npm publish"
+    "publish-all": "npm publish && cd ./packages/eslint-config-airbnb && npm publish",
+    "script": "./node_modules/.bin/babel-node ./script/index.js"
   },
   "repository": {
     "type": "git",
@@ -24,5 +25,16 @@
   "bugs": {
     "url": "https://github.com/airbnb/javascript/issues"
   },
-  "homepage": "https://github.com/airbnb/javascript"
+  "homepage": "https://github.com/airbnb/javascript",
+  "devDependencies": {
+    "babel": "^5.8.21",
+    "concat-stream": "^1.5.0",
+    "glob": "^5.0.14",
+    "locus": "^1.0.4",
+    "mdast": "^1.1.0",
+    "mdast-util-heading-range": "^1.0.1",
+    "mdast-util-to-string": "^1.0.0",
+    "unist-util-inspect": "^1.0.0",
+    "unist-util-visit": "^1.0.0"
+  }
 }

packages/eslint-config-airbnb/package.json

@@ -30,6 +30,7 @@
     "babel-tape-runner": "1.2.0",
     "eslint": "1.1.0",
     "eslint-plugin-react": "3.2.3",
+    "marked": "0.3.5",
     "react": "0.13.3",
     "tape": "4.2.0"
   }

packages/eslint-config-airbnb/test/generateTestCases.js

@@ -0,0 +1,131 @@
+/**
+ * this file loads code examples from markdown README.
+ */
+const marked = require('marked');
+const path = require('path');
+const fs = require('fs');
+
+const GOOD = '// good\n';
+const BAD = '// bad\n';
+const BEST = '// best\n';
+
+export const README_PATH = path.normalize(path.join(__dirname, '../../../README.md'));
+
+const DELIM_MAP = {
+  [GOOD]: 'good',
+  [BAD]: 'bad',
+  [BEST]: 'best',
+};
+
+const DELIMETERS = [
+  GOOD,
+  BAD,
+  BEST,
+];
+
+function log(...things) {
+  if (true) return;
+  console.error(...things);
+}
+
+/**
+ * given a block of JS code with // good and // bad delimiters,
+ * split out the good and bad code, and return them.
+ *
+ * Returns an object of shape { good :: Array<String>, bad :: Array<string> }
+ */
+function extractExamples(text) {
+  // TODO this won't lint, switch to es6 :)
+  const delimiter = new RegExp(`(${DELIMETERS.join('|')})`);
+  const result = DELIMETERS.reduce((memo, delim) => {
+    memo[DELIM_MAP[delim]] = [];
+    return memo;
+  }, {});
+  log(result);
+  const parts = text.split(delimiter).filter(Boolean);
+  let currentDelimeter = null;
+  let first = true;
+
+  parts.forEach(part => {
+    log(`saw part "${part}"`);
+    if (DELIMETERS.indexOf(part) >= 0) {
+      // this is a delimeter
+      currentDelimeter = part;
+      first = false;
+      log('  its a delimiter');
+      return;
+    }
+
+    if (currentDelimeter) {
+      result[DELIM_MAP[currentDelimeter]].push(part);
+      log(`added to result[${currentDelimeter}]: `, result[currentDelimeter]);
+      currentDelimeter = null;
+      return;
+    }
+
+    if (first) {
+      result.prepend = part;
+      first = false;
+      log('  was first');
+    }
+
+    log('  skipped.');
+  });
+
+  return result;
+}
+
+function buildExamples(extracted) {
+  if (!(extracted.good.length || extracted.bad.length || extracted.best.length)) {
+    // it was a code block without examples.
+    return null;
+  }
+
+  const good = extracted.good.map(code => `${extracted.prepend || ''}\n${code}`);
+  const bad = extracted.bad.map(code => `${extracted.prepend || ''}\n${code}`);
+  const best = extracted.best.map(code => `${extracted.prepend || ''}\n${code}`);
+
+  return { good, bad, best };
+}
+
+function group(arrayOfExamples) {
+  const result = {
+    good: [],
+    bad: [],
+    best: [],
+  };
+
+  return arrayOfExamples.filter(Boolean).reduce((result, examples) => {
+    result.good = result.good.concat(examples.good);
+    result.bad = result.bad.concat(examples.bad);
+    result.best = result.best.concat(examples.best);
+    return result;
+  }, result);
+}
+
+/**
+ * parse out the grouped code examples from some markdown text
+ */
+export function analyze(markdown) {
+  const tokens = marked.lexer(markdown);
+  const codes = tokens.filter(token => token.type === 'code' && token.lang === 'javascript')
+    .map(token => token.text);
+  const extracted = codes.map(extractExamples);
+  const examples = extracted.map(buildExamples);
+  const grouped = group(examples);
+  log(grouped);
+  return grouped;
+}
+
+export function loadReadme() {
+  return fs.readFileSync(README_PATH, 'utf-8');
+}
+
+function main() {
+  const readme = loadReadme();
+  analyze(readme);
+}
+
+if (require.main === module) {
+  main();
+}

packages/eslint-config-airbnb/test/helpers.js

@@ -0,0 +1,15 @@
+import { CLIEngine } from 'eslint';
+import eslintrc from '../';
+
+export function makeLint(config = eslintrc) {
+  const cli = new CLIEngine({
+    useEslintrc: false,
+    baseConfig: eslintrc,
+  });
+
+  // @see http://eslint.org/docs/developer-guide/nodejs-api.html#executeonfiles
+  // @see http://eslint.org/docs/developer-guide/nodejs-api.html#executeontext
+  return function lint(text) {
+    return cli.executeOnText(text).results[0];
+  };
+}

packages/eslint-config-airbnb/test/test-README.js

@@ -0,0 +1,74 @@
+/**
+ * this may be too ambitious.
+ *
+ * 1. Parse our styleguide for code examples.
+ * 2. split out "good", "bad", and "best" examples using cursory parsing
+ *
+ * For each example:
+ *  - fail a test for anything that isn't valid javascript. This indicates we
+ *    need to improve the format of the README and the code of generateTestCases.
+ *
+ * For good:
+ * For best:
+ *  - fail if it doesn't lint.
+ *
+ * For bad:
+ *  - fail if it lints successfully.
+ *
+ * Lots of things will fail to start with, but it gives us a view on how
+ * true-to-spec our .eslintrc is.
+ */
+import { makeLint } from './helpers';
+import { analyze, loadReadme } from './generateTestCases';
+import test from 'tape';
+
+const lint = makeLint();
+const examples = analyze(loadReadme());
+
+// run callback for each example
+// callback gets parameters (t, example, lintResult)
+function testEachExample(t, testName, address, examples, callback) {
+  t.test(testName, t => {
+    examples.forEach((example, idx) => {
+      t.test(`${address}[${idx}]`, t => {
+        // always make sure that the test is valid JS code.
+        const lintResult = lint(example);
+        const fatalErrors = lintResult.messages.filter(m => m.fatal);
+        // deepEquals give us a nice print out of the errors themselves.
+        t.deepEquals(fatalErrors, [], 'there are no fatal parsing/linting errors');
+
+        // callback to run other tests
+        callback(t, example, lintResult);
+      });
+    });
+  });
+}
+
+test('GENERATED TESTS FROM README.MD', t => {
+
+  testEachExample(t, 'good examples pass linting', 'examples.good', examples.good,
+    (t, example, lintResult) => {
+      const hasRules = lintResult.messages.filter(m => !!m.ruleId);
+      t.deepEquals(hasRules, [], 'no messages with rules (perfect validation)');
+      t.end();
+    }
+  );
+
+  testEachExample(t, 'best examples pass linting', 'examples.best', examples.best,
+    (t, example, lintResult) => {
+      const hasRules = lintResult.messages.filter(m => !!m.ruleId);
+      t.deepEquals(hasRules, [], 'no messages with rules (perfect validation)');
+      t.end();
+    }
+  );
+
+  testEachExample(t, 'bad examples fail linting', 'examples.bad', examples.bad,
+    (t, example, lintResult) => {
+      const hasRules = lintResult.messages.filter(m => !!m.ruleId);
+      t.ok(lintResult.errorCount || lintResult.warningCount, 'has warnings or errors');
+      t.equal(hasRules.length, lintResult.errorCount + lintResult.warningCount,
+             'has exactly zero syntax errors while having other errors');
+      t.end();
+    }
+  );
+});

packages/eslint-config-airbnb/test/test-react-order.js

@@ -1,17 +1,8 @@
 import test from 'tape';
-import { CLIEngine } from 'eslint';
 import eslintrc from '../';
+import { makeLint } from './helpers';
 
-const cli = new CLIEngine({
-  useEslintrc: false,
-  baseConfig: eslintrc,
-});
-
-function lint(text) {
-  // @see http://eslint.org/docs/developer-guide/nodejs-api.html#executeonfiles
-  // @see http://eslint.org/docs/developer-guide/nodejs-api.html#executeontext
-  return cli.executeOnText(text).results[0];
-}
+const lint = makeLint();
 
 function wrapComponent(body) {
   return `