feat(parser): rewrite lexer to make it faster (#50)

* feat(parser): first iteration of new lexer

* feat(parser): convert token string props to number props

* refactor(parser): optimize char grabber

* refactor(parser): working on new lexer

* refactor(parser): convert token string props to number props

* refactor(parser): rebuild lexer, add tag attrs parsing

* refactor(parser): rework word parsing and tag parsing

* refactor(parser): rework to pass tests

* refactor(parser): rework tag parsing

* refactor(parser): rework escape tags parsing

* refactor(parser): rework tests

* refactor(parser): all test pass

* refactor(parser): make lexer faster by move mode switching in loop

* refactor(parser): remove all state map objects

* refactor(parser): order of parsing states

* refactor(parser): state switching without return

* refactor(parser): rename buffers to chars

* refactor(lexer): reduce function calls

* feat(lexer): add new parser tests and code to pass it

* fix(utils): remove unused variable in char grabber

* feat(lexer): add test for new lexer bug

* chore(*): add lexer and lexer2 to benchmark

* chore(lexer): add some debug info for char grabber

* feat(parser): add new test for single attributes without values

* fix(lexer): paired tags tests

* refactor(lexer): comment breaking changes tests for future releases

* feat(core): improve tests

* refactor(parser): add more tests, reduce char grabber size

* refactor(parser): reduce utils size

* refactor(parser): remove unused code from tag parsing code

* refactor(parser): remove unused code from word to tag transforming code

* chore(benchmark): fix benchmark imports

benchmark/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 const Benchmark = require('benchmark');
 const stub = require('./test/stub');
 
@@ -38,9 +39,22 @@ suite
       addInLineBreaks: false,
     });
   })
-  .add('@bbob/parser', () => require('../packages/bbob-parser/lib/index').parse(stub, {
-    onlyAllowTags: ['ch'],
-  }))
+  .add('@bbob/parser lexer old', () => {
+    const lexer1 = require('../packages/bbob-parser/lib/lexer_old');
+
+    return require('../packages/bbob-parser/lib/index').parse(stub, {
+      onlyAllowTags: ['ch'],
+      createTokenizer: lexer1.createLexer,
+    });
+  })
+  .add('@bbob/parser lexer', () => {
+    const lexer2 = require('../packages/bbob-parser/lib/lexer');
+
+    return require('../packages/bbob-parser/lib/index').parse(stub, {
+      onlyAllowTags: ['ch'],
+      createTokenizer: lexer2.createLexer,
+    });
+  })
 // add listeners
   .on('cycle', (event) => {
     console.log(String(event.target));

packages/bbob-core/test/utils.test.js

@@ -1,4 +1,7 @@
-import { iterate } from '../src/utils';
+import { iterate, match } from '../src/utils';
+
+const stringify = val => JSON.stringify(val);
+
 
 describe('@bbob/core utils', () => {
   test('iterate', () => {
@@ -16,14 +19,47 @@ describe('@bbob/core utils', () => {
       return node;
     });
 
-    expect(resultArr).toEqual([{
-      one: true,
-      pass: 1,
-      content: [{ oneInside: true, pass: 1, }]
-    }, {
-      two: true,
-      pass: 1,
-      content: [{ twoInside: true, pass: 1, }]
-    }]);
+    const expected = [
+      {
+        one: true,
+        content: [{ oneInside: true, pass: 1, }],
+        pass: 1,
+      }, {
+        two: true,
+        content: [{ twoInside: true, pass: 1, }],
+        pass: 1,
+      }
+    ];
+
+    expect(stringify(resultArr)).toEqual(stringify(expected));
   });
+  test('match', () => {
+    const testArr = [
+      { tag: 'mytag1', one: 1 },
+      { tag: 'mytag2', two: 1 },
+      { tag: 'mytag3', three: 1 },
+      { tag: 'mytag4', four: 1 },
+      { tag: 'mytag5', five: 1 },
+      { tag: 'mytag6', six: 1 },
+    ];
+
+    testArr.match = match;
+
+    const resultArr = testArr.match([{ tag: 'mytag1' }, { tag: 'mytag2' }], node => {
+      node.pass = 1;
+
+      return node;
+    });
+
+    const expected = [
+      { tag: 'mytag1', one: 1, pass: 1 },
+      { tag: 'mytag2', two: 1, pass: 1 },
+      { tag: 'mytag3', three: 1 },
+      { tag: 'mytag4', four: 1 },
+      { tag: 'mytag5', five: 1 },
+      { tag: 'mytag6', six: 1 },
+    ];
+
+    expect(stringify(resultArr)).toEqual(stringify(expected))
+  })
 });

packages/bbob-parser/package.json

@@ -35,7 +35,7 @@
     "build:es": "../../node_modules/.bin/cross-env BABEL_ENV=es NODE_ENV=production ../../node_modules/.bin/babel src --out-dir es",
     "build:umd": "../../node_modules/.bin/cross-env BABEL_ENV=rollup NODE_ENV=production ../../node_modules/.bin/rollup --config ../../rollup.config.js",
     "build": "npm run build:commonjs && npm run build:es && npm run build:umd",
-    "test": "../../node_modules/.bin/jest --",
+    "test": "../../node_modules/.bin/jest",
     "cover": "../../node_modules/.bin/jest --coverage",
     "lint": "../../node_modules/.bin/eslint .",
     "size": "../../node_modules/.bin/cross-env NODE_ENV=production ../../node_modules/.bin/size-limit",

packages/bbob-parser/src/Token.js

@@ -10,12 +10,12 @@ const TOKEN_VALUE_ID = 'value'; // 1;
 const TOKEN_COLUMN_ID = 'row'; // 2;
 const TOKEN_LINE_ID = 'line'; // 3;
 
-const TOKEN_TYPE_WORD = 'word';
-const TOKEN_TYPE_TAG = 'tag';
-const TOKEN_TYPE_ATTR_NAME = 'attr-name';
-const TOKEN_TYPE_ATTR_VALUE = 'attr-value';
-const TOKEN_TYPE_SPACE = 'space';
-const TOKEN_TYPE_NEW_LINE = 'new-line';
+const TOKEN_TYPE_WORD = 1; // 'word';
+const TOKEN_TYPE_TAG = 2; // 'tag';
+const TOKEN_TYPE_ATTR_NAME = 3; // 'attr-name';
+const TOKEN_TYPE_ATTR_VALUE = 4; // 'attr-value';
+const TOKEN_TYPE_SPACE = 5; // 'space';
+const TOKEN_TYPE_NEW_LINE = 6; // 'new-line';
 
 /**
  * @param {Token} token
@@ -105,14 +105,15 @@ class Token {
    * @param row
    */
   constructor(type, value, line, row) {
-    this[TOKEN_TYPE_ID] = String(type);
+    this[TOKEN_TYPE_ID] = Number(type);
     this[TOKEN_VALUE_ID] = String(value);
     this[TOKEN_LINE_ID] = Number(line);
     this[TOKEN_COLUMN_ID] = Number(row);
   }
 
   isEmpty() {
-    return !!this[TOKEN_TYPE_ID];
+    // eslint-disable-next-line no-restricted-globals
+    return isNaN(this[TOKEN_TYPE_ID]);
   }
 
   isText() {