Calculator example: fixing some weird behavior about whitespaces

.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: Test
+
+on: push
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ] # [ windows-latest ] # [ubuntu-latest, macos-latest, windows-latest]
+        node-version: [16] # [12, 16]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+          check-latest: true
+      - run: npm i
+        env: 
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # no used
+      - run: npm test

README.md

@@ -1,6 +1,7 @@
 # [nearley](http://nearley.js.org) ↗️
 [![JS.ORG](https://img.shields.io/badge/js.org-nearley-ffb400.svg?style=flat-square)](http://js.org)
 [![npm version](https://badge.fury.io/js/nearley.svg)](https://badge.fury.io/js/nearley)
+[![Test](https://github.com//crguezl/nearley/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/crguezl/nearley/actions/workflows/test.yaml)
 
 nearley is a simple, fast and powerful parsing toolkit. It consists of:
 1. [A powerful, modular DSL for describing

examples/calculator/arithmetic.ne

@@ -1,62 +1,91 @@
-# This is a nice little grammar to familiarize yourself
-# with the nearley syntax.
+# How to Implement Lexical Analysis in Tools Whithout a separated Lexer
+# To use it run: 
+# nearleyc arithmetic-parenthesis.ne -o grammar.js && export NODE_PATH=$NODE_PATH:`npm root -g` && node calculator.js
+# This is a nice little grammar to familiarize yourself with the nearley syntax.
 
-# It parses valid calculator input, obeying OOO and stuff.
+# It parses valid calculator input
 #   ln (3 + 2*(8/e - sin(pi/5)))
 # is valid input.
 
-# This is (hopefully) pretty self-evident.
+@{%
+const bin = (([x, op, y]) => op(x,y));
+const Null = (d => null);
+const fac = n => (n===0)?1:n*fac(n-1);
+const unaryPost = (([p, op]) => op(p));
+const funApply = ([fun, arg]) => fun(arg);
+%}
 
-# `main` is the nonterminal that nearley tries to parse, so
-# we define it first.
-# The _'s are defined as whitespace below. This is a mini-
-# -idiom.
-
-main -> _ AS _ {% function(d) {return d[1]; } %}
+main => null {% d => "" %} # Allow for empty lines
+    | AS _ {% function(d) {return d[0]; } %}
 
 # PEMDAS!
 # We define each level of precedence as a nonterminal.
 
-# Parentheses
-P -> "(" _ AS _ ")" {% function(d) {return d[2]; } %}
-    | N             {% id %}
+# Addition and subtraction
+AS -> AS PLUS MD  {% bin %}  # Prefer this syntax
+    | AS MINUS MD {% bin %}
+    | MD          {% id %}
+
+# Multiplication and division
+MD -> MD MULT E  {% bin %}
+    | MD DIV E   {% bin %}
+    | E          {% id %}
 
 # Exponents
-E -> P _ "^" _ E    {% function(d) {return Math.pow(d[0], d[4]); } %}
-    | P             {% id %}
+E -> F EXP E    {% bin %}
+    | F         {% id %}
 
-# Multiplication and division
-MD -> MD _ "*" _ E  {% function(d) {return d[0]*d[4]; } %}
-    | MD _ "/" _ E  {% function(d) {return d[0]/d[4]; } %}
-    | E             {% id %}
+# Factorial 
+F ->  P FACTORIAL    {% unaryPost %}
+    | P              {% id %} 
 
-# Addition and subtraction
-AS -> AS _ "+" _ MD {% function(d) {return d[0]+d[4]; } %}
-    | AS _ "-" _ MD {% function(d) {return d[0]-d[4]; } %}
-    | MD            {% id %}
-
-# A number or a function of a number
-N -> float          {% id %}
-    | "sin" _ P     {% function(d) {return Math.sin(d[2]); } %}
-    | "cos" _ P     {% function(d) {return Math.cos(d[2]); } %}
-    | "tan" _ P     {% function(d) {return Math.tan(d[2]); } %}
-
-    | "asin" _ P    {% function(d) {return Math.asin(d[2]); } %}
-    | "acos" _ P    {% function(d) {return Math.acos(d[2]); } %}
-    | "atan" _ P    {% function(d) {return Math.atan(d[2]); } %}
-
-    | "pi"          {% function(d) {return Math.PI; } %}
-    | "e"           {% function(d) {return Math.E; } %}
-    | "sqrt" _ P    {% function(d) {return Math.sqrt(d[2]); } %}
-    | "ln" _ P      {% function(d) {return Math.log(d[2]); }  %}
+# Fixed "bug" sinpi
+
+P -> Q
+    | FLOAT     {% id %}
+    | SIN  Q    {% funApply %}
+    | COS Q     {% funApply %}
+    | TAN Q     {% funApply %}
+    | ASIN Q    {% funApply %}
+    | ACOS Q    {% funApply %}
+    | ATAN Q    {% funApply %}
+    | PI        {% id %}
+    | EULER     {% id %}
+    | SQRT Q    {% funApply %}
+    | LN Q      {% funApply %}
+
+# Parentheses
+Q ->  LP AS RP  {% ([lp, as, rp]) => as %}
+
+##### LEXICAL ANALYSIS #################################################
 
 # I use `float` to basically mean a number with a decimal point in it
+FLOAT -> _ float   {% d => d[1] %} 
 float ->
-      int "." int   {% function(d) {return parseFloat(d[0] + d[1] + d[2])} %}
-	| int           {% function(d) {return parseInt(d[0])} %}
+      int "." int  {% function(d) {return parseFloat(d[0] + d[1] + d[2])} %}
+	| int          {% function(d) {return parseInt(d[0])} %}
 
-int -> [0-9]:+        {% function(d) {return d[0].join(""); } %}
+int -> [0-9]:+     {% function(d) {return d[0].join(""); } %}
 
 # Whitespace. The important thing here is that the postprocessor
 # is a null-returning function. This is a memory efficiency trick.
-_ -> [\s]:*     {% function(d) {return null; } %}
+_ -> [\s]:*        {% function(d) {return null; } %}
+
+PLUS -> _ "+"      {% function(d) {return ((a,b) => a+b); } %}
+MINUS -> _ "-"     {% function(d) {return ((a,b) => a-b); } %}
+MULT -> _ "*"      {% function(d) {return ((a,b) => a*b); } %}
+DIV -> _ "/"       {% function(d) {return ((a,b) => a/b); } %}
+EXP -> _ "^"       {% function(d) {return ((a,b) => Math.pow(a,b)); } %}
+FACTORIAL -> "!"   {% d => fac %}
+LP -> _ "("        {% Null %}
+RP -> _ ")"        {% Null %}
+SIN -> _ "sin"i    {% d => Math.sin %}
+COS -> _ "cos"i    {% d => Math.cos %}
+TAN -> _ "tan"i    {% d => Math.tan %}
+ASIN -> _ "asin"i  {% d => Math.asin %}
+ACOS -> _ "acos"i  {% d => Math.acos %}
+ATAN -> _ "atan"i  {% d => Math.atan %}
+PI -> _ "pi"i      {% d => Math.PI %}
+EULER -> _ "e"i    {% d => Math.E  %}
+SQRT -> _ "sqrt"i  {% d => Math.sqrt %}
+LN -> _ "ln"i      {% d => Math.log %}

examples/calculator/grammar.js

@@ -1,48 +1,78 @@
-// Generated automatically by nearley
+// Generated automatically by nearley, version 2.20.1
 // http://github.com/Hardmath123/nearley
 (function () {
-function id(x) {return x[0]; }
+function id(x) { return x[0]; }
+
+const bin = (([x, op, y]) => op(x,y));
+const Null = (d => null);
+const fac = n => (n===0)?1:n*fac(n-1);
+const unaryPost = (([p, op]) => op(p));
+const funApply = ([fun, arg]) => fun(arg);
 var grammar = {
+    Lexer: undefined,
     ParserRules: [
-    {"name": "main", "symbols": ["_", "AS", "_"], "postprocess": function(d) {return d[1]; }},
-    {"name": "P", "symbols": [{"literal":"("}, "_", "AS", "_", {"literal":")"}], "postprocess": function(d) {return d[2]; }},
-    {"name": "P", "symbols": ["N"], "postprocess": id},
-    {"name": "E", "symbols": ["P", "_", {"literal":"^"}, "_", "E"], "postprocess": function(d) {return Math.pow(d[0], d[4]); }},
-    {"name": "E", "symbols": ["P"], "postprocess": id},
-    {"name": "MD", "symbols": ["MD", "_", {"literal":"*"}, "_", "E"], "postprocess": function(d) {return d[0]*d[4]; }},
-    {"name": "MD", "symbols": ["MD", "_", {"literal":"/"}, "_", "E"], "postprocess": function(d) {return d[0]/d[4]; }},
-    {"name": "MD", "symbols": ["E"], "postprocess": id},
-    {"name": "AS", "symbols": ["AS", "_", {"literal":"+"}, "_", "MD"], "postprocess": function(d) {return d[0]+d[4]; }},
-    {"name": "AS", "symbols": ["AS", "_", {"literal":"-"}, "_", "MD"], "postprocess": function(d) {return d[0]-d[4]; }},
+    {"name": "main", "symbols": [], "postprocess": d => ""},
+    {"name": "main", "symbols": ["AS", "_"], "postprocess": function(d) {return d[0]; }},
+    {"name": "AS", "symbols": ["AS", "PLUS", "MD"], "postprocess": bin},
+    {"name": "AS", "symbols": ["AS", "MINUS", "MD"], "postprocess": bin},
     {"name": "AS", "symbols": ["MD"], "postprocess": id},
-    {"name": "N", "symbols": ["float"], "postprocess": id},
-    {"name": "N$string$1", "symbols": [{"literal":"s"}, {"literal":"i"}, {"literal":"n"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$1", "_", "P"], "postprocess": function(d) {return Math.sin(d[2]); }},
-    {"name": "N$string$2", "symbols": [{"literal":"c"}, {"literal":"o"}, {"literal":"s"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$2", "_", "P"], "postprocess": function(d) {return Math.cos(d[2]); }},
-    {"name": "N$string$3", "symbols": [{"literal":"t"}, {"literal":"a"}, {"literal":"n"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$3", "_", "P"], "postprocess": function(d) {return Math.tan(d[2]); }},
-    {"name": "N$string$4", "symbols": [{"literal":"a"}, {"literal":"s"}, {"literal":"i"}, {"literal":"n"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$4", "_", "P"], "postprocess": function(d) {return Math.asin(d[2]); }},
-    {"name": "N$string$5", "symbols": [{"literal":"a"}, {"literal":"c"}, {"literal":"o"}, {"literal":"s"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$5", "_", "P"], "postprocess": function(d) {return Math.acos(d[2]); }},
-    {"name": "N$string$6", "symbols": [{"literal":"a"}, {"literal":"t"}, {"literal":"a"}, {"literal":"n"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$6", "_", "P"], "postprocess": function(d) {return Math.atan(d[2]); }},
-    {"name": "N$string$7", "symbols": [{"literal":"p"}, {"literal":"i"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$7"], "postprocess": function(d) {return Math.PI; }},
-    {"name": "N", "symbols": [{"literal":"e"}], "postprocess": function(d) {return Math.E; }},
-    {"name": "N$string$8", "symbols": [{"literal":"s"}, {"literal":"q"}, {"literal":"r"}, {"literal":"t"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$8", "_", "P"], "postprocess": function(d) {return Math.sqrt(d[2]); }},
-    {"name": "N$string$9", "symbols": [{"literal":"l"}, {"literal":"n"}], "postprocess": function joiner(d) {return d.join('');}},
-    {"name": "N", "symbols": ["N$string$9", "_", "P"], "postprocess": function(d) {return Math.log(d[2]); }},
+    {"name": "MD", "symbols": ["MD", "MULT", "E"], "postprocess": bin},
+    {"name": "MD", "symbols": ["MD", "DIV", "E"], "postprocess": bin},
+    {"name": "MD", "symbols": ["E"], "postprocess": id},
+    {"name": "E", "symbols": ["F", "EXP", "E"], "postprocess": bin},
+    {"name": "E", "symbols": ["F"], "postprocess": id},
+    {"name": "F", "symbols": ["P", "FACTORIAL"], "postprocess": unaryPost},
+    {"name": "F", "symbols": ["P"], "postprocess": id},
+    {"name": "P", "symbols": ["Q"]},
+    {"name": "P", "symbols": ["FLOAT"], "postprocess": id},
+    {"name": "P", "symbols": ["SIN", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["COS", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["TAN", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["ASIN", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["ACOS", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["ATAN", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["PI"], "postprocess": id},
+    {"name": "P", "symbols": ["EULER"], "postprocess": id},
+    {"name": "P", "symbols": ["SQRT", "Q"], "postprocess": funApply},
+    {"name": "P", "symbols": ["LN", "Q"], "postprocess": funApply},
+    {"name": "Q", "symbols": ["LP", "AS", "RP"], "postprocess": ([lp, as, rp]) => as},
+    {"name": "FLOAT", "symbols": ["_", "float"], "postprocess": d => d[1]},
     {"name": "float", "symbols": ["int", {"literal":"."}, "int"], "postprocess": function(d) {return parseFloat(d[0] + d[1] + d[2])}},
     {"name": "float", "symbols": ["int"], "postprocess": function(d) {return parseInt(d[0])}},
     {"name": "int$ebnf$1", "symbols": [/[0-9]/]},
-    {"name": "int$ebnf$1", "symbols": [/[0-9]/, "int$ebnf$1"], "postprocess": function arrconcat(d) {return [d[0]].concat(d[1]);}},
+    {"name": "int$ebnf$1", "symbols": ["int$ebnf$1", /[0-9]/], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}},
     {"name": "int", "symbols": ["int$ebnf$1"], "postprocess": function(d) {return d[0].join(""); }},
     {"name": "_$ebnf$1", "symbols": []},
-    {"name": "_$ebnf$1", "symbols": [/[\s]/, "_$ebnf$1"], "postprocess": function arrconcat(d) {return [d[0]].concat(d[1]);}},
-    {"name": "_", "symbols": ["_$ebnf$1"], "postprocess": function(d) {return null; }}
+    {"name": "_$ebnf$1", "symbols": ["_$ebnf$1", /[\s]/], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}},
+    {"name": "_", "symbols": ["_$ebnf$1"], "postprocess": function(d) {return null; }},
+    {"name": "PLUS", "symbols": ["_", {"literal":"+"}], "postprocess": function(d) {return ((a,b) => a+b); }},
+    {"name": "MINUS", "symbols": ["_", {"literal":"-"}], "postprocess": function(d) {return ((a,b) => a-b); }},
+    {"name": "MULT", "symbols": ["_", {"literal":"*"}], "postprocess": function(d) {return ((a,b) => a*b); }},
+    {"name": "DIV", "symbols": ["_", {"literal":"/"}], "postprocess": function(d) {return ((a,b) => a/b); }},
+    {"name": "EXP", "symbols": ["_", {"literal":"^"}], "postprocess": function(d) {return ((a,b) => Math.pow(a,b)); }},
+    {"name": "FACTORIAL", "symbols": [{"literal":"!"}], "postprocess": d => fac},
+    {"name": "LP", "symbols": ["_", {"literal":"("}], "postprocess": Null},
+    {"name": "RP", "symbols": ["_", {"literal":")"}], "postprocess": Null},
+    {"name": "SIN$subexpression$1", "symbols": [/[sS]/, /[iI]/, /[nN]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "SIN", "symbols": ["_", "SIN$subexpression$1"], "postprocess": d => Math.sin},
+    {"name": "COS$subexpression$1", "symbols": [/[cC]/, /[oO]/, /[sS]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "COS", "symbols": ["_", "COS$subexpression$1"], "postprocess": d => Math.cos},
+    {"name": "TAN$subexpression$1", "symbols": [/[tT]/, /[aA]/, /[nN]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "TAN", "symbols": ["_", "TAN$subexpression$1"], "postprocess": d => Math.tan},
+    {"name": "ASIN$subexpression$1", "symbols": [/[aA]/, /[sS]/, /[iI]/, /[nN]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "ASIN", "symbols": ["_", "ASIN$subexpression$1"], "postprocess": d => Math.asin},
+    {"name": "ACOS$subexpression$1", "symbols": [/[aA]/, /[cC]/, /[oO]/, /[sS]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "ACOS", "symbols": ["_", "ACOS$subexpression$1"], "postprocess": d => Math.acos},
+    {"name": "ATAN$subexpression$1", "symbols": [/[aA]/, /[tT]/, /[aA]/, /[nN]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "ATAN", "symbols": ["_", "ATAN$subexpression$1"], "postprocess": d => Math.atan},
+    {"name": "PI$subexpression$1", "symbols": [/[pP]/, /[iI]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "PI", "symbols": ["_", "PI$subexpression$1"], "postprocess": d => Math.PI},
+    {"name": "EULER$subexpression$1", "symbols": [/[eE]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "EULER", "symbols": ["_", "EULER$subexpression$1"], "postprocess": d => Math.E},
+    {"name": "SQRT$subexpression$1", "symbols": [/[sS]/, /[qQ]/, /[rR]/, /[tT]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "SQRT", "symbols": ["_", "SQRT$subexpression$1"], "postprocess": d => Math.sqrt},
+    {"name": "LN$subexpression$1", "symbols": [/[lL]/, /[nN]/], "postprocess": function(d) {return d.join(""); }},
+    {"name": "LN", "symbols": ["_", "LN$subexpression$1"], "postprocess": d => Math.log}
 ]
   , ParserStart: "main"
 }

lib/nearley.d.ts

@@ -0,0 +1,120 @@
+// Type definitions for runtime classes
+//  - Parser (and ParserOptions)
+//  - Lexer (and Token, LexerState)
+//  - Grammar (and Rule, ParserRule, CompiledRules)
+//
+// Based on https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/nearley
+//
+// Definitions by: Nikita Litvin <https://github.com/deltaidea>
+//                 BendingBender <https://github.com/BendingBender>
+
+export as namespace nearley;
+
+export class Parser {
+    /**
+     * Reserved token for indicating a parse fail.
+     */
+    static readonly fail: {};
+
+    grammar: Grammar;
+    options: ParserOptions;
+    lexer: Lexer;
+    lexerState?: LexerState;
+    current: number;
+    /**
+     * An array of possible parsings. Each element is the thing returned by your grammar.
+     *
+     * Note that this is undefined before the first feed() call.
+     * It isn't typed as `any[] | undefined` to spare you the null checks when it's
+     * definitely an array.
+     */
+    results: any[];
+
+    constructor(grammar: Grammar, options?: ParserOptions);
+    constructor(rules: CompiledRules, start: string, options?: ParserOptions);
+
+    /**
+     * The Parser object can be fed data in parts with .feed(data).
+     * You can then find an array of parsings with the .results property.
+     * If results is empty, then there are no parsings.
+     * If results contains multiple values, then that combination is ambiguous.
+     *
+     * @throws If there are no possible parsings, nearley will throw an error
+     * whose `offset` property is the index of the offending token.
+     */
+    feed(chunk: string): this;
+    finish(): any[];
+    restore(column: {[key: string]: any, lexerState: LexerState}): void;
+    save(): {[key: string]: any, lexerState: LexerState};
+}
+
+export interface ParserOptions {
+    keepHistory?: boolean;
+    lexer?: Lexer;
+}
+
+export class Rule {
+    static highestId: number;
+
+    id: number;
+    name: string;
+    symbols: any[];
+    postprocess?: Postprocessor;
+
+    constructor(name: string, symbols: any[], postprocess?: Postprocessor);
+
+    toString(withCursorAt?: number): string;
+}
+
+export class Grammar {
+    static fromCompiled(rules: CompiledRules): Grammar;
+
+    rules: Rule[];
+    start: string;
+    byName: {[ruleName: string]: Rule[]};
+    lexer?: Lexer;
+
+    constructor(rules: Rule[]);
+}
+
+export interface CompiledRules {
+    Lexer?: Lexer;
+    ParserStart: string;
+    ParserRules: ParserRule[];
+}
+
+export interface ParserRule {
+    name: string;
+    symbols: any[];
+    postprocess?: Postprocessor;
+}
+
+export type Postprocessor = (data: any[], reference?: number, wantedBy?: {}) => void;
+
+export interface Lexer {
+    /**
+     * Sets the internal buffer to data, and restores line/col/state info taken from save().
+     */
+    reset(data: string, state?: LexerState): void;
+    /**
+     * Returns e.g. {type, value, line, col, …}. Only the value attribute is required.
+     */
+    next(): Token | undefined;
+    /**
+     * Returns an object describing the current line/col etc. This allows us
+     * to preserve this information between feed() calls, and also to support Parser#rewind().
+     * The exact structure is lexer-specific; nearley doesn't care what's in it.
+     */
+    save(): LexerState;
+    /**
+     * Returns a string with an error message describing the line/col of the offending token.
+     * You might like to include a preview of the line in question.
+     */
+    formatError(token: Token, message: string): string;
+}
+
+export type Token = string | { value: string; };
+
+export interface LexerState {
+    [x: string]: any;
+}