Merge pull request #12 from superindustries/feature/script-parsing-su…

…pport

Feature/script parsing support

package.json

@@ -27,7 +27,7 @@
     "lint": "eslint src/",
     "format": "prettier -c src/",
     "format:fix": "prettier --write src/",
-    "prepush": "yarn test && yarn lint && yarn format"
+    "prepush": "yarn test --clear-cache && yarn test && yarn lint && yarn format"
   },
   "devDependencies": {
     "@types/jest": "^25.2.1",

src/language/error.ts

@@ -158,6 +158,24 @@ function generateErrorVisualization(
   };
 }
 
+export const enum SyntaxErrorCategory {
+  /** Lexer token error */
+  LEXER,
+  /** Parser rule error */
+  PARSER,
+  /** Jessie syntax error */
+  JESSIE_SYNTAX,
+  /** Jessie forbidden construct error */
+  JESSIE_FORBIDDEN_CONSTRUCT,
+}
+
+export type ProtoError = {
+  readonly relativeSpan: Span;
+  readonly detail?: string;
+  readonly category: SyntaxErrorCategory;
+  readonly hint?: string;
+};
+
 export class SyntaxError {
   /** Additional message attached to the error. */
   readonly detail: string;
@@ -169,7 +187,11 @@ export class SyntaxError {
     readonly location: Location,
     /** Span of the error. */
     readonly span: Span,
-    detail?: string
+    /** Category of this error. */
+    readonly category: SyntaxErrorCategory,
+    detail?: string,
+    /** Optional hint that is emitted to help with the resolution. */
+    readonly hint?: string
   ) {
     this.detail = detail ?? 'Invalid or unexpected token';
   }
@@ -220,6 +242,7 @@ export class SyntaxError {
       source,
       location,
       span,
+      SyntaxErrorCategory.PARSER,
       `Expected ${expected} but found ${actual}`
     );
   }
@@ -236,7 +259,9 @@ export class SyntaxError {
     const locationLinePrefix = ' '.repeat(maxLineNumberLog) + '--> ';
     const locationLine = `${locationLinePrefix}${this.source.fileName}:${sourceLocation.line}:${sourceLocation.column}`;
 
-    return `${errorLine}\n${locationLine}\n${visualization}\n`;
+    const maybeHint = this.hint ? `Hint: ${this.hint}\n` : '';
+
+    return `${errorLine}\n${locationLine}\n${visualization}\n${maybeHint}`;
   }
 
   get message(): string {

src/language/jessie/glue.ts

@@ -0,0 +1,32 @@
+import {
+  JessieSyntaxProtoError,
+  transpileScript,
+} from './transpiler/transpiler';
+import {
+  ForbiddenConstructProtoError,
+  validateScript,
+} from './validator/validator';
+
+type Success = { output: string; sourceMap: string };
+
+/**
+ * Validates and transpiles Jessie script.
+ *
+ * This functions combines the transpiler and validator in a more efficient way
+ */
+export function validateAndTranspile(
+  input: string
+): Success | JessieSyntaxProtoError | ForbiddenConstructProtoError {
+  const { output, sourceMap, syntaxProtoError } = transpileScript(input, true);
+
+  if (syntaxProtoError) {
+    return syntaxProtoError;
+  }
+
+  const subsetErrors = validateScript(input);
+  if (subsetErrors.length > 0) {
+    return subsetErrors[0]; // TODO
+  }
+
+  return { output, sourceMap };
+}

src/language/jessie/index.ts

@@ -0,0 +1 @@
+export { validateAndTranspile } from './glue';

src/language/jessie/transpiler/transpiler.test.ts

@@ -0,0 +1,37 @@
+import * as st from './transpiler';
+
+test('transpiler basics', () => {
+  const { output, sourceMap } = st.transpileScript(
+    `let a = { hello: 1, world: 2 + "3" }
+console.log(a)`
+  );
+
+  expect(output).toBe(
+    `var a = { hello: 1, world: 2 + "3" };
+console.log(a);`
+  );
+
+  expect(sourceMap).toBe(
+    'AAAA,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;AACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA'
+  );
+});
+
+test('transpiler ES2020', () => {
+  const { output, sourceMap } = st.transpileScript(
+    `let nullishCoalescing = undefined ?? (false ?? "truthy")
+		const optionalChaining = console?.log?.(nullishCoalescing)`
+  );
+
+  expect(output).toMatch('var nullishCoalescing =');
+  expect(output).toMatch('var optionalChaining =');
+  expect(output).toMatch(
+    'undefined !== null && undefined !== void 0 ? undefined'
+  );
+  expect(output).toMatch(
+    'console === null || console === void 0 ? void 0 : console.log'
+  );
+
+  expect(sourceMap).toMatch(
+    /[;,]?([a-zA-Z_+]|[a-zA-Z_+]{4}|[a-zA-Z_+]{5})([;,]([a-zA-Z_+]|[a-zA-Z_+]{4}|[a-zA-Z_+]{5}))*/
+  );
+});

src/language/jessie/transpiler/transpiler.ts

@@ -0,0 +1,81 @@
+import * as ts from 'typescript';
+
+import { ProtoError, SyntaxErrorCategory } from '../../error';
+
+const SCRIPT_OUTPUT_TARGET = ts.ScriptTarget.ES3;
+
+const AFTER_TRANSFORMERS: ts.TransformerFactory<ts.SourceFile>[] = [];
+
+export type JessieSyntaxProtoError = ProtoError & {
+  category: SyntaxErrorCategory.JESSIE_SYNTAX;
+};
+
+export function transpileScript(
+  input: string,
+  reportDiagnostics: true
+): {
+  output: string;
+  sourceMap: string;
+  syntaxProtoError: JessieSyntaxProtoError;
+};
+export function transpileScript(
+  input: string,
+  reportDiagnostics?: false
+): { output: string; sourceMap: string };
+
+export function transpileScript(
+  input: string,
+  reportDiagnostics?: boolean
+): {
+  output: string;
+  sourceMap: string;
+  syntaxProtoError?: JessieSyntaxProtoError;
+} {
+  // This will transpile the code, generate a source map and run transformers
+  const { outputText, diagnostics, sourceMapText } = ts.transpileModule(input, {
+    compilerOptions: {
+      allowJs: true,
+      target: SCRIPT_OUTPUT_TARGET,
+      sourceMap: true,
+    },
+    transformers: {
+      after: AFTER_TRANSFORMERS,
+    },
+    reportDiagnostics,
+  });
+
+  // Strip the source mapping comment from the end of the output
+  const outputTextStripped = outputText
+    .replace('//# sourceMappingURL=module.js.map', '')
+    .trimRight();
+
+  // `sourceMapText` will be here because we requested it by setting the compiler flag
+  if (!sourceMapText) {
+    throw 'Source map text is not present';
+  }
+  const sourceMapJson: { mappings: string } = JSON.parse(sourceMapText);
+
+  let syntaxProtoError: JessieSyntaxProtoError | undefined;
+  if (diagnostics && diagnostics.length > 0) {
+    const diag = diagnostics[0];
+    let detail = diag.messageText;
+    if (typeof detail === 'object') {
+      detail = detail.messageText;
+    }
+
+    syntaxProtoError = {
+      category: SyntaxErrorCategory.JESSIE_SYNTAX,
+      relativeSpan: {
+        start: diag.start ?? 0,
+        end: (diag.start ?? 0) + (diag.length ?? 1),
+      },
+      detail,
+    };
+  }
+
+  return {
+    output: outputTextStripped,
+    syntaxProtoError,
+    sourceMap: sourceMapJson.mappings,
+  };
+}

src/validator/ScriptValidator.test.ts → src/language/jessie/validator/validator.test.ts

@@ -1,4 +1,4 @@
-import { validateScript, ValidationError } from './ScriptValidator';
+import { ForbiddenConstructProtoError, validateScript } from './validator';
 
 // Declare custom matcher for sake of Typescript
 declare global {
@@ -12,13 +12,13 @@ declare global {
 // Add the actual custom matcher
 expect.extend({
   toBeValidScript(script: string, ...errors: string[]) {
-    function formatError(err: ValidationError): string {
+    function formatError(err: ForbiddenConstructProtoError): string {
       const hint = err.hint ?? 'not provided';
 
-      return `${err.message} (hint: ${hint})`;
+      return `${err.detail} (hint: ${hint})`;
     }
 
-    const report = validateScript(script);
+    const protoErrors = validateScript(script);
 
     let pass = true;
     let message = '';
@@ -27,14 +27,14 @@ expect.extend({
       // Expecting to fail
       pass = false; // Flip
 
-      if (report.isValid === true) {
+      if (protoErrors.length === 0) {
         pass = !pass;
         message = 'expected to fail';
       } else {
         for (let i = 0; i < errors.length; i++) {
-          const err = report.errors[i];
+          const err = protoErrors[i];
           if (
-            !err.message.includes(errors[i]) &&
+            !err.detail.includes(errors[i]) &&
             !err.hint?.includes(errors[i])
           ) {
             pass = !pass;
@@ -47,9 +47,9 @@ expect.extend({
       }
     } else {
       // Expecting to pass
-      if (report.isValid === false) {
+      if (protoErrors.length > 0) {
         pass = !pass;
-        const messages = report.errors
+        const messages = protoErrors
           .map(err => `\n\t${formatError(err)}`)
           .join('');
         message = `expected to pass, errors: ${messages}`;

src/language/jessie/validator/validator.ts

@@ -0,0 +1,97 @@
+import * as ts from 'typescript';
+
+import { ProtoError, SyntaxErrorCategory } from '../../error';
+import { ALLOWED_SYNTAX, FORBIDDEN_CONSTRUCTS } from './constructs';
+
+function constructDebugVisualTree(root: ts.Node): string {
+  let debugTree = '';
+  let debugDepth = 0;
+
+  function nodeVisitor<T extends ts.Node>(node: T): void {
+    const nodeCode = node.getText().replace('\r\n', ' ').trim();
+    const treeIndent = ''.padStart(debugDepth, '\t');
+    debugTree += `${treeIndent}NODE ${ts.SyntaxKind[node.kind]} "${nodeCode}"`;
+
+    // Go over forbidden constructs and check if any of them applies
+    let anyRuleBroken = false;
+    const rules = FORBIDDEN_CONSTRUCTS[node.kind] ?? [];
+    for (const rule of rules) {
+      if (rule.predicate?.(node) ?? true) {
+        anyRuleBroken = true;
+      }
+    }
+    if (anyRuleBroken) {
+      debugTree += ' [R]';
+    }
+
+    // If none of the rules applied, but the syntax is not valid anyway, add an error without a hint
+    if (!anyRuleBroken && !ALLOWED_SYNTAX.includes(node.kind)) {
+      debugTree += ' [S]';
+    }
+
+    debugTree += '\n';
+
+    // Recurse into children
+    debugDepth += 1;
+    ts.forEachChild(node, nodeVisitor);
+    debugDepth -= 1;
+  }
+  nodeVisitor(root);
+
+  return debugTree;
+}
+
+export type ForbiddenConstructProtoError = ProtoError & {
+  detail: string;
+  category: SyntaxErrorCategory.JESSIE_FORBIDDEN_CONSTRUCT;
+};
+export function validateScript(input: string): ForbiddenConstructProtoError[] {
+  const errors: ForbiddenConstructProtoError[] = [];
+
+  const rootNode = ts.createSourceFile(
+    'scripts.js',
+    input,
+    ts.ScriptTarget.ES2015,
+    true,
+    ts.ScriptKind.JS
+  );
+
+  function nodeVisitor<T extends ts.Node>(node: T): void {
+    // Go over forbidden constructs and check if any of them applies
+    let anyRuleBroken = false;
+    const rules = FORBIDDEN_CONSTRUCTS[node.kind] ?? [];
+    for (const rule of rules) {
+      if (rule.predicate?.(node) ?? true) {
+        anyRuleBroken = true;
+
+        errors.push({
+          detail: `${ts.SyntaxKind[node.kind]} construct is not supported`,
+          hint: rule.hint(input, node),
+          relativeSpan: { start: node.pos, end: node.end },
+          category: SyntaxErrorCategory.JESSIE_FORBIDDEN_CONSTRUCT,
+        });
+      }
+    }
+
+    // If none of the rules applied, but the syntax is not valid anyway, add an error without a hint
+    if (!anyRuleBroken && !ALLOWED_SYNTAX.includes(node.kind)) {
+      errors.push({
+        detail: `${ts.SyntaxKind[node.kind]} construct is not supported`,
+        relativeSpan: { start: node.pos, end: node.end },
+        category: SyntaxErrorCategory.JESSIE_FORBIDDEN_CONSTRUCT,
+      });
+    }
+
+    // Recurse into children
+    ts.forEachChild(node, nodeVisitor);
+  }
+  nodeVisitor(rootNode);
+
+  if (process.env.LOG_LEVEL === 'debug') {
+    if (errors.length > 0) {
+      console.debug(constructDebugVisualTree(rootNode));
+    }
+  }
+
+  return errors;
+}

src/language/lexer/index.ts

@@ -1,2 +1,2 @@
-export { Lexer } from './lexer';
+export { Lexer, LexerContext, LexerTokenKindFilter } from './lexer';
 export { LexerToken, LexerTokenData, LexerTokenKind } from './token';