feat: add parse package (#406)

Fixes #405

packages/parse/.eslintignore

@@ -0,0 +1 @@
+dist

packages/parse/.eslintrc.cjs

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: ['../../.eslintrc-ts-common.cjs']
+}

packages/parse/.gitignore

@@ -0,0 +1,3 @@
+dist
+docs/*
+!docs/index.md

packages/parse/.prettierignore

@@ -0,0 +1,3 @@
+dist
+docs
+CHANGELOG.md

packages/parse/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Climate Interactive / New Venture Fund
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/parse/README.md

@@ -0,0 +1,8 @@
+# @sdeverywhere/parse
+
+This package contains the parsing layer used by the [SDEverywhere](https://github.com/climateinteractive/SDEverywhere) compiler.
+It defines an AST (abstract syntax tree) structure that can be used to express a system dynamics model, and provides an API for parsing a model into an AST structure.
+Currently the only implemented input format is Vensim's `mdl` format, but support for the XMILE format is under discussion.
+
+Note: The `parse` API has not yet stabilized, and the package is primarily intended as an implementation detail of the `compile` package, so documentation is not provided at this time.
+If you would like to help with the task of stabilizing and formalizing the API for external consumption, please get in touch on the [discussion board](https://github.com/climateinteractive/SDEverywhere/discussions).

packages/parse/docs/index.md

@@ -0,0 +1,4 @@
+# @sdeverywhere/parse
+
+Note: The `parse` API has not yet stabilized, so documentation is not provided at this time.
+If you would like to help with this task, please get in touch on the [discussion board](https://github.com/climateinteractive/SDEverywhere/discussions).

packages/parse/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@sdeverywhere/parse",
+  "version": "0.1.0",
+  "files": [
+    "dist/**"
+  ],
+  "type": "module",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "clean": "rm -rf dist",
+    "lint": "eslint src --ext .ts --max-warnings 0",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write .",
+    "precommit": "../../scripts/precommit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ci": "vitest run",
+    "type-check": "tsc --noEmit -p tsconfig-test.json",
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "docs": "../../scripts/gen-docs.js",
+    "ci:build": "run-s clean lint prettier:check type-check test:ci build docs"
+  },
+  "dependencies": {
+    "antlr4": "4.12.0",
+    "antlr4-vensim": "0.6.2",
+    "assert-never": "^1.2.1",
+    "split-string": "^6.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.7"
+  },
+  "author": "Climate Interactive",
+  "license": "MIT",
+  "homepage": "https://sdeverywhere.org",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/climateinteractive/SDEverywhere.git",
+    "directory": "packages/parse"
+  },
+  "bugs": {
+    "url": "https://github.com/climateinteractive/SDEverywhere/issues"
+  }
+}

packages/parse/src/_shared/names.js

@@ -0,0 +1,42 @@
+// Copyright (c) 2023 Climate Interactive / New Venture Fund
+
+/**
+ * Format a model variable name into a valid C identifier (with special characters
+ * converted to underscore).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "Variable name".
+ * @returns {string} The C identifier for the given name, e.g., "_variable_name".
+ */
+export function canonicalName(name) {
+  // TODO: This is also defined in the compile package.  Would be good to
+  // define it in one place to reduce the chance of them getting out of sync.
+  return (
+    '_' +
+    name
+      .trim()
+      .replace(/"/g, '_')
+      .replace(/\s+!$/g, '!')
+      .replace(/\s/g, '_')
+      .replace(/,/g, '_')
+      .replace(/-/g, '_')
+      .replace(/\./g, '_')
+      .replace(/\$/g, '_')
+      .replace(/'/g, '_')
+      .replace(/&/g, '_')
+      .replace(/%/g, '_')
+      .replace(/\//g, '_')
+      .replace(/\|/g, '_')
+      .toLowerCase()
+  )
+}
+
+/**
+ * Format a model function name into a valid C identifier (with special characters
+ * converted to underscore).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "FUNCTION NAME".
+ * @returns {string} The C identifier for the given name, e.g., "_FUNCTION_NAME".
+ */
+export function cFunctionName(name) {
+  return canonicalName(name).toUpperCase()
+}

packages/parse/src/ast/ast-builders.ts

@@ -0,0 +1,249 @@
+// Copyright (c) 2023 Climate Interactive / New Venture Fund
+
+import { canonicalName, cFunctionName } from '../_shared/names'
+
+import type {
+  BinaryOp,
+  BinaryOpExpr,
+  DimensionDef,
+  DimName,
+  DimOrSubName,
+  Equation,
+  Expr,
+  FunctionCall,
+  FunctionName,
+  Keyword,
+  LookupCall,
+  LookupDef,
+  LookupPoint,
+  LookupRange,
+  Model,
+  NumberLiteral,
+  ParensExpr,
+  StringLiteral,
+  SubName,
+  SubscriptMapping,
+  SubscriptRef,
+  UnaryOp,
+  UnaryOpExpr,
+  VariableDef,
+  VariableName,
+  VariableRef
+} from './ast-types'
+
+//
+// NOTE: This file contains functions that allow for tersely defining AST nodes.
+// It is intended for internal use only (primarily in tests), so it is not exported
+// as part of the public API at this time.
+//
+
+//
+// DIMENSIONS + SUBSCRIPTS
+//
+
+export function subRef(dimOrSubName: DimOrSubName): SubscriptRef {
+  return {
+    subName: dimOrSubName,
+    subId: canonicalName(dimOrSubName)
+  }
+}
+
+export function subMapping(toDimName: DimName, dimOrSubNames: DimOrSubName[] = []): SubscriptMapping {
+  return {
+    toDimName,
+    toDimId: canonicalName(toDimName),
+    subscriptRefs: dimOrSubNames.map(subRef)
+  }
+}
+
+export function dimDef(
+  dimName: DimName,
+  familyName: DimName,
+  dimOrSubNames: SubName[],
+  subscriptMappings: SubscriptMapping[] = [],
+  comment = ''
+): DimensionDef {
+  return {
+    dimName,
+    dimId: canonicalName(dimName),
+    familyName,
+    familyId: canonicalName(familyName),
+    subscriptRefs: dimOrSubNames.map(subRef),
+    subscriptMappings,
+    comment
+  }
+}
+
+//
+// EXPRESSIONS
+//
+
+export function num(value: number, text?: string): NumberLiteral {
+  return {
+    kind: 'number',
+    value,
+    text: text || value.toString()
+  }
+}
+
+export function stringLiteral(text: string): StringLiteral {
+  return {
+    kind: 'string',
+    text
+  }
+}
+
+export function keyword(text: string): Keyword {
+  return {
+    kind: 'keyword',
+    text
+  }
+}
+
+export function varRef(varName: VariableName, subscriptNames?: DimOrSubName[]): VariableRef {
+  return {
+    kind: 'variable-ref',
+    varName,
+    varId: canonicalName(varName),
+    subscriptRefs: subscriptNames?.map(subRef)
+  }
+}
+
+export function unaryOp(op: UnaryOp, expr: Expr): UnaryOpExpr {
+  return {
+    kind: 'unary-op',
+    op,
+    expr
+  }
+}
+
+export function binaryOp(lhs: Expr, op: BinaryOp, rhs: Expr): BinaryOpExpr {
+  return {
+    kind: 'binary-op',
+    lhs,
+    op,
+    rhs
+  }
+}
+
+export function parens(expr: Expr): ParensExpr {
+  return {
+    kind: 'parens',
+    expr
+  }
+}
+
+export function lookupDef(points: LookupPoint[], range?: LookupRange): LookupDef {
+  return {
+    kind: 'lookup-def',
+    range,
+    points
+  }
+}
+
+export function lookupCall(varRef: VariableRef, arg: Expr): LookupCall {
+  return {
+    kind: 'lookup-call',
+    varRef,
+    arg
+  }
+}
+
+export function call(fnName: FunctionName, ...args: Expr[]): FunctionCall {
+  return {
+    kind: 'function-call',
+    fnName,
+    fnId: cFunctionName(fnName),
+    args
+  }
+}
+
+//
+// EQUATIONS
+//
+
+export function varDef(
+  varName: VariableName,
+  subscriptNames?: DimOrSubName[],
+  exceptSubscriptNames?: DimOrSubName[][]
+): VariableDef {
+  return {
+    kind: 'variable-def',
+    varName,
+    varId: canonicalName(varName),
+    subscriptRefs: subscriptNames?.map(subRef),
+    exceptSubscriptRefSets: exceptSubscriptNames?.map(namesForSet => namesForSet.map(subRef))
+  }
+}
+
+export function exprEqn(varDef: VariableDef, expr: Expr, units = '', comment = ''): Equation {
+  return {
+    lhs: {
+      varDef
+    },
+    rhs: {
+      kind: 'expr',
+      expr
+    },
+    units,
+    comment
+  }
+}
+
+export function constListEqn(varDef: VariableDef, constants: NumberLiteral[][], units = '', comment = ''): Equation {
+  // For now, assume that the original text had a trailing semicolon if there are multiple groups
+  let text = constants.map(arr => arr.map(constant => constant.text).join(',')).join(';')
+  if (constants.length > 1) {
+    text += ';'
+  }
+  return {
+    lhs: {
+      varDef
+    },
+    rhs: {
+      kind: 'const-list',
+      constants: constants.flat(),
+      text
+    },
+    units,
+    comment
+  }
+}
+
+export function dataVarEqn(varDef: VariableDef, units = '', comment = ''): Equation {
+  return {
+    lhs: {
+      varDef
+    },
+    rhs: {
+      kind: 'data'
+    },
+    units,
+    comment
+  }
+}
+
+export function lookupVarEqn(varDef: VariableDef, lookupDef: LookupDef, units = '', comment = ''): Equation {
+  return {
+    lhs: {
+      varDef
+    },
+    rhs: {
+      kind: 'lookup',
+      lookupDef
+    },
+    units,
+    comment
+  }
+}
+
+//
+// MODEL
+//
+
+export function model(dimensions: DimensionDef[], equations: Equation[]): Model {
+  return {
+    dimensions,
+    equations
+  }
+}