LSP Server Support

Jakefile.js

@@ -142,6 +142,20 @@ task("web", [], function() {
   });
 },{async:true})
 
+//-----------------------------------------------------
+// Tasks: api 
+//-----------------------------------------------------
+desc("build madoko api")
+task("api", [], function() {
+  // fixVersion("web/client/editor.html");
+  var args = Array.prototype.slice.call(arguments).join(" ")
+  var cmd = kokaCmd + " -v -l " + args + " " + "api"
+  jake.logger.log("> " + cmd);
+  jake.exec(cmd, {interactive: true}, function(err) {
+    complete();
+  });
+},{async:true})
+
 var localTexDir = "c:/texlive/texmf-local/tex/latex/local";
 desc("setup web");
 task("justcopy", [], function() {

madoko.d.ts

@@ -0,0 +1,37 @@
+declare module 'madoko' {
+  type Options = any;
+  type Position = { path: string, line: number }
+  type Range = { path: string, from: number, to: number }
+  type ReferenceInfo = {
+      id: string,
+      element: string,
+      caption: string,
+      position?: Position
+  }
+  type DocumentInfo = {
+      labels: ReferenceInfo[],
+      blocks: Block[],
+      context: any,
+      log: string
+  }
+  type Block = {
+    kind: string,
+    // the element id, empty if not present
+    id: string,
+    // the tag name, empty if not present
+    name: string,
+    // the child nodes
+    content: Block[],
+    // the element classes
+    classes: string[],
+    // the annotated attributes
+    attributes: any,
+    // the range (lines are 1-based and inclusive)
+    range?: Range
+  }
+
+  export function analyze(
+      inputName: string, 
+      content: string,
+      callback: (DocumentInfo) => any): any;
+}

package.json

@@ -1,24 +1,26 @@
 {
-  "name"        : "madoko",
-  "author"      : "Daan Leijen, Microsoft Corp.",
-  "version"     : "1.1.6",
-  "homepage"    : "http://madoko.codeplex.com",
-  "description" : "Madoko is a fast scholarly Markdown processor written in Koka",
-  "licenses": [{
-    "type"      : "Apache License 2.0", 
-    "url"       : "http://www.apache.org/licenses/LICENSE-2.0.html"
-  }],
+  "name": "madoko",
+  "author": "Daan Leijen, Microsoft Corp.",
+  "version": "1.1.6",
+  "homepage": "http://madoko.codeplex.com",
+  "description": "Madoko is a fast scholarly Markdown processor written in Koka",
+  "licenses": [
+    {
+      "type": "Apache License 2.0",
+      "url": "http://www.apache.org/licenses/LICENSE-2.0.html"
+    }
+  ],
   "keywords": [
-      "Madoko",
-      "Markdown",
-      "LaTeX",
-      "Koka", 
-      "Microsoft",
-      "Scholarly",
-      "Academic"
+    "Madoko",
+    "Markdown",
+    "LaTeX",
+    "Koka",
+    "Microsoft",
+    "Scholarly",
+    "Academic"
   ],
   "preferGlobal": true,
-  "main": "./lib/madoko.js",
+  "main": "./lib/api.js",
   "bin": {
     "madoko": "./lib/cli.js"
   },
@@ -33,19 +35,20 @@
     "readme.md",
     "license.txt"
   ],
-  "engines" : { 
-    "node": ">=0.10.0" 
+  "types": "madoko.d.ts",
+  "engines": {
+    "node": ">=0.10.0"
   },
   "dependencies": {
-    "amdefine"    : ">=0.0.4",    
-    "requirejs"   : ">=2.1",
-    "mkdirp"      : ">=0.3.5"
+    "amdefine": ">=0.0.4",
+    "requirejs": ">=2.1",
+    "mkdirp": ">=0.3.5"
   },
   "devDependencies": {
-    "jake"        : ">=0.7.6"
+    "jake": ">=0.7.6"
   },
   "repository": {
-    "type": "hg", 
-    "url" : "https://hg.codeplex.com/madoko"
+    "type": "hg",
+    "url": "https://hg.codeplex.com/madoko"
   }
 }

src/api.kk

@@ -0,0 +1,234 @@
+module api
+
+import std/dict
+import std/regex
+import std/log
+import std/path
+
+import backports
+import version
+
+import block        // just for the type block
+import formatBlock  // just for formatContext
+import common       // for label, lineMap, and attrs
+import inline       // for parseLineInfo
+import driver       // for outputName, processContentLSP
+import metadata     // for parseMeta
+import madoko       // for normalizeSource, lastPathSegment
+import definitions  // for parseBody
+import options
+import includes
+
+
+// Types that are part of the public API
+// -------------------------------------
+
+// TODO also migrate to range
+public struct position (path: string, line: int)
+
+// Line information is 1-based and inclusive.
+public struct range (path: string, from: int, to: int)
+
+public struct referenceInfo (
+  id: string,
+  element: string,
+  caption: string,
+  position: null<position>
+)
+
+public struct documentInfo (
+  labels: vector<referenceInfo>,
+  blocks: vector<blockInfo>,
+  // just for debugging purposes we also include the original formatContext
+  context : formatContext,
+  log: string
+)
+
+// the type of block nodes we export to js / typescript
+public struct blockInfo (
+  // the block kind
+  kind: string,
+  // the element id, empty if not present
+  id: string,
+  // the tag name, empty if not present
+  name: string,
+  // the child nodes
+  content: vector<blockInfo>,
+  // the text content
+  text: string,
+  // the classes
+  classes: vector<string>,
+  // attributes of the block node (like "caption")
+  attributes: dict<string>,
+  // TODO also include position and range information.
+  position: null<range>
+)
+
+// Main Entrypoint
+// ---------------
+
+// Runs the frontend to parse and analyze the document but does not
+// generate an html or tex document.
+//
+// Resolves includes and also processes those files.
+//
+// TODO Currently the logs are very sparse... Did I omit too much of the
+// processing to catch errors?
+public function analyze(
+  inputName : string, content : string, continue : (documentInfo) -> io ()
+) : io () {
+  printRedirect( fun(s) { log("stdout", s) });
+
+  withLog("stdout") {
+    processContent(inputName, content) fun (blocks, ctx) {
+      val stdout = getLog("stdout")
+      val labeledElems = ctx.inlineContext.labels.list().map(labelInfo)
+      val res = DocumentInfo(vector(labeledElems), blocks.toBlockInfos(inputName), ctx, stdout)
+      continue(res)
+    }
+  }
+  ()
+}
+
+
+function processContent(
+  inName : string, content : string,
+  continue : (list<block>, formatContext) -> io ()
+) : io () {
+  // is this ever used?
+  val outName = "/tmp/madoko/out.mdk"
+
+  val searchDirs = [inName.dirname]
+
+  val opts0 = Options(
+    version=version/version,
+    lineNoWeb=True,
+    math=Mathoptions(mode = Dynamic), // don't generate math latex
+    embedLimit=0,
+    verbose=1,
+    copyStyles=False)
+
+  // running fastMode destroys positioning
+  val fastMode = False
+
+  content.include(fastMode, inName, outName, searchDirs, opts0) fun(icontent0,lmap) {
+    // remove madoko comments
+    val icontent = icontent0 //.removeMadokoComments
+
+    val opts1  = opts0(lineMap=lmap,
+                       processTimeout=if (opts0.sandbox) then 60000 else opts0.processTimeout,
+                       metadata=opts0.metadata)
+
+    // Just extract metadata and symbols (copied from markdownNormal)
+    // first normalize the input: all tabs to 4 spaces.
+    val (options, src) = parseMeta(opts1, FmtHtml, icontent.normalizeSource)
+
+    val icontext = inlineContext(FmtHtml,
+      options.metadata.dict,
+      options.math.dim,
+      options.embedinfos,
+      options.citestyle.maybe(citeNumeric, id),
+      options.sanitize,
+      options.bench,
+      options.verbose,
+      options.math.mode.isStatic,
+      options.highlight,
+      options.starBold,
+      options.sandbox,
+      options.prettyAlign)
+
+    val (warns0, (blocks, fcontext)) = withLog("warning") {
+      parseBody(
+        initialFormatContext(icontext, options.lineMap, options.headingBase, options.pedantic, FmtHtml),
+        options.lineNo,
+        src,
+        options.metadata, options.tocDepth, options.sectionBase, options.sectionMax)
+    }
+    // TODO don't collect warnings stringly typed.
+    log("stdout", warns0)
+    // val warns = fixWarnings(warns0)
+    // if (warns != "") {
+    //   log("stdout", warns)
+    // }
+
+    continue(blocks, fcontext)
+  }
+}
+
+
+// Computing position and range information
+// ---------------------------------------
+
+function range(attr: attrs, inputName: string): maybe<range> {
+  match (attr.hasKey("data-line-start")) {
+    Just(start) -> catch({
+      val end = attr.lookupKey("data-line-end", start)
+      val (path, startLine) = start.extractPosition(inputName)
+      val (_, endLine) = end.extractPosition(inputName)
+      Just(Range(path, startLine, endLine))
+    }, fun(exn) { Nothing })
+    Nothing -> Nothing
+  }
+}
+
+function extractPosition(locationString: string, inputName: string): exn (string, int) {
+  val locs = (inputName + ":" + locationString).split(";").list
+  val last = locs.last.split(":")
+  (last[0], last[1].parseInt.unJust)
+}
+
+// we need the inputName to normalize position information
+function toBlockInfos(bs: list<block>, inputName: string): div vector<blockInfo> {
+  vector(bs.map(fun(b){ b.toBlockInfo(inputName) }).concatMaybe)
+}
+
+// TODO also include range information in extracted block structure
+function toBlockInfo(b: block, inputName: string): div maybe<blockInfo> {
+  match(b) {
+    HLine( attrs ) ->
+      Just(BlockInfo("hline", attrs.name, attrs.elem, vector(), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Para( text, attrs ) ->
+      Just(BlockInfo("para", attrs.name, attrs.elem, vector(), text, vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Code( text, _, attrs) ->
+      Just(BlockInfo("code", attrs.name, attrs.elem, vector(), text, vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Quote( content, attrs) ->
+      Just(BlockInfo("quote", attrs.name, attrs.elem, content.toBlockInfos(inputName), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    List( _, content, attrs) ->
+      Just(BlockInfo("list", attrs.name, attrs.elem, content.toBlockInfos(inputName), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Item( content, attrs) ->
+      Just(BlockInfo("item", attrs.name, attrs.elem, content.toBlockInfos(inputName), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Heading( _, text, attrs) ->
+      Just(BlockInfo("heading", attrs.name, attrs.elem, vector(), text, vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    // currently table cells are ignored
+    Table( _, _, _, attrs) ->
+      Just(BlockInfo("table", attrs.name, attrs.elem, vector(), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Div( content, attrs) ->
+      Just(BlockInfo("div", attrs.name, attrs.elem, content.toBlockInfos(inputName), "", vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    Source( text, _, attrs) ->
+      Just(BlockInfo("source", attrs.name, attrs.elem, vector(), text, vector(attrs.classes), dict(attrs.keyvals), attrs.range(inputName).null))
+    // DefLink( id, link) ->
+    // DefFootnote( id, content) ->
+    // Empty() ->
+    // Special( name, value, attrs) ->
+    // Line( text, _, attrs) ->
+    _ -> Nothing
+  }
+}
+
+
+function labelInfo(elem: (string, label)): referenceInfo {
+  val (id, label) = elem
+  ReferenceInfo(id, label.element, label.labelCaption, null(label.labelPosition))
+}
+
+// TODO change to labelRange
+function labelPosition(label : label) : maybe<position> {
+  match(label.labelAttrs.hasKey("data-line")) { // || label.labelAttrs.hasKey("data-line-first")
+    Just(info) -> {
+      val (path, lineno) = parseLineInfo(info)
+      Just(Position(path.lastPathSegment, lineno))
+      // if (lineno <= 0 || info.startsWith("0;")) then Nothing else Just((path.lastPathSegment, lineno))
+    }
+    Nothing    -> Nothing
+  }
+}

src/backports.kk

@@ -0,0 +1,32 @@
+module backports
+
+// backporting null from newer Koka versions:
+
+// Abstract type used for passing `null` values to external functions
+public type null<a>
+
+// Transform a `:maybe` type to a `:null` type (using `null` for `Nothing`).
+public external null(x : maybe<a>) : null<a> {
+  cs inline "(#1.tag_ == __std_core._maybe_Tag.Nothing ? default(##1) : #1.@value)"
+  js inline "(#1==null ? null : #1.unJust)"
+}
+
+// Transform a `:null` type to a `:maybe` type. Note that it is not
+// always the case that `id(x) == maybe(null(x))` (e.g. when `x = Just(Nothing)`).
+public external maybe( n : null<a> ) : maybe<a> {
+  cs inline "(EqualityComparer<##1>.Default.Equals(#1,default(##1)) ? __std_core._maybe<##1>.Nothing_ : new __std_core._maybe<##1>(#1))"
+  js inline "(#1==null ? $std_core.Nothing : $std_core.Just(#1))"
+}
+
+function unjust( m : maybe<a> ) : exn a {
+  match(m) {
+    Just(x) -> x
+  }
+}
+
+function (||)( m1 : maybe<a>, m2: maybe<a> ) : maybe<a> {
+  match(m1) {
+    Nothing -> m2
+    _       -> m1
+  }
+}