Introduce Fs2 based PlatformIO supporting scalajs

.github/workflows/ci.yml

@@ -108,6 +108,27 @@ jobs:
           - '2.13.15'
         java:
           - '8'
+  testWithNode:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: "actions/[email protected]"
+      - uses: "coursier/cache-action@v2"
+      - name: "graalvm setup"
+        uses: "olafurpg/setup-scala@v13"
+        with:
+          java-version: "${{matrix.java}}"
+      - name: "build node"
+        run: |
+          sbt "++${{matrix.scala}}; cliJSJS/fullOptJS"
+      - name: "run bosatsu tests"
+        run: |
+          ./bosatsu_node test --input_dir test_workspace/ --package_root test_workspace/
+    strategy:
+      matrix:
+        scala:
+          - '2.13.15'
+        java:
+          - '8'
   testC:
     runs-on: ubuntu-latest
     strategy:

bosatsu_node

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+# hide the punycode deprecation warning
+export NODE_OPTIONS="--no-deprecation"
+# make sure to run sbt cliJSJS/fullOptJS
+node $SCRIPT_DIR/cliJS/.js/target/scala-2.13/bosatsu-clijs-opt "$@"

build.sbt

@@ -208,6 +208,24 @@ lazy val core =
 lazy val coreJVM = core.jvm
 lazy val coreJS = core.js
 
+lazy val cliJS = 
+  (crossProject(JSPlatform).crossType(CrossType.Pure) in file("cliJS"))
+    .settings(
+      commonSettings,
+      commonJsSettings,
+      name := "bosatsu-clijs",
+      assembly / test := {},
+      mainClass := Some("org.bykn.bosatsu.tool.Fs2Main"),
+      libraryDependencies ++= Seq(fs2core.value, fs2io.value, catsEffect.value),
+    )
+    .jsSettings(
+      scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.CommonJSModule) },
+      scalaJSLinkerConfig ~= { _.withSourceMap(false).withOptimizer(true) },
+      mainClass := Some("org.bykn.bosatsu.tool.Fs2Main"),
+      scalaJSUseMainModuleInitializer := true,
+    )
+    .dependsOn(base, core)
+
 lazy val jsapi =
   (crossProject(JSPlatform).crossType(CrossType.Pure) in file("jsapi"))
     .settings(

cli/src/main/scala/org/bykn/bosatsu/PlatformModule.scala → cli/src/main/scala/org/bykn/bosatsu/IOPlatformIO.scala

@@ -103,11 +103,8 @@ object IOPlatformIO extends PlatformIO[IO, JPath] {
       .flatMap(write(_, path))
 
   def writePackages[A](packages: List[Package.Typed[A]], path: Path): IO[Unit] =
-    IO.fromTry {
-      packages
-        .traverse(ProtoConverter.packageToProto(_))
-        .map(proto.Packages(_))
-    }.flatMap(write(_, path))
+    IO.fromTry(ProtoConverter.packagesToProto(packages))
+      .flatMap(write(_, path))
 
   def unfoldDir: Option[Path => IO[Option[IO[List[Path]]]]] = Some {
     (path: Path) =>
@@ -126,38 +123,15 @@ object IOPlatformIO extends PlatformIO[IO, JPath] {
     path.toString.endsWith(str)
   }
 
-  def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] = {
-    import scala.jdk.CollectionConverters._
-
-    def getP(p: Path): Option[PackageName] = {
-      val subPath = p
-        .relativize(packFile)
-        .asScala
-        .map { part =>
-          part.toString.toLowerCase.capitalize
-        }
-        .mkString("/")
-
-      val dropExtension = """(.*)\.[^.]*$""".r
-      val toParse = subPath match {
-        case dropExtension(prefix) => prefix
-        case _                     => subPath
+  def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] =
+    PlatformIO.pathPackage(roots, packFile) { (root, pf) =>
+      if (pf.startsWith(root)) Some {
+        import scala.jdk.CollectionConverters._
+        root.relativize(pf).asScala.map(_.toString)
       }
-      PackageName.parse(toParse)
+      else None
     }
 
-    @annotation.tailrec
-    def loop(roots: List[Path]): Option[PackageName] =
-      roots match {
-        case Nil                              => None
-        case h :: _ if packFile.startsWith(h) => getP(h)
-        case _ :: t                           => loop(t)
-      }
-
-    if (packFile.toString.isEmpty) None
-    else loop(roots)
-  }
-
   def writeDoc(p: Path, d: Doc): IO[Unit] =
     IO.blocking {
       Option(p.getParent).foreach(_.toFile.mkdirs)

cliJS/src/main/scala/org/bykn/bosatsu/Fs2PlatformIO.scala

@@ -0,0 +1,142 @@
+package org.bykn.bosatsu
+
+import _root_.bosatsu.{TypedAst => proto}
+import cats.MonadError
+import cats.data.Validated
+import cats.effect.IO
+import fs2.io.file.{Files, Path}
+import com.monovore.decline.Argument
+import org.typelevel.paiges.Doc
+import scala.util.{Failure, Success, Try}
+
+import cats.syntax.all._
+
+object Fs2PlatformIO extends PlatformIO[IO, Path] {
+  def moduleIOMonad: MonadError[IO, Throwable] =
+    IO.asyncForIO
+
+  val pathArg: Argument[Path] =
+    new Argument[Path] {
+      def read(string: String) =
+        Try(Path(string)) match {
+          case Success(value) =>
+            Validated.valid(value)
+          case Failure(exception) =>
+            Validated.invalidNel(s"could not parse $string as path: ${exception.getMessage()}") 
+        }
+
+      def defaultMetavar: String = "path"
+    }
+
+  val pathOrdering: Ordering[Path] = Path.instances.toOrdering
+
+  private val FilesIO = Files.forIO
+
+  def readPath(p: Path): IO[String] =
+    FilesIO.readUtf8(p).compile.string
+
+  def readPackages(paths: List[Path]): IO[List[Package.Typed[Unit]]] =
+    paths.parTraverse { path =>
+      for {
+        bytes <- FilesIO.readAll(path).compile.to(Array)
+        ppack <- IO(proto.Packages.parseFrom(bytes))
+        packs <- IO.fromTry(ProtoConverter.packagesFromProto(Nil, ppack.packages))
+      } yield packs._2
+    }
+    .map(_.flatten)
+
+  def readInterfaces(paths: List[Path]): IO[List[Package.Interface]] =
+    paths.parTraverse { path =>
+      for {
+        bytes <- FilesIO.readAll(path).compile.to(Array)
+        pifaces <- IO(proto.Interfaces.parseFrom(bytes))
+        ifaces <- IO.fromTry(ProtoConverter.packagesFromProto(pifaces.interfaces, Nil))
+      } yield ifaces._1
+    }
+    .map(_.flatten)
+
+  /** given an ordered list of prefered roots, if a packFile starts with one of
+    * these roots, return a PackageName based on the rest
+    */
+  def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] =
+    PlatformIO.pathPackage(roots, packFile) { (root, pf) =>
+      if (pf.startsWith(root)) Some {
+        root.relativize(pf).names.map(_.toString)
+      }
+      else None
+    }
+
+  /** Modules optionally have the capability to combine paths into a tree
+    */
+  val resolvePath: Option[(Path, PackageName) => IO[Option[Path]]] = Some {
+    (root: Path, pack: PackageName) => {
+      val dir = pack.parts.init.foldLeft(root)(_.resolve(_))
+      val filePath = dir.resolve(pack.parts.last + ".bosatsu")
+      FilesIO.exists(filePath, true)
+        .map {
+          case true => Some(filePath)
+          case false => None
+        }
+    }
+  }
+
+  /** some modules have paths that form directory trees
+    *
+    * if the given path is a directory, return Some and all the first children.
+    */
+  def unfoldDir: Option[Path => IO[Option[IO[List[Path]]]]] = Some {
+    (path: Path) => {
+      FilesIO.isDirectory(path, followLinks = true)
+        .map {
+          case true => Some {
+            // create a list of children
+            FilesIO.list(path).compile.toList
+          }
+          case false => None
+        }
+    }
+  }
+
+  def hasExtension(str: String): Path => Boolean =
+    { (path: Path) => path.extName == str }
+
+  private def docStream(doc: Doc): fs2.Stream[IO, String] =
+    fs2.Stream.fromIterator[IO](doc.renderStream(100).iterator, chunkSize = 128)
+
+  def writeDoc(p: Path, d: Doc): IO[Unit] = {
+    val pipe = Files.forIO.writeUtf8(p)
+    pipe(docStream(d)).compile.drain
+  }
+
+  def writeStdout(doc: Doc): IO[Unit] =
+    docStream(doc)
+      .evalMapChunk(part => IO.print(part))
+      .compile
+      .drain
+
+  def resolve(base: Path, p: List[String]): Path =
+    p.foldLeft(base)(_.resolve(_))
+
+  // this is println actually
+  def print(str: String): IO[Unit] =
+    IO.println(str)
+
+  def writeInterfaces(
+      interfaces: List[Package.Interface],
+      path: Path
+  ): IO[Unit] =
+    for {
+      protoIfaces <- IO.fromTry(ProtoConverter.interfacesToProto(interfaces))
+      bytes = protoIfaces.toByteArray
+      pipe = Files.forIO.writeAll(path)
+      _ <- pipe(fs2.Stream.chunk(fs2.Chunk.array(bytes))).compile.drain
+    } yield ()
+
+  def writePackages[A](packages: List[Package.Typed[A]], path: Path): IO[Unit] =
+    for {
+      protoPacks <- IO.fromTry(ProtoConverter.packagesToProto(packages))
+      bytes = protoPacks.toByteArray
+      pipe = Files.forIO.writeAll(path)
+      _ <- pipe(fs2.Stream.chunk(fs2.Chunk.array(bytes))).compile.drain
+    } yield ()
+}

cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Main.scala

@@ -0,0 +1,16 @@
+package org.bykn.bosatsu.tool
+
+import cats.effect.{ExitCode, IO, IOApp}
+
+object Fs2Main extends IOApp {
+  def run(args: List[String]): IO[ExitCode] =
+    Fs2Module.run(args) match {
+      case Right(getOutput) =>
+        Fs2Module.report(getOutput)
+      case Left(help) =>
+        IO.blocking {
+          System.err.println(help.toString)
+          ExitCode.Error
+        }
+    }
+}

cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Module.scala

@@ -0,0 +1,37 @@
+package org.bykn.bosatsu.tool
+
+import cats.{effect => ce}
+import cats.effect.{IO, Resource}
+import fs2.io.file.Path
+import org.bykn.bosatsu.{Par, MainModule, Fs2PlatformIO}
+
+object Fs2Module extends MainModule[IO, Path](Fs2PlatformIO) { self =>
+  val parResource: Resource[IO, Par.EC] =
+    Resource.make(IO(Par.newService()))(es => IO(Par.shutdownService(es)))
+      .map(Par.ecFromService(_))
+
+  def withEC[A](fn: Par.EC => IO[A]): IO[A] =
+    parResource.use(fn)
+
+  def fromToolExit(ec: ExitCode): ce.ExitCode =
+    ec match {
+      case ExitCode.Success => ce.ExitCode.Success
+      case ExitCode.Error => ce.ExitCode.Error
+    }
+
+  def report(io: IO[Output[Path]]): IO[ce.ExitCode] =
+    io.attempt.flatMap {
+      case Right(out) => reportOutput(out).map(fromToolExit)
+      case Left(err)  => reportException(err).as(ce.ExitCode.Error)
+    }
+
+  def reportException(ex: Throwable): IO[Unit] =
+    mainExceptionToString(ex) match {
+      case Some(msg) =>
+        IO.consoleForIO.errorln(msg)
+      case None =>
+        IO.consoleForIO.errorln("unknown error:\n") *>
+          IO.blocking(ex.printStackTrace(System.err))
+    }
+
+}

core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala

@@ -4,6 +4,8 @@ import cats.MonadError
 import com.monovore.decline.Argument
 import org.typelevel.paiges.Doc
 
+import cats.syntax.all._
+
 trait PlatformIO[F[_], Path] {
   implicit def moduleIOMonad: MonadError[F, Throwable]
   implicit def pathArg: Argument[Path]
@@ -53,3 +55,28 @@ trait PlatformIO[F[_], Path] {
 
   def writePackages[A](packages: List[Package.Typed[A]], path: Path): F[Unit]
 }
+
+object PlatformIO {
+  def pathPackage[Path](roots: List[Path], packFile: Path)(relativeParts: (Path, Path) => Option[Iterable[String]]): Option[PackageName] = {
+    def getP(p: Path): Option[PackageName] =
+      relativeParts(p, packFile).flatMap { parts =>
+        val subPath = parts
+          .iterator
+          .map { part =>
+            part.toLowerCase.capitalize
+          }
+          .mkString("/")
+
+        val dropExtension = """(.*)\.[^.]*$""".r
+        val toParse = subPath match {
+          case dropExtension(prefix) => prefix
+          case _                     => subPath
+        }
+        PackageName.parse(toParse)
+      }
+
+    if (packFile.toString.isEmpty) None
+    else roots.collectFirstSome(getP)
+  }
+
+}

core/src/main/scala/org/bykn/bosatsu/ProtoConverter.scala

@@ -1516,6 +1516,16 @@ object ProtoConverter {
     runTab(tab).map { case (ss, fn) => fn(ss) }
   }
 
+  def packagesToProto[F[_]: Foldable, A](
+      ps: F[Package.Typed[A]]
+  ): Try[proto.Packages] =
+    // sort so we are deterministic
+    ps.toList.sortBy(_.name)
+      .traverse(packageToProto(_))
+      .map { packs =>
+        proto.Packages(packs)
+      }
+
   private val anonBind: Success[Bindable] =
     Success(Identifier.Name("$anon"))
 

project/Dependencies.scala

@@ -8,6 +8,8 @@ object Dependencies {
   lazy val catsParse = Def.setting("org.typelevel" %%% "cats-parse" % "1.1.0")
   lazy val decline = Def.setting("com.monovore" %%% "decline" % "2.4.1")
   lazy val ff4s = Def.setting("io.github.buntec" %%% "ff4s" % "0.24.0")
+  lazy val fs2core = Def.setting("co.fs2" %%% "fs2-core" % "3.11.0")
+  lazy val fs2io = Def.setting("co.fs2" %%% "fs2-io" % "3.11.0")
   lazy val jawnParser = Def.setting("org.typelevel" %%% "jawn-parser" % "1.6.0")
   lazy val jawnAst = Def.setting("org.typelevel" %%% "jawn-ast" % "1.6.0")
   lazy val jython = Def.setting("org.python" % "jython-standalone" % "2.7.4")