diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dedb84b66..987d11396 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,27 @@ jobs: - '2.13.15' java: - '8' + testWithNode: + runs-on: ubuntu-latest + steps: + - uses: "actions/checkout@v2.1.0" + - uses: "coursier/cache-action@v2" + - name: "graalvm setup" + uses: "olafurpg/setup-scala@v13" + with: + java-version: "${{matrix.java}}" + - name: "build node" + run: | + sbt "++${{matrix.scala}}; cliJSJS/fullOptJS" + - name: "run bosatsu tests" + run: | + ./bosatsu_node test --input_dir test_workspace/ --package_root test_workspace/ + strategy: + matrix: + scala: + - '2.13.15' + java: + - '8' testC: runs-on: ubuntu-latest strategy: diff --git a/bosatsu_node b/bosatsu_node new file mode 100755 index 000000000..b814d5e87 --- /dev/null +++ b/bosatsu_node @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +# hide the punycode deprecation warning +export NODE_OPTIONS="--no-deprecation" +# make sure to run sbt cliJSJS/fullOptJS +node $SCRIPT_DIR/cliJS/.js/target/scala-2.13/bosatsu-clijs-opt "$@" diff --git a/build.sbt b/build.sbt index 2705aa719..e66e23682 100644 --- a/build.sbt +++ b/build.sbt @@ -208,6 +208,24 @@ lazy val core = lazy val coreJVM = core.jvm lazy val coreJS = core.js +lazy val cliJS = + (crossProject(JSPlatform).crossType(CrossType.Pure) in file("cliJS")) + .settings( + commonSettings, + commonJsSettings, + name := "bosatsu-clijs", + assembly / test := {}, + mainClass := Some("org.bykn.bosatsu.tool.Fs2Main"), + libraryDependencies ++= Seq(fs2core.value, fs2io.value, catsEffect.value), + ) + .jsSettings( + scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.CommonJSModule) }, + scalaJSLinkerConfig ~= { _.withSourceMap(false).withOptimizer(true) }, + mainClass := Some("org.bykn.bosatsu.tool.Fs2Main"), + scalaJSUseMainModuleInitializer := true, + ) + .dependsOn(base, core) + lazy val jsapi = (crossProject(JSPlatform).crossType(CrossType.Pure) in file("jsapi")) .settings( diff --git a/cli/src/main/scala/org/bykn/bosatsu/PlatformModule.scala b/cli/src/main/scala/org/bykn/bosatsu/IOPlatformIO.scala similarity index 80% rename from cli/src/main/scala/org/bykn/bosatsu/PlatformModule.scala rename to cli/src/main/scala/org/bykn/bosatsu/IOPlatformIO.scala index 2d8c45168..47f637b8d 100644 --- a/cli/src/main/scala/org/bykn/bosatsu/PlatformModule.scala +++ b/cli/src/main/scala/org/bykn/bosatsu/IOPlatformIO.scala @@ -103,11 +103,8 @@ object IOPlatformIO extends PlatformIO[IO, JPath] { .flatMap(write(_, path)) def writePackages[A](packages: List[Package.Typed[A]], path: Path): IO[Unit] = - IO.fromTry { - packages - .traverse(ProtoConverter.packageToProto(_)) - .map(proto.Packages(_)) - }.flatMap(write(_, path)) + IO.fromTry(ProtoConverter.packagesToProto(packages)) + .flatMap(write(_, path)) def unfoldDir: Option[Path => IO[Option[IO[List[Path]]]]] = Some { (path: Path) => @@ -126,38 +123,15 @@ object IOPlatformIO extends PlatformIO[IO, JPath] { path.toString.endsWith(str) } - def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] = { - import scala.jdk.CollectionConverters._ - - def getP(p: Path): Option[PackageName] = { - val subPath = p - .relativize(packFile) - .asScala - .map { part => - part.toString.toLowerCase.capitalize - } - .mkString("/") - - val dropExtension = """(.*)\.[^.]*$""".r - val toParse = subPath match { - case dropExtension(prefix) => prefix - case _ => subPath + def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] = + PlatformIO.pathPackage(roots, packFile) { (root, pf) => + if (pf.startsWith(root)) Some { + import scala.jdk.CollectionConverters._ + root.relativize(pf).asScala.map(_.toString) } - PackageName.parse(toParse) + else None } - @annotation.tailrec - def loop(roots: List[Path]): Option[PackageName] = - roots match { - case Nil => None - case h :: _ if packFile.startsWith(h) => getP(h) - case _ :: t => loop(t) - } - - if (packFile.toString.isEmpty) None - else loop(roots) - } - def writeDoc(p: Path, d: Doc): IO[Unit] = IO.blocking { Option(p.getParent).foreach(_.toFile.mkdirs) diff --git a/cliJS/src/main/scala/org/bykn/bosatsu/Fs2PlatformIO.scala b/cliJS/src/main/scala/org/bykn/bosatsu/Fs2PlatformIO.scala new file mode 100644 index 000000000..64c741806 --- /dev/null +++ b/cliJS/src/main/scala/org/bykn/bosatsu/Fs2PlatformIO.scala @@ -0,0 +1,142 @@ +package org.bykn.bosatsu + +import _root_.bosatsu.{TypedAst => proto} +import cats.MonadError +import cats.data.Validated +import cats.effect.IO +import fs2.io.file.{Files, Path} +import com.monovore.decline.Argument +import org.typelevel.paiges.Doc +import scala.util.{Failure, Success, Try} + +import cats.syntax.all._ + +object Fs2PlatformIO extends PlatformIO[IO, Path] { + def moduleIOMonad: MonadError[IO, Throwable] = + IO.asyncForIO + + val pathArg: Argument[Path] = + new Argument[Path] { + def read(string: String) = + Try(Path(string)) match { + case Success(value) => + Validated.valid(value) + case Failure(exception) => + Validated.invalidNel(s"could not parse $string as path: ${exception.getMessage()}") + } + + def defaultMetavar: String = "path" + } + + val pathOrdering: Ordering[Path] = Path.instances.toOrdering + + private val FilesIO = Files.forIO + + def readPath(p: Path): IO[String] = + FilesIO.readUtf8(p).compile.string + + def readPackages(paths: List[Path]): IO[List[Package.Typed[Unit]]] = + paths.parTraverse { path => + for { + bytes <- FilesIO.readAll(path).compile.to(Array) + ppack <- IO(proto.Packages.parseFrom(bytes)) + packs <- IO.fromTry(ProtoConverter.packagesFromProto(Nil, ppack.packages)) + } yield packs._2 + } + .map(_.flatten) + + def readInterfaces(paths: List[Path]): IO[List[Package.Interface]] = + paths.parTraverse { path => + for { + bytes <- FilesIO.readAll(path).compile.to(Array) + pifaces <- IO(proto.Interfaces.parseFrom(bytes)) + ifaces <- IO.fromTry(ProtoConverter.packagesFromProto(pifaces.interfaces, Nil)) + } yield ifaces._1 + } + .map(_.flatten) + + /** given an ordered list of prefered roots, if a packFile starts with one of + * these roots, return a PackageName based on the rest + */ + def pathPackage(roots: List[Path], packFile: Path): Option[PackageName] = + PlatformIO.pathPackage(roots, packFile) { (root, pf) => + if (pf.startsWith(root)) Some { + root.relativize(pf).names.map(_.toString) + } + else None + } + + /** Modules optionally have the capability to combine paths into a tree + */ + val resolvePath: Option[(Path, PackageName) => IO[Option[Path]]] = Some { + (root: Path, pack: PackageName) => { + val dir = pack.parts.init.foldLeft(root)(_.resolve(_)) + val filePath = dir.resolve(pack.parts.last + ".bosatsu") + FilesIO.exists(filePath, true) + .map { + case true => Some(filePath) + case false => None + } + } + } + + /** some modules have paths that form directory trees + * + * if the given path is a directory, return Some and all the first children. + */ + def unfoldDir: Option[Path => IO[Option[IO[List[Path]]]]] = Some { + (path: Path) => { + FilesIO.isDirectory(path, followLinks = true) + .map { + case true => Some { + // create a list of children + FilesIO.list(path).compile.toList + } + case false => None + } + } + } + + def hasExtension(str: String): Path => Boolean = + { (path: Path) => path.extName == str } + + private def docStream(doc: Doc): fs2.Stream[IO, String] = + fs2.Stream.fromIterator[IO](doc.renderStream(100).iterator, chunkSize = 128) + + def writeDoc(p: Path, d: Doc): IO[Unit] = { + val pipe = Files.forIO.writeUtf8(p) + pipe(docStream(d)).compile.drain + } + + def writeStdout(doc: Doc): IO[Unit] = + docStream(doc) + .evalMapChunk(part => IO.print(part)) + .compile + .drain + + def resolve(base: Path, p: List[String]): Path = + p.foldLeft(base)(_.resolve(_)) + + // this is println actually + def print(str: String): IO[Unit] = + IO.println(str) + + def writeInterfaces( + interfaces: List[Package.Interface], + path: Path + ): IO[Unit] = + for { + protoIfaces <- IO.fromTry(ProtoConverter.interfacesToProto(interfaces)) + bytes = protoIfaces.toByteArray + pipe = Files.forIO.writeAll(path) + _ <- pipe(fs2.Stream.chunk(fs2.Chunk.array(bytes))).compile.drain + } yield () + + def writePackages[A](packages: List[Package.Typed[A]], path: Path): IO[Unit] = + for { + protoPacks <- IO.fromTry(ProtoConverter.packagesToProto(packages)) + bytes = protoPacks.toByteArray + pipe = Files.forIO.writeAll(path) + _ <- pipe(fs2.Stream.chunk(fs2.Chunk.array(bytes))).compile.drain + } yield () +} \ No newline at end of file diff --git a/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Main.scala b/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Main.scala new file mode 100644 index 000000000..e3e227a91 --- /dev/null +++ b/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Main.scala @@ -0,0 +1,16 @@ +package org.bykn.bosatsu.tool + +import cats.effect.{ExitCode, IO, IOApp} + +object Fs2Main extends IOApp { + def run(args: List[String]): IO[ExitCode] = + Fs2Module.run(args) match { + case Right(getOutput) => + Fs2Module.report(getOutput) + case Left(help) => + IO.blocking { + System.err.println(help.toString) + ExitCode.Error + } + } +} \ No newline at end of file diff --git a/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Module.scala b/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Module.scala new file mode 100644 index 000000000..f489f3d8a --- /dev/null +++ b/cliJS/src/main/scala/org/bykn/bosatsu/tool/Fs2Module.scala @@ -0,0 +1,37 @@ +package org.bykn.bosatsu.tool + +import cats.{effect => ce} +import cats.effect.{IO, Resource} +import fs2.io.file.Path +import org.bykn.bosatsu.{Par, MainModule, Fs2PlatformIO} + +object Fs2Module extends MainModule[IO, Path](Fs2PlatformIO) { self => + val parResource: Resource[IO, Par.EC] = + Resource.make(IO(Par.newService()))(es => IO(Par.shutdownService(es))) + .map(Par.ecFromService(_)) + + def withEC[A](fn: Par.EC => IO[A]): IO[A] = + parResource.use(fn) + + def fromToolExit(ec: ExitCode): ce.ExitCode = + ec match { + case ExitCode.Success => ce.ExitCode.Success + case ExitCode.Error => ce.ExitCode.Error + } + + def report(io: IO[Output[Path]]): IO[ce.ExitCode] = + io.attempt.flatMap { + case Right(out) => reportOutput(out).map(fromToolExit) + case Left(err) => reportException(err).as(ce.ExitCode.Error) + } + + def reportException(ex: Throwable): IO[Unit] = + mainExceptionToString(ex) match { + case Some(msg) => + IO.consoleForIO.errorln(msg) + case None => + IO.consoleForIO.errorln("unknown error:\n") *> + IO.blocking(ex.printStackTrace(System.err)) + } + +} diff --git a/core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala b/core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala index 3cde210ae..1657e9e0b 100644 --- a/core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala +++ b/core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala @@ -4,6 +4,8 @@ import cats.MonadError import com.monovore.decline.Argument import org.typelevel.paiges.Doc +import cats.syntax.all._ + trait PlatformIO[F[_], Path] { implicit def moduleIOMonad: MonadError[F, Throwable] implicit def pathArg: Argument[Path] @@ -53,3 +55,28 @@ trait PlatformIO[F[_], Path] { def writePackages[A](packages: List[Package.Typed[A]], path: Path): F[Unit] } + +object PlatformIO { + def pathPackage[Path](roots: List[Path], packFile: Path)(relativeParts: (Path, Path) => Option[Iterable[String]]): Option[PackageName] = { + def getP(p: Path): Option[PackageName] = + relativeParts(p, packFile).flatMap { parts => + val subPath = parts + .iterator + .map { part => + part.toLowerCase.capitalize + } + .mkString("/") + + val dropExtension = """(.*)\.[^.]*$""".r + val toParse = subPath match { + case dropExtension(prefix) => prefix + case _ => subPath + } + PackageName.parse(toParse) + } + + if (packFile.toString.isEmpty) None + else roots.collectFirstSome(getP) + } + +} \ No newline at end of file diff --git a/core/src/main/scala/org/bykn/bosatsu/ProtoConverter.scala b/core/src/main/scala/org/bykn/bosatsu/ProtoConverter.scala index f95d12a1a..325a8962f 100644 --- a/core/src/main/scala/org/bykn/bosatsu/ProtoConverter.scala +++ b/core/src/main/scala/org/bykn/bosatsu/ProtoConverter.scala @@ -1516,6 +1516,16 @@ object ProtoConverter { runTab(tab).map { case (ss, fn) => fn(ss) } } + def packagesToProto[F[_]: Foldable, A]( + ps: F[Package.Typed[A]] + ): Try[proto.Packages] = + // sort so we are deterministic + ps.toList.sortBy(_.name) + .traverse(packageToProto(_)) + .map { packs => + proto.Packages(packs) + } + private val anonBind: Success[Bindable] = Success(Identifier.Name("$anon")) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ba27b268d..0dac12861 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -8,6 +8,8 @@ object Dependencies { lazy val catsParse = Def.setting("org.typelevel" %%% "cats-parse" % "1.1.0") lazy val decline = Def.setting("com.monovore" %%% "decline" % "2.4.1") lazy val ff4s = Def.setting("io.github.buntec" %%% "ff4s" % "0.24.0") + lazy val fs2core = Def.setting("co.fs2" %%% "fs2-core" % "3.11.0") + lazy val fs2io = Def.setting("co.fs2" %%% "fs2-io" % "3.11.0") lazy val jawnParser = Def.setting("org.typelevel" %%% "jawn-parser" % "1.6.0") lazy val jawnAst = Def.setting("org.typelevel" %%% "jawn-ast" % "1.6.0") lazy val jython = Def.setting("org.python" % "jython-standalone" % "2.7.4")