JSON codec for scala AST (#291)

- Collect all marker comments in one place, seal trait, fix API, prepare to enable writing a codec
- Use prefixes for more ADTs
- Rename utils to core, move ASTs to core
- Uniform style for deriving json codecs
- Rewrite circe auto derivation to semiauto and be explicit about instances for all types
- The circe codec for the scala AST costs a lot of compile time unfortunately, so let's offset it by turning off optimizations locally. They will only be turned on in CI and for releases

.circleci/config.yml

@@ -19,7 +19,7 @@ jobs:
       # Customize the JVM maximum heap limit
       JVM_OPTS: -Xmx3200m
       TERM: dumb
-
+      CI: 'true'
     steps:
       - checkout
 

build.sbt

@@ -8,7 +8,7 @@ Global / bspEnabled := false
 autoStartServer := false
 Global / excludeLintKeys += autoStartServer
 
-lazy val utils = project
+lazy val core = project
   .configure(baseSettings)
   .settings(libraryDependencies ++= Seq(Deps.ammoniteOps, Deps.osLib, Deps.sourcecode) ++ Deps.circe)
 
@@ -18,7 +18,7 @@ lazy val logging = project
 
 lazy val ts = project
   .configure(baseSettings, optimize)
-  .dependsOn(utils, logging)
+  .dependsOn(core, logging)
   .settings(libraryDependencies += Deps.parserCombinators)
 
 lazy val docs = project
@@ -31,12 +31,12 @@ lazy val docs = project
   .enablePlugins(MdocPlugin, DocusaurusPlugin)
 
 lazy val scalajs = project
-  .dependsOn(utils, logging)
+  .dependsOn(core, logging)
   .configure(baseSettings, optimize)
   .settings(libraryDependencies ++= Seq(Deps.scalaXml))
 
 lazy val phases = project
-  .dependsOn(utils, logging)
+  .dependsOn(core, logging)
   .configure(baseSettings, optimize)
 
 lazy val `importer-portable` = project
@@ -101,17 +101,7 @@ lazy val root = project
     skip in publish := true,
   )
   .configure(baseSettings)
-  .aggregate(
-    logging,
-    utils,
-    phases,
-    ts,
-    scalajs,
-    `importer-portable`,
-    `sbt-converter`,
-    importer,
-    cli,
-  )
+  .aggregate(logging, core, phases, ts, scalajs, `importer-portable`, `sbt-converter`, importer, cli)
 
 lazy val baseSettings: Project => Project =
   _.settings(
@@ -136,17 +126,20 @@ lazy val baseSettings: Project => Project =
 
 lazy val optimize: Project => Project =
   _.settings(
-    scalacOptions ++= Seq(
-      "-opt:l:inline",
-      "-opt:l:method",
-      "-opt:simplify-jumps",
-      "-opt:compact-locals",
-      "-opt:copy-propagation",
-      "-opt:redundant-casts",
-      "-opt:box-unbox",
-      "-opt:nullness-tracking",
-      //      "-opt:closure-invocations",
-      "-opt-inline-from:org.scalablytyped.converter.internal.**",
-      "-opt-warnings",
-    ),
+    scalacOptions ++= {
+      if (insideCI.value || !isSnapshot.value)
+        Seq(
+          "-opt:l:inline",
+          "-opt:l:method",
+          "-opt:simplify-jumps",
+          "-opt:compact-locals",
+          "-opt:copy-propagation",
+          "-opt:redundant-casts",
+          "-opt:box-unbox",
+          "-opt:nullness-tracking",
+          "-opt-inline-from:org.scalablytyped.converter.internal.**",
+          "-opt-warnings",
+        )
+      else Nil
+    },
   )

cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala

@@ -1,20 +1,17 @@
 package org.scalablytyped.converter.cli
 
-import java.net.URI
-
 import com.olvind.logging.{stdout, storing, LogLevel, Logger}
 import fansi.{Attr, Color, Str}
 import org.scalablytyped.converter.internal.importer._
 import org.scalablytyped.converter.internal.importer.build.{BloopCompiler, PublishedSbtProject, SbtProject}
 import org.scalablytyped.converter.internal.importer.documentation.Npmjs
-import org.scalablytyped.converter.internal.importer.jsonCodecs._
 import org.scalablytyped.converter.internal.phases.PhaseListener.NoListener
 import org.scalablytyped.converter.internal.phases.{PhaseRes, PhaseRunner, RecPhase}
 import org.scalablytyped.converter.internal.scalajs.{Name, Versions}
 import org.scalablytyped.converter.internal.sets.SetOps
 import org.scalablytyped.converter.internal.ts.CalculateLibraryVersion.PackageJsonOnly
 import org.scalablytyped.converter.internal.ts.{PackageJsonDeps, TsIdentLibrary}
-import org.scalablytyped.converter.internal.{constants, files, sets, BuildInfo, IArray, InFolder, Json}
+import org.scalablytyped.converter.internal.{constants, files, sets, BuildInfo, InFolder, Json}
 import org.scalablytyped.converter.{Flavour, Selection}
 import scopt.{OParser, OParserBuilder, Read}
 
@@ -74,13 +71,13 @@ object Main {
     storing().zipWith(stdout.filter(LogLevel.warn))
 
   implicit val ReadsFlavour: Read[Flavour] =
-    Read.reads(s => Flavour.all.getOrElse(s, sys.error(s"'$s' is not among ${Flavour.all.keys}")))
+    Read.reads(s => Flavour.byName.getOrElse(s, sys.error(s"'$s' is not among ${Flavour.byName.keys}")))
 
   implicit val ReadsVersionsScalaJs: Read[Versions.ScalaJs] =
-    Read.stringRead.map(Versions.ScalaJs)
+    Read.stringRead.map(Versions.ScalaJs.apply)
 
   implicit val ReadsVersionsScala: Read[Versions.Scala] =
-    Read.stringRead.map(Versions.Scala)
+    Read.stringRead.map(Versions.Scala.apply)
 
   implicit val ReadsTsIdentLibrary: Read[TsIdentLibrary] =
     Read.stringRead.map(TsIdentLibrary.apply)
@@ -133,7 +130,7 @@ object Main {
       opt[Flavour]('f', "flavour")
         .action((x, c) => c.mapConversion(_.copy(flavour = x)))
         .text(
-          s"One of ${Flavour.all.keys.mkString(", ")}. See https://scalablytyped.org/docs/flavour",
+          s"One of ${Flavour.byName.keys.mkString(", ")}. See https://scalablytyped.org/docs/flavour",
         ),
       opt[Versions.ScalaJs]("scalajs")
         .action((x, c) => c.mapConversion(cc => cc.copy(versions = cc.versions.copy(scalaJs = x))))

utils/src/main/scala/org/scalablytyped/converter/Selection.scala → core/src/main/scala/org/scalablytyped/converter/Selection.scala

@@ -1,5 +1,7 @@
 package org.scalablytyped.converter
 
+import io.circe013.{Decoder, Encoder}
+
 import scala.collection.immutable.{SortedSet, TreeSet}
 
 sealed trait Selection[T] {
@@ -47,4 +49,6 @@ object Selection {
 
   final case class Or[T](_1: Selection[T], _2: Selection[T]) extends Selection[T]
 
+  implicit def encodes[T: Encoder: Ordering]: Encoder[Selection[T]] = io.circe013.generic.semiauto.deriveEncoder
+  implicit def decodes[T: Decoder: Ordering]: Decoder[Selection[T]] = io.circe013.generic.semiauto.deriveDecoder
 }

core/src/main/scala/org/scalablytyped/converter/internal/Comment.scala

@@ -0,0 +1,48 @@
+package org.scalablytyped.converter.internal
+
+import io.circe013.{Decoder, Encoder}
+import org.scalablytyped.converter.internal.scalajs.{ExprTree, QualifiedName, TypeRef}
+import org.scalablytyped.converter.internal.ts.TsIdentModule
+
+sealed trait Comment
+
+/* We need a few pieces of out of band information bundled within the tree,
+  to be used like annotations. Instead of actually inventing annotations on
+  the typescript side we rather just work with special comments.
+ */
+sealed trait Marker extends Comment
+
+object Marker {
+  case object CouldBeScalaJsDefined extends Marker
+  case object IsTrivial extends Marker
+  case object ExpandedCallables extends Marker
+  case object ExpandedClass extends Marker
+  case object EnumObject extends Marker
+  case class NameHint(value:        String) extends Marker
+  case class ModuleAliases(aliases: IArray[TsIdentModule]) extends Marker
+  case class WasLiteral(lit:        ExprTree.Lit) extends Marker
+  case class WasUnion(related:      IArray[TypeRef]) extends Marker
+
+  /* Disable the minimizer for object with this marker */
+  final case class MinimizationKeep(related: IArray[TypeRef]) extends Marker
+
+  /* Similar to above, but it's conditional. If the object with this marker is included, only then include the related objects as well */
+  final case class MinimizationRelated(related: IArray[TypeRef]) extends Marker
+
+  case class WasDefaulted(among: Set[QualifiedName]) extends Marker
+
+  case object ManglerLeaveAlone extends Marker
+  case object ManglerWasJsNative extends Marker
+}
+
+object Comment {
+  final case class Raw(raw: String) extends Comment
+
+  def apply(raw: String): Comment = Comment.Raw(raw)
+
+  def warning(s: String)(implicit e: sourcecode.Enclosing): Comment =
+    Comment(s"/* import warning: ${e.value.split("\\.").takeRight(2).mkString(".")} $s */")
+
+  implicit lazy val encodes: Encoder[Comment] = io.circe013.generic.semiauto.deriveEncoder
+  implicit lazy val decodes: Decoder[Comment] = io.circe013.generic.semiauto.deriveDecoder
+}

utils/src/main/scala/org/scalablytyped/converter/internal/comments.scala → core/src/main/scala/org/scalablytyped/converter/internal/Comments.scala

@@ -1,38 +1,26 @@
 package org.scalablytyped.converter.internal
 
-import scala.collection.mutable
-import seqs._
+import io.circe013.{Decoder, Encoder}
+import org.scalablytyped.converter.internal.seqs._
 
+import scala.collection.mutable
 import scala.reflect.ClassTag
 
-sealed trait Comment
-final case class CommentRaw(raw:   String) extends Comment
-final case class CommentData(data: Comment.Data) extends Comment
-
-object Comment {
-  trait Data
-
-  def apply(raw: String): Comment = CommentRaw(raw)
-
-  def warning(s: String)(implicit e: sourcecode.Enclosing): Comment =
-    Comment(s"/* import warning: ${e.value.split("\\.").takeRight(2).mkString(".")} $s */")
-}
-
 @SerialVersionUID(8167323919307012581L) // something about this class seems brittle
 sealed class Comments(val cs: List[Comment]) extends Serializable {
-  def rawCs = cs.collect { case CommentRaw(raw) => raw }
+  def rawCs = cs.collect { case Comment.Raw(raw) => raw }
 
-  def extract[T](pf: PartialFunction[Comment.Data, T]): Option[(T, Comments)] =
+  def extract[T](pf: PartialFunction[Marker, T]): Option[(T, Comments)] =
     cs.partitionCollect {
-      case CommentData(data) if pf.isDefinedAt(data) => pf(data)
+      case marker: Marker if pf.isDefinedAt(marker) => pf(marker)
     } match {
       case (Nil, _)     => None
       case (some, rest) => Some((some.head, Comments(rest)))
     }
 
-  def has[T <: Comment.Data: ClassTag]: Boolean =
+  def has[T <: Marker: ClassTag]: Boolean =
     cs.exists {
-      case CommentData(_: T) => true
+      case _: T => true
       case _ => false
     }
 
@@ -80,6 +68,9 @@ case object NoComments extends Comments(Nil) {
 }
 
 object Comments {
+  implicit val encodes: Encoder[Comments] = Encoder[List[Comment]].contramap(_.cs)
+  implicit val decodes: Decoder[Comments] = Decoder[List[Comment]].map(Comments.apply)
+
   def apply(h: String, tail: String*): Comments =
     new Comments(Comment(h) +: tail.map(Comment.apply).toList)
 
@@ -119,4 +110,5 @@ object Comments {
 
   def format(comments: Comments, keepComments: Boolean): String =
     if (keepComments) format(comments) else ""
+
 }

utils/src/main/scala/org/scalablytyped/converter/internal/IArray.scala → core/src/main/scala/org/scalablytyped/converter/internal/IArray.scala

@@ -2,13 +2,20 @@ package org.scalablytyped.converter.internal
 
 import java.util
 
+import io.circe013.{Decoder, Encoder}
 import org.scalablytyped.converter.internal.IArray.fromArrayAndSize
 
 import scala.collection.immutable.{Range, SortedSet}
 import scala.collection.mutable.WrappedArray.ofRef
 import scala.collection.{immutable, mutable, GenTraversableOnce, Iterator}
 
 object IArray {
+  implicit def IArrayEncoder[T <: AnyRef: Encoder]: Encoder[IArray[T]] =
+    Encoder[List[T]].contramap[IArray[T]](_.toList)
+
+  implicit def IArrayDecoder[T <: AnyRef: Decoder]: Decoder[IArray[T]] =
+    Decoder[List[T]].map[IArray[T]](IArray.fromTraversable)
+
   def apply[A <: AnyRef](as: A*): IArray[A] =
     as match {
       case x: ofRef[A]                => fromArray(x.array)

utils/src/main/scala/org/scalablytyped/converter/internal/ProtectionLevel.scala → core/src/main/scala/org/scalablytyped/converter/internal/ProtectionLevel.scala

@@ -1,9 +1,14 @@
 package org.scalablytyped.converter.internal
 
+import io.circe013.{Decoder, Encoder}
+
 sealed trait ProtectionLevel
 
 object ProtectionLevel {
   case object Default extends ProtectionLevel
   case object Private extends ProtectionLevel
   case object Protected extends ProtectionLevel
+
+  implicit val encodes: Encoder[ProtectionLevel] = io.circe013.generic.semiauto.deriveEncoder
+  implicit val decodes: Decoder[ProtectionLevel] = io.circe013.generic.semiauto.deriveDecoder
 }

utils/src/main/scala/org/scalablytyped/converter/internal/files.scala → core/src/main/scala/org/scalablytyped/converter/internal/files.scala

@@ -6,6 +6,7 @@ import java.nio.file.{Files, Path}
 import java.util
 
 import ammonite.ops.%
+import io.circe013.{Decoder, Encoder}
 import org.scalablytyped.converter.internal.environment.OpSystem
 
 import scala.util.Try
@@ -30,6 +31,10 @@ object InFile {
 final case class InFolder(path: os.Path) {
   def name: String = path.last
 }
+object InFolder {
+  implicit val encodes: Encoder[InFolder] = orphanCodecs.PathEncoder.contramap(_.path)
+  implicit val decodes: Decoder[InFolder] = orphanCodecs.PathDecoder.map(InFolder.apply)
+}
 
 trait Layout[F, V] {
   def all: Map[F, V]

core/src/main/scala/org/scalablytyped/converter/internal/orphanCodecs.scala

@@ -0,0 +1,25 @@
+package org.scalablytyped.converter.internal
+
+import java.io.File
+import java.net.URI
+
+import io.circe013.{Decoder, Encoder}
+
+import scala.collection.immutable.{SortedSet, TreeSet}
+
+object orphanCodecs {
+  implicit def SortedSetEncoder[T: Encoder: Ordering]: Encoder[SortedSet[T]] =
+    Encoder[Vector[T]].contramap[SortedSet[T]](x => x.toVector)
+
+  implicit def SortedSetDecoder[T: Decoder: Ordering]: Decoder[SortedSet[T]] =
+    Decoder[Vector[T]].map[SortedSet[T]](ts => TreeSet.empty[T] ++ ts)
+
+  implicit val FileEncoder:    Encoder[File]       = Encoder[String].contramap[File](_.toString)
+  implicit val FileDecoder:    Decoder[File]       = Decoder[String].map[File](new File(_))
+  implicit val RelPathDecoder: Decoder[os.RelPath] = Decoder[String].map(str => os.RelPath(str.dropWhile(_ === '/')))
+  implicit val RelPathEncoder: Encoder[os.RelPath] = Encoder[String].contramap[os.RelPath](_.toString)
+  implicit val PathDecoder:    Decoder[os.Path]    = Decoder[String].map(str => os.Path(str))
+  implicit val PathEncoder:    Encoder[os.Path]    = Encoder[String].contramap[os.Path](_.toString)
+  implicit val URIDecoder:     Decoder[URI]        = Decoder[String].map(new URI(_))
+  implicit val URIEncoder:     Encoder[URI]        = Encoder[String].contramap[URI](_.toString)
+}

scalajs/src/main/scala/org/scalablytyped/converter/internal/scalajs/Annotation.scala → core/src/main/scala/org/scalablytyped/converter/internal/scalajs/Annotation.scala

@@ -1,6 +1,8 @@
 package org.scalablytyped.converter.internal
 package scalajs
 
+import io.circe013.{Decoder, Encoder}
+
 sealed trait Annotation extends Product with Serializable
 sealed trait LocationAnnotation extends Annotation
 
@@ -9,6 +11,9 @@ object Imported {
   case object Namespace extends Imported
   case object Default extends Imported
   case class Named(name: IArray[Name]) extends Imported
+
+  implicit val encodes: Encoder[Imported] = io.circe013.generic.semiauto.deriveEncoder
+  implicit val decodes: Decoder[Imported] = io.circe013.generic.semiauto.deriveDecoder
 }
 
 object Annotation {
@@ -24,6 +29,11 @@ object Annotation {
   case class JsImport(module:   String, imported: Imported, global: Option[JsGlobal]) extends LocationAnnotation
   case class JsGlobal(name:     QualifiedName) extends LocationAnnotation
 
+  object JsGlobal {
+    implicit val encodes: Encoder[JsGlobal] = io.circe013.generic.semiauto.deriveEncoder
+    implicit val decodes: Decoder[JsGlobal] = io.circe013.generic.semiauto.deriveDecoder
+  }
+
   def renamedFrom(newName: Name)(oldAnnotations: IArray[Annotation]): IArray[Annotation] = {
     val (names, others) =
       oldAnnotations.partition {
@@ -40,4 +50,7 @@ object Annotation {
 
     others ++ updatedNames
   }
+
+  implicit lazy val encodes: Encoder[Annotation] = io.circe013.generic.semiauto.deriveEncoder
+  implicit lazy val decodes: Decoder[Annotation] = io.circe013.generic.semiauto.deriveDecoder
 }