Include all modules

So far ST has looked at package.json to determine a subset of typings to include. Going forward *all* files will be translated. This solves #463

There are loads of details for computing shorter module names. These will be solved similar to how `exports` was solved in #486, by injecting "proxy modules" into the AST.

Note that some libraries ship typings for multiple hierarchies, typically in folders named `src`, `es`, `es6` and so on. we'll now get all those duplicates as well.

core/src/main/scala/org/scalablytyped/converter/internal/Comment.scala

@@ -21,7 +21,6 @@ object Marker {
   case object HasClassParent extends Marker
 
   case class NameHint(value:        String) extends Marker
-  case class ModuleAliases(aliases: IArray[TsIdentModule]) extends Marker
   case class WasLiteral(lit:        ExprTree.Lit) extends Marker
   case class WasUnion(related:      IArray[TypeRef]) extends Marker
 

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/LibTsSource.scala

@@ -18,13 +18,11 @@ sealed trait LibTsSource extends TsTreeScope.TsLib {
 
   lazy val tsConfig: Option[TsConfig] =
     Json.opt[TsConfig](folder.path / "tsconfig.json")
-
-  lazy val shortenedFiles: IArray[InFile] = LibTsSource.findShortenedFiles(this)
 }
 
 object LibTsSource {
   def hasTypescriptSources(folder: InFolder): Boolean =
-    os.walk.stream(folder.path, _.last == "node_modules").exists(_.last.endsWith(".d.ts"))
+    os.walk.stream(folder.path, skip = _.last == "node_modules").exists(_.last.endsWith(".d.ts"))
 
   final case class StdLibSource(folder: InFolder, files: IArray[InFile], libName: TsIdentLibrary) extends LibTsSource
 
@@ -34,48 +32,4 @@ object LibTsSource {
     Ordering.by[S, String](_.pathString)
   implicit val SourceFormatter: Formatter[LibTsSource] =
     _.libName.value
-
-  /* for files referenced through here we must shorten the paths */
-  def findShortenedFiles(src: LibTsSource): IArray[InFile] = {
-    def fromTypingsJson(fromFolder: LibTsSource.FromFolder, files: Option[IArray[String]]): IArray[InFile] =
-      files.getOrElse(IArray.Empty).collect {
-        case path if path.endsWith("typings.json") =>
-          val typingsJsonPath = fromFolder.folder.path / os.RelPath(path)
-          val typingsJson     = Json.force[TypingsJson](typingsJsonPath)
-          InFile(typingsJsonPath / os.up / typingsJson.main)
-      }
-
-    def fromFileEntry(fromFolder: LibTsSource.FromFolder, files: Option[IArray[String]]): IArray[InFile] =
-      files.getOrElse(IArray.Empty).mapNotNone(file => LibraryResolver.file(fromFolder.folder, file))
-
-    def fromModuleDeclaration(
-        fromFolder: LibTsSource.FromFolder,
-        files:      Option[Map[String, String]],
-    ): IArray[InFile] = {
-      val files1 = files match {
-        case Some(files) => IArray.fromTraversable(files.values)
-        case None        => IArray.Empty
-      }
-
-      files1
-        .mapNotNone(file => LibraryResolver.file(fromFolder.folder, file))
-        .mapNotNone {
-          case existingFile if LibTsSource.hasTypescriptSources(existingFile.folder) => Some(existingFile)
-          case _                                                                     => None
-        }
-    }
-
-    src match {
-      case _: StdLibSource => Empty
-      case f: FromFolder =>
-        val fromTypings =
-          IArray(
-            fromFileEntry(f, f.packageJsonOpt.flatMap(_.parsedTypes).orElse(f.packageJsonOpt.flatMap(_.parsedTypings))),
-            fromTypingsJson(f, f.packageJsonOpt.flatMap(_.parsedTypings)),
-          ).flatten
-
-        if (fromTypings.nonEmpty) fromTypings
-        else fromModuleDeclaration(f, f.packageJsonOpt.flatMap(_.parsedModules))
-    }
-  }
 }

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/LibraryResolver.scala

@@ -17,7 +17,7 @@ class LibraryResolver(
     value match {
       case LocalPath(localPath) =>
         file(folder, localPath).map { inFile =>
-          ResolvedModule.Local(inFile, LibraryResolver.moduleNameFor(source, inFile).head)
+          ResolvedModule.Local(inFile, LibraryResolver.moduleNameFor(source, inFile))
         }
       case globalRef =>
         val modName = ModuleNameParser(globalRef.split("/").to[List], keepIndexFragment = true)
@@ -56,44 +56,16 @@ object LibraryResolver {
   case class Ignored(name:      TsIdentLibrary) extends Res[Nothing]
   case class NotAvailable(name: TsIdentLibrary) extends Res[Nothing]
 
-  def moduleNameFor(source: LibTsSource, file: InFile): IArray[TsIdentModule] = {
-    val shortened: Option[TsIdentModule] =
-      if (source.shortenedFiles.contains(file)) {
-        source.libName match {
-          case TsIdentLibraryScoped(scope, name) =>
-            Some(TsIdentModule(Some(scope), List(name)))
-          case TsIdentLibrarySimple(value) =>
-            Some(TsIdentModule(None, value :: Nil))
-        }
-      } else None
-
-    val longName: TsIdentModule = {
-      val keepIndexPath = file.path match {
-        case base / segment / "index.d.ts" => files.exists(base / segment.concat(".d.ts"))
-        case _                             => false
-      }
-
-      ModuleNameParser(
-        source.libName.`__value` +: file.path.relativeTo(source.folder.path).segments.to[List],
-        keepIndexPath,
-      )
-    }
-
-    val ret = IArray.fromOptions(shortened, Some(longName))
-
-    /* some libraries contain multiple directory trees with type definitions, and refer to just one of them through
-     * `typings` in package.json for instance.
-     *
-     * Remarkably it can reach into one of the other trees even if the current tree has everything needed.
-     * This resolves the most common case, and fixes antd 4 in particular */
-    val inParallelDirectory = ret.collect {
-      case TsIdentModule(scopeOpt, fragments) if fragments.contains("lib") =>
-        TsIdentModule(scopeOpt, fragments.map { case "lib" => "es"; case other => other })
-      case TsIdentModule(scopeOpt, fragments) if fragments.contains("es") =>
-        TsIdentModule(scopeOpt, fragments.map { case "es" => "lib"; case other => other })
+  def moduleNameFor(source: LibTsSource, file: InFile): TsIdentModule = {
+    val keepIndexPath = file.path match {
+      case base / segment / "index.d.ts" => files.exists(base / segment.concat(".d.ts"))
+      case _                             => false
     }
 
-    ret ++ inParallelDirectory
+    ModuleNameParser(
+      source.libName.`__value` +: file.path.relativeTo(source.folder.path).segments.to[List],
+      keepIndexPath,
+    )
   }
 
   def file(within: InFolder, fragment: String): Option[InFile] =

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/PathsFromTsLibSource.scala

@@ -15,8 +15,6 @@ object PathsFromTsLibSource {
          */
         case "amd" => true
         case "umd" => true
-        case "es"  => true
-        case "es6" => true
         /* DefinitelyTyped uses this pattern for newer versions of typescript. We just use the default */
         case TS() => true
         /* DefinitelyTyped uses this pattern for old versions of the library */

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Phase1ReadTypescript.scala

@@ -54,11 +54,9 @@ class Phase1ReadTypescript(
               PathsFromTsLibSource.filesFrom(files.head.folder)
             case f @ LibTsSource.FromFolder(_, TsIdentLibrarySimple("typescript")) =>
               /* don't include std */
-              f.shortenedFiles
+              PathsFromTsLibSource.filesFrom(f.folder).filterNot(_.path.segments.contains("lib"))
             case f: LibTsSource.FromFolder =>
-              /* There are often whole trees parallel to what is specified in `typings` (or similar). This ignores them */
-              val bound = f.shortenedFiles.headOption.map(_.folder).getOrElse(f.folder)
-              PathsFromTsLibSource.filesFrom(bound)
+              PathsFromTsLibSource.filesFrom(f.folder)
           }
 
         val includedViaDirective = mutable.Set.empty[InFile]
@@ -84,26 +82,26 @@ class Phase1ReadTypescript(
                           LibraryResolver.file(resolve.stdLib.folder, s"lib.$value.d.ts").toRight(dir)
                       }
 
-                    val moduleNames = LibraryResolver.moduleNameFor(source, file)
+                    val moduleName = LibraryResolver.moduleNameFor(source, file)
 
-                    val withInferredModule = modules.InferredDefaultModule(parsed, moduleNames.head, logger)
+                    val withInferredModule = modules.InferredDefaultModule(parsed, moduleName, fileLogger)
 
                     withInferredModule.directives.foreach {
                       case dir @ Directive.TypesRef(value) =>
                         resolve.module(source, file.folder, value) match {
                           case Some(ResolvedModule.NotLocal(depSource, _)) =>
                             deps += depSource
                           case Some(ResolvedModule.Local(depSource, _)) =>
-                            logger.fatal(s"unexpected typeref from local file $depSource")
+                            fileLogger.fatal(s"unexpected typeref from local file $depSource")
                           case _ =>
-                            logger.warn(s"directives: couldn't resolve $dir")
+                            fileLogger.warn(s"directives: couldn't resolve $dir")
                         }
                       case _ => ()
                     }
 
                     /* Resolve all references to other modules in `from` clauses, rename modules */
                     val ResolveExternalReferences.Result(withExternals, resolvedModules, unresolvedModules) =
-                      ResolveExternalReferences(resolve, source, file.folder, withInferredModule, logger)
+                      ResolveExternalReferences(resolve, source, file.folder, withInferredModule, fileLogger)
 
                     resolvedModules.foreach {
                       case ResolvedModule.NotLocal(source, _) => deps += source
@@ -129,15 +127,15 @@ class Phase1ReadTypescript(
                         source.libName,
                         withOrigin,
                         unresolvedModules,
-                        logger,
+                        fileLogger,
                       )
 
                     inferredDepNames.foreach { libraryName =>
                       resolve.module(source, file.folder, libraryName.value) match {
                         case Some(ResolvedModule.NotLocal(dep, _)) =>
                           deps += dep
                         case _ =>
-                          logger.warn(s"Couldn't resolve inferred dependency ${libraryName.value}")
+                          fileLogger.warn(s"Couldn't resolve inferred dependency ${libraryName.value}")
                       }
                     }
 
@@ -163,18 +161,10 @@ class Phase1ReadTypescript(
                           parsed
                       }
 
-                    val _3 = moduleNames match {
-                      case IArray.exactlyOne(_) => withInlined
-                      case more =>
-                        withInlined.copy(members = withInlined.members.map {
-                          case m: TsDeclModule if more.contains(m.name) =>
-                            m.copy(comments = m.comments + Marker.ModuleAliases(more.filterNot(_ === m.name)))
-                          case other => other
-                        })
-                    }
-                    val _4 = T.SetCodePath.visitTsParsedFile(CodePath.HasPath(source.libName, TsQIdent.empty))(_3)
+                    val _3 =
+                      T.SetCodePath.visitTsParsedFile(CodePath.HasPath(source.libName, TsQIdent.empty))(withInlined)
 
-                    (_4, deps.result())
+                    (_3, deps.result())
                 }
               }
             }
@@ -212,7 +202,20 @@ class Phase1ReadTypescript(
         val flattened     = FlattenTrees(preparedFiles.map(_._1))
         val depsFromFiles = preparedFiles.foldLeft(Set.empty[LibTsSource]) { case (acc, (_, deps)) => acc ++ deps }
 
-        val withExportedModules = source.packageJsonOpt.flatMap(_.parsedExported).foldLeft(flattened) {
+        val topLevelProxyModule = {
+          val maybeProxy: Option[ProxyModule] =
+            source match {
+              case _:      LibTsSource.StdLibSource => None
+              case source: LibTsSource.FromFolder =>
+                ProxyModule.topLevel(source, existing = flattened.membersByName.contains)
+            }
+
+          maybeProxy.foldLeft(flattened) {
+            case (file, pm) => file.copy(members = pm.asModule +: file.members)
+          }
+        }
+
+        val withExportedModules = source.packageJsonOpt.flatMap(_.parsedExported).foldLeft(topLevelProxyModule) {
           case (file, exports) =>
             val proxyModules = ProxyModule.fromExports(
               source,

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ProxyModule.scala

@@ -28,7 +28,64 @@ case class ProxyModule(
 }
 
 object ProxyModule {
-  val FromExports = Comments("/* from `exports` in `package.json` */\n")
+  val FromExports     = Comments("/* from `exports` in `package.json` */\n")
+  val TopLevelModule = Comments("/* Inferred short module name */\n")
+
+  /* some things kind of implicitly end up being the top-level module. this will add such a default module if none exist */
+  def topLevel(source: LibTsSource.FromFolder, existing: TsIdent => Boolean): Option[ProxyModule] = {
+    def fromTypingsJson(files: Option[IArray[String]]): IArray[InFile] =
+      files.getOrElse(IArray.Empty).collect {
+        case path if path.endsWith("typings.json") =>
+          val typingsJsonPath = source.folder.path / os.RelPath(path)
+          val typingsJson     = Json.force[TypingsJson](typingsJsonPath)
+          InFile(typingsJsonPath / os.up / typingsJson.main)
+      }
+
+    def fromFileEntry(files: Option[IArray[String]]): IArray[InFile] =
+      files.getOrElse(IArray.Empty).mapNotNone(file => LibraryResolver.file(source.folder, file))
+
+    def fromModuleDeclaration(files: Option[Map[String, String]]): IArray[InFile] = {
+      val files1 = files match {
+        case Some(files) => IArray.fromTraversable(files.values)
+        case None        => IArray.Empty
+      }
+
+      files1
+        .mapNotNone(file => LibraryResolver.file(source.folder, file))
+        .mapNotNone {
+          case existingFile if LibTsSource.hasTypescriptSources(existingFile.folder) => Some(existingFile)
+          case _                                                                     => None
+        }
+    }
+
+    val found = {
+      val fromTypings = {
+        val types = source.packageJsonOpt.flatMap(_.parsedTypes).orElse(source.packageJsonOpt.flatMap(_.parsedTypings))
+        IArray(
+          fromFileEntry(types),
+          fromTypingsJson(source.packageJsonOpt.flatMap(_.parsedTypings)),
+        ).flatten
+      }
+
+      if (fromTypings.nonEmpty) fromTypings
+      else fromModuleDeclaration(source.packageJsonOpt.flatMap(_.parsedModules))
+    }
+
+    val sorted = found.sortBy(inFile => (inFile.path.last.startsWith("index"), inFile.path.toString().length))
+    if (sorted.length > 1) {
+      print(0)
+    }
+    val maybeChosenFile =
+      sorted.lastOption
+
+    maybeChosenFile
+      .map { chosenFile =>
+        val fromModule = LibraryResolver.moduleNameFor(source, chosenFile)
+        val toModule   = TsIdentModule.fromLibrary(source.libName)
+        ProxyModule(TopLevelModule, source.libName, fromModule, toModule)
+      }
+      .filterNot(pm => existing(pm.toModule))
+  }
 
   def fromExports(
       source:   LibTsSource,

tests/antd/in/antd/lib/form/Form.d.ts

@@ -1,4 +1,4 @@
-import { FormProps as RcFormProps } from 'rc-field-form/lib/Form';
+import { FormProps as RcFormProps } from 'rc-field-form/es/Form';
 
 export interface FormProps extends Omit<RcFormProps, 'form'> {
     prefixCls?: string;

tests/commander/check-3/c/commander/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "commander"
-version := "2.15.1-fc4aa5"
+version := "2.15.1-0e0384"
 scalaVersion := "3.1.2"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(