[WIP] Import typespec project into a single file

.vscode/launch.json

@@ -93,6 +93,27 @@
         "order": 2
       }
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug importer",
+      "program": "${workspaceFolder}/packages/importer/dist/cli.js",
+      "args": [
+        // "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/batch/Azure.Batch/main.tsp"
+        "main.tsp"
+      ],
+      "smartStep": true,
+      "sourceMaps": true,
+      "skipFiles": ["<node_internals>/**/*.js"],
+      "outFiles": [
+        "${workspaceFolder}/packages/*/dist/**/*.js",
+        "${workspaceFolder}/packages/*/dist-dev/**/*.js"
+      ],
+      "cwd": "${workspaceFolder}/packages/samples/scratch",
+      "presentation": {
+        "order": 2
+      }
+    },
     {
       "name": "Regenerate .tmlanguage",
       "type": "node",

packages/compiler/src/core/formatter.ts

@@ -1,6 +1,24 @@
 import type { Options } from "prettier";
 import { check, format } from "prettier/standalone";
 import * as typespecPrettierPlugin from "../formatter/index.js";
+import type { Node } from "./types.js";
+
+export function printTypeSpecNode(node: Node): Promise<string> {
+  return format(".", {
+    parser: "typespec",
+    plugins: [
+      {
+        ...typespecPrettierPlugin,
+        parsers: {
+          typespec: {
+            ...typespecPrettierPlugin.parsers.typespec,
+            parse: () => node,
+          },
+        },
+      },
+    ],
+  });
+}
 
 export async function formatTypeSpec(code: string, prettierConfig?: Options): Promise<string> {
   const output = await format(code, {

packages/compiler/src/core/index.ts

@@ -53,5 +53,6 @@ export { isProjectedProgram } from "./projected-program.js";
 export * from "./scanner.js";
 export * from "./semantic-walker.js";
 export { createSourceFile, getSourceFileKindFromExt } from "./source-file.js";
+export { createSourceLoader } from "./source-loader.js";
 export * from "./type-utils.js";
 export * from "./types.js";

packages/compiler/src/core/module-resolver.ts

@@ -109,9 +109,10 @@ export async function resolveModule(
   const { baseDir } = options;
   const absoluteStart = baseDir === "" ? "." : await realpath(resolvePath(baseDir));
 
-  if (!(await isDirectory(host, absoluteStart))) {
-    throw new TypeError(`Provided basedir '${baseDir}'is not a directory.`);
-  }
+  // TODO: can this check be disabled?
+  // if (!(await isDirectory(host, absoluteStart))) {
+  //   throw new TypeError(`Provided basedir '${baseDir}'is not a directory.`);
+  // }
 
   // Check if the module name is referencing a path(./foo, /foo, file:/foo)
   if (/^(?:\.\.?(?:\/|$)|\/|([A-Za-z]:)?[/\\])/.test(name)) {
@@ -123,6 +124,7 @@ export async function resolveModule(
   }
 
   const module = await findAsNodeModule(name, absoluteStart);
+
   if (module) return module;
 
   throw new ResolveModuleError(

packages/compiler/src/core/source-loader.ts

@@ -38,6 +38,9 @@ export interface SourceResolution {
   readonly locationContexts: WeakMap<SourceFile, LocationContext>;
   readonly loadedLibraries: Map<string, TypeSpecLibraryReference>;
 
+  /** List of imports that were marked as external and not loaded. */
+  readonly externals: string[];
+
   readonly diagnostics: readonly Diagnostic[];
 }
 
@@ -50,6 +53,10 @@ export interface LoadSourceOptions {
   readonly parseOptions?: ParseOptions;
   readonly tracer?: Tracer;
   getCachedScript?: (file: SourceFile) => TypeSpecScriptNode | undefined;
+  /**
+   * List or callback to determine if a module is external and should not be loaded.
+   */
+  externals?: string[] | ((path: string) => boolean);
 }
 
 export interface SourceLoader {
@@ -83,10 +90,18 @@ export async function createSourceLoader(
   const sourceFiles = new Map<string, TypeSpecScriptNode>();
   const jsSourceFiles = new Map<string, JsSourceFileNode>();
   const loadedLibraries = new Map<string, TypeSpecLibraryReference>();
+  const externals: string[] = [];
+
+  const externalsOpts = options?.externals;
+  const isExternal = externalsOpts
+    ? typeof externalsOpts === "function"
+      ? externalsOpts
+      : (x: string) => externalsOpts.includes(x)
+    : () => false;
 
   async function importFile(
     path: string,
-    locationContext: LocationContext,
+    locationContext: LocationContext = { type: "project" },
     kind: "import" | "entrypoint" = "import"
   ) {
     const sourceFileKind = host.getSourceFileKind(path);
@@ -117,6 +132,7 @@ export async function createSourceLoader(
       locationContexts: sourceFileLocationContexts,
       loadedLibraries: loadedLibraries,
       diagnostics: diagnostics.diagnostics,
+      externals,
     },
   };
 
@@ -181,7 +197,11 @@ export async function createSourceLoader(
 
   function getSourceFileLocationContext(sourcefile: SourceFile): LocationContext {
     const locationContext = sourceFileLocationContexts.get(sourcefile);
-    compilerAssert(locationContext, "SourceFile should have a declaration locationContext.");
+    compilerAssert(
+      locationContext,
+      `SourceFile ${sourcefile.path} should have a declaration locationContext.`,
+      { file: sourcefile, pos: 0, end: 0 }
+    );
     return locationContext;
   }
 
@@ -202,10 +222,15 @@ export async function createSourceLoader(
     relativeTo: string,
     locationContext: LocationContext = { type: "project" }
   ) {
+    if (isExternal(path)) {
+      externals.push(path);
+      return;
+    }
     const library = await resolveTypeSpecLibrary(path, relativeTo, target);
     if (library === undefined) {
       return;
     }
+
     if (library.type === "module") {
       loadedLibraries.set(library.manifest.name, {
         path: library.path,
@@ -222,8 +247,8 @@ export async function createSourceLoader(
         metadata,
       };
     }
-    const importFilePath = library.type === "module" ? library.mainFile : library.path;
 
+    const importFilePath = library.type === "module" ? library.mainFile : library.path;
     const isDirectory = (await host.stat(importFilePath)).isDirectory();
     if (isDirectory) {
       await loadDirectory(importFilePath, locationContext, target);

packages/compiler/src/formatter/parser.ts

@@ -25,7 +25,7 @@ export function parse(text: string, options: ParserOptions<any>): TypeSpecScript
  * This causes prettier to not know where comments belong.
  * https://github.com/microsoft/typespec/pull/2061
  */
-function flattenNamespaces(base: Node) {
+export function flattenNamespaces(base: Node) {
   visitChildren(base, (node) => {
     if (node.kind === SyntaxKind.NamespaceStatement) {
       let current = node;

packages/compiler/src/formatter/print/printer.ts

@@ -520,7 +520,7 @@ export function printComment(
     case SyntaxKind.BlockComment:
       return printBlockComment(commentPath as AstPath<BlockComment>, options);
     case SyntaxKind.LineComment:
-      return `${options.originalText.slice(comment.pos, comment.end).trimEnd()}`;
+      return `${getRawText(comment, options).trimEnd()}`;
     default:
       throw new Error(`Not a comment: ${JSON.stringify(comment)}`);
   }
@@ -1660,7 +1660,7 @@ function printNumberLiteral(
   options: TypeSpecPrettierOptions
 ): Doc {
   const node = path.node;
-  return getRawText(node, options);
+  return node.valueAsString;
 }
 
 function printBooleanLiteral(
@@ -1975,7 +1975,10 @@ function printItemList<T extends Node>(
  * @param options Prettier options
  * @returns Raw text in the file for the given node.
  */
-function getRawText(node: TextRange, options: TypeSpecPrettierOptions) {
+function getRawText(node: TextRange, options: TypeSpecPrettierOptions): string {
+  if ("rawText" in node) {
+    return node.rawText as string;
+  }
   return options.originalText.slice(node.pos, node.end);
 }
 

packages/compiler/src/index.ts

@@ -4,7 +4,6 @@ export * from "./lib/decorators.js";
 export * from "./server/index.js";
 import * as formatter from "./formatter/index.js";
 export const TypeSpecPrettierPlugin = formatter;
-
 // DO NOT ADD ANYMORE EXPORTS HERE, this is for backcompat. Utils should be exported from the utils folder.
 export {
   /** @deprecated use import from  @typespec/compiler/utils */

packages/importer/cmd/cli.js

@@ -0,0 +1,5 @@
+/**
+ * File serving as an entrypoint to resolve a local tsp install from a global install.
+ * DO NOT MOVE or this will create a breaking change for user of global cli.
+ */
+import "../dist/cli.js";

packages/importer/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@typespec/importer",
+  "private": true,
+  "version": "0.0.1",
+  "author": "Microsoft Corporation",
+  "description": "Package to import TypeSpec files into a single one",
+  "homepage": "https://typespec.io",
+  "readme": "https://github.com/microsoft/typespec/blob/main/README.md",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/typespec.git"
+  },
+  "bugs": {
+    "url": "https://github.com/microsoft/typespec/issues"
+  },
+  "keywords": [
+    "typespec"
+  ],
+  "type": "module",
+  "main": "dist/src/index.js",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "bin": "dist/src/cli.js",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "bundle": "node ./dist/src/cli.js",
+    "clean": "rimraf ./dist ./temp",
+    "build": "tsc -p .",
+    "watch": "tsc -p . --watch",
+    "test": "vitest run",
+    "test:ui": "vitest --ui",
+    "test:ci": "vitest run --coverage --reporter=junit --reporter=default",
+    "lint": "eslint . --max-warnings=0",
+    "lint:fix": "eslint . --fix"
+  },
+  "files": [
+    "lib/*.tsp",
+    "dist/**",
+    "!dist/test/**"
+  ],
+  "dependencies": {
+    "@typespec/compiler": "workspace:~",
+    "picocolors": "~1.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "~18.11.19",
+    "@vitest/coverage-v8": "^1.6.0",
+    "@vitest/ui": "^1.6.0",
+    "c8": "^10.1.2",
+    "rimraf": "~5.0.7",
+    "source-map-support": "~0.5.21",
+    "typescript": "~5.5.3",
+    "vite": "^5.3.2",
+    "vitest": "^1.6.0"
+  }
+}

packages/importer/src/cli.ts

@@ -0,0 +1,42 @@
+try {
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  await import("source-map-support/register.js");
+} catch {
+  // package only present in dev.
+}
+
+import { getDirectoryPath, logDiagnostics, NodeHost, normalizePath } from "@typespec/compiler";
+import { mkdir, writeFile } from "fs/promises";
+import { resolve } from "path";
+import { parseArgs } from "util";
+import { ImporterHost } from "./importer-host.js";
+import { combineProjectIntoFile } from "./importer.js";
+
+function log(...args: any[]) {
+  // eslint-disable-next-line no-console
+  console.log(...args);
+}
+const args = parseArgs({
+  options: {},
+  args: process.argv.slice(2),
+  allowPositionals: true,
+});
+
+const rawEntrypoint = normalizePath(resolve(args.positionals[0]));
+
+const { content, diagnostics } = await combineProjectIntoFile(ImporterHost, rawEntrypoint);
+
+if (diagnostics.length > 0) {
+  logDiagnostics(diagnostics, NodeHost.logSink);
+  process.exit(1);
+}
+if (content) {
+  const outputFile = "tsp-output/main.tsp";
+  await mkdir(getDirectoryPath(outputFile), { recursive: true });
+  log(`Writing output to ${outputFile}`);
+  await writeFile(outputFile, content);
+  process.exit(0);
+} else {
+  process.exit(1);
+}

packages/importer/src/importer-host.ts

@@ -0,0 +1,10 @@
+import { NodeHost, type CompilerHost } from "@typespec/compiler";
+import { createRemoteHost } from "./remote-host.js";
+
+/**
+ * Special host that tries to load data from additional locations
+ */
+export const ImporterHost: CompilerHost = {
+  ...NodeHost,
+  ...createRemoteHost(NodeHost),
+};