chore: Expose async API for parsing C++ code

Signed-off-by: Roberto Raggi <[email protected]>
robertoraggi · Nov 10, 2024 · 5ee8198 · 5ee8198
1 parent d0ae3e4
commit 5ee8198
Show file tree

Hide file tree

Showing 12 changed files with 217 additions and 138 deletions.
diff --git a/packages/cxx-frontend/src/Parser.ts b/packages/cxx-frontend/src/Parser.ts
@@ -22,7 +22,7 @@ import initCxx, { cxx } from "./cxx";
 import { Unit } from "./Unit";
 import { AST } from "./AST";
 
-interface ParseParams {
+interface ParserParams {
   /**
    * Path to the file to parse.
    */
@@ -32,6 +32,12 @@ interface ParseParams {
    * Source code to parse.
    */
   source: string;
+
+  resolve?: (
+    name: string,
+    kind: "quoted" | "angled",
+    next: boolean,
+  ) => Promise<string | undefined>;
 }
 
 export class Parser {
@@ -69,8 +75,8 @@ export class Parser {
     return cxx !== undefined && cxx !== null;
   }
 
-  constructor(options: ParseParams) {
-    const { path, source } = options;
+  constructor(options: ParserParams) {
+    const { path, source, resolve } = options;
 
     if (typeof path !== "string") {
       throw new TypeError("expected parameter 'path' of type 'string'");
@@ -80,14 +86,14 @@ export class Parser {
       throw new TypeError("expected parameter 'source' of type 'string'");
     }
 
-    this.#unit = cxx.createUnit(source, path);
+    this.#unit = cxx.createUnit(source, path, { resolve });
   }
 
-  parse() {
+  async parse() {
     if (!this.#unit) {
       return;
     }
-    this.#unit.parse();
+    await this.#unit.parse();
     this.#ast = AST.from(this.#unit.getHandle(), this);
   }
 

diff --git a/packages/cxx-frontend/src/Preprocessor.ts b/packages/cxx-frontend/src/Preprocessor.ts
@@ -22,11 +22,6 @@ import { cxx } from "./cxx";
 
 interface PreprocessorOptions {
   systemIncludePaths?: string[];
-
-  fs?: {
-    existsSync(path: string): boolean;
-    readFileSync(path: string): string;
-  };
 }
 
 export class Preprocessor {
@@ -39,7 +34,7 @@ export class Preprocessor {
    *
    * @param source
    */
-  constructor({ systemIncludePaths, fs }: PreprocessorOptions = {}) {
+  constructor({ systemIncludePaths }: PreprocessorOptions = {}) {
     this.#control = new cxx.Control();
     this.#diagnosticClient = new cxx.DiagnosticsClient();
     this.#handle = new cxx.Preprocessor(this.#control, this.#diagnosticClient);
@@ -48,13 +43,6 @@ export class Preprocessor {
     systemIncludePaths?.forEach((path) => {
       this.#handle.addIncludePath(path);
     });
-
-    if (fs) {
-      const { existsSync, readFileSync } = fs;
-
-      this.#handle.setCanResolveFiles(true);
-      this.#handle.setup(existsSync, readFileSync);
-    }
   }
 
   /**

diff --git a/packages/cxx-frontend/src/Unit.ts b/packages/cxx-frontend/src/Unit.ts
@@ -22,7 +22,7 @@ import { Diagnostic } from "./Diagnostic";
 
 export interface Unit {
   delete(): void;
-  parse(): boolean;
+  parse(): Promise<boolean>;
   getHandle(): number;
   getUnitHandle(): number;
   getDiagnostics(): Diagnostic[];

diff --git a/packages/cxx-frontend/src/cxx-js.d.ts b/packages/cxx-frontend/src/cxx-js.d.ts
@@ -83,14 +83,22 @@ interface Lexer {
   tokenText(): string;
 }
 
+interface Api {
+  resolve?: (
+    name: string,
+    kind: "quoted" | "angled",
+    isIncludeNext: boolean,
+  ) => Promise<string | undefined>;
+}
+
 export type CXX = {
   Control: Control;
   DiagnosticsClient: DiagnosticsClient;
   Preprocessor: Preprocessor;
   Lexer: Lexer;
   TranslationUnit: TranslationUnit;
 
-  createUnit(source: string, path: string): Unit;
+  createUnit(source: string, path: string, api: Api): Unit;
   getASTKind(handle: number): number;
   getASTSlot(handle: number, slot: number): number;
   getASTSlotKind(handle: number, slot: number): ASTSlotKind;

diff --git a/src/js/cxx/api.cc b/src/js/cxx/api.cc
@@ -32,6 +32,7 @@
 #include <emscripten/val.h>
 
 #include <format>
+#include <iostream>
 #include <sstream>
 
 using namespace emscripten;
@@ -61,42 +62,121 @@ struct DiagnosticsClient final : cxx::DiagnosticsClient {
 struct WrappedUnit {
   std::unique_ptr<DiagnosticsClient> diagnosticsClient;
   std::unique_ptr<cxx::TranslationUnit> unit;
+  val api;
 
-  WrappedUnit(std::string source, std::string filename) {
+  WrappedUnit(std::string source, std::string filename, val api = {})
+      : api(api) {
     diagnosticsClient = std::make_unique<DiagnosticsClient>();
+
     unit = std::make_unique<cxx::TranslationUnit>(diagnosticsClient.get());
+
     if (auto preprocessor = unit->preprocessor()) {
-      preprocessor->setCanResolveFiles(false);
+      preprocessor->setCanResolveFiles(true);
     }
 
-    unit->setSource(std::move(source), std::move(filename));
+    unit->beginPreprocessing(std::move(source), std::move(filename));
+  }
+
+  auto getUnitHandle() const -> std::intptr_t {
+    return (std::intptr_t)unit.get();
   }
 
-  std::intptr_t getUnitHandle() const { return (std::intptr_t)unit.get(); }
+  auto getHandle() const -> std::intptr_t { return (std::intptr_t)unit->ast(); }
 
-  std::intptr_t getHandle() const { return (std::intptr_t)unit->ast(); }
+  auto getDiagnostics() const -> val { return diagnosticsClient->messages; }
 
-  val getDiagnostics() const { return diagnosticsClient->messages; }
+  auto parse() -> val {
+    val resolve = val::undefined();
+
+    if (!api.isUndefined()) {
+      resolve = api["resolve"];
+    }
+
+    struct {
+      auto operator()(const cxx::SystemInclude& include) -> val {
+        return val(include.fileName);
+      }
+      auto operator()(const cxx::QuoteInclude& include) -> val {
+        return val(include.fileName);
+      }
+    } getHeaderName;
+
+    struct {
+      val quoted{"quoted"};
+      val angled{"angled"};
+
+      auto operator()(const cxx::SystemInclude& include) -> val {
+        return angled;
+      }
+      auto operator()(const cxx::QuoteInclude& include) -> val {
+        return quoted;
+      }
+    } getIncludeType;
+
+    while (true) {
+      auto state = unit->continuePreprocessing();
+
+      if (std::holds_alternative<cxx::ProcessingComplete>(state)) break;
+
+      if (auto pendingInclude = std::get_if<cxx::PendingInclude>(&state)) {
+        if (resolve.isUndefined()) {
+          pendingInclude->resolveWith(std::nullopt);
+          continue;
+        }
+
+        auto header = std::visit(getHeaderName, pendingInclude->include);
+        auto includeType = std::visit(getIncludeType, pendingInclude->include);
+
+        val resolved = co_await resolve(header, includeType,
+                                        pendingInclude->isIncludeNext);
+
+        if (resolved.isString()) {
+          pendingInclude->resolveWith(resolved.as<std::string>());
+        } else {
+          pendingInclude->resolveWith(std::nullopt);
+        }
+
+      } else if (auto pendingHasIncludes =
+                     std::get_if<cxx::PendingHasIncludes>(&state)) {
+        for (auto& request : pendingHasIncludes->requests) {
+          if (resolve.isUndefined()) {
+            request.setExists(false);
+            continue;
+          }
+
+          auto header = std::visit(getHeaderName, request.include);
+          auto includeType = std::visit(getIncludeType, request.include);
+
+          val resolved =
+              co_await resolve(header, includeType, request.isIncludeNext);
+
+          request.setExists(resolved.isString());
+        }
+      }
+    }
+
+    unit->endPreprocessing();
 
-  bool parse() {
     unit->parse();
-    return true;
+
+    co_return val{true};
   }
 };
 
-std::string getTokenText(std::intptr_t handle, std::intptr_t unitHandle) {
+auto getTokenText(std::intptr_t handle, std::intptr_t unitHandle)
+    -> std::string {
   auto unit = reinterpret_cast<cxx::TranslationUnit*>(unitHandle);
   auto text = unit->tokenText(cxx::SourceLocation(handle));
   return text;
 }
 
-int getTokenKind(std::intptr_t handle, std::intptr_t unitHandle) {
+auto getTokenKind(std::intptr_t handle, std::intptr_t unitHandle) -> int {
   auto unit = reinterpret_cast<cxx::TranslationUnit*>(unitHandle);
   auto kind = unit->tokenKind(cxx::SourceLocation(handle));
   return static_cast<int>(kind);
 }
 
-val getTokenLocation(std::intptr_t handle, std::intptr_t unitHandle) {
+auto getTokenLocation(std::intptr_t handle, std::intptr_t unitHandle) -> val {
   auto unit = reinterpret_cast<cxx::TranslationUnit*>(unitHandle);
 
   cxx::SourceLocation loc(handle);
@@ -114,71 +194,71 @@ val getTokenLocation(std::intptr_t handle, std::intptr_t unitHandle) {
   return result;
 }
 
-val getStartLocation(std::intptr_t handle, std::intptr_t unitHandle) {
+auto getStartLocation(std::intptr_t handle, std::intptr_t unitHandle) -> val {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   const auto loc = ast->firstSourceLocation();
   if (!loc) return {};
   return getTokenLocation(loc.index(), unitHandle);
 }
 
-val getEndLocation(std::intptr_t handle, std::intptr_t unitHandle) {
+auto getEndLocation(std::intptr_t handle, std::intptr_t unitHandle) -> val {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   const auto loc = ast->lastSourceLocation().previous();
   if (!loc) return {};
   return getTokenLocation(loc.index(), unitHandle);
 }
 
-val getIdentifierValue(std::intptr_t handle) {
+auto getIdentifierValue(std::intptr_t handle) -> val {
   auto id = reinterpret_cast<const cxx::Identifier*>(handle);
   if (!id) return {};
   return val(id->value());
 }
 
-val getLiteralValue(std::intptr_t handle) {
+auto getLiteralValue(std::intptr_t handle) -> val {
   auto id = reinterpret_cast<const cxx::Literal*>(handle);
   if (!id) return {};
   return val(id->value());
 }
 
-int getASTKind(std::intptr_t handle) {
+auto getASTKind(std::intptr_t handle) -> int {
   return static_cast<int>(((cxx::AST*)handle)->kind());
 }
 
-int getListValue(std::intptr_t handle) {
+auto getListValue(std::intptr_t handle) -> int {
   auto list = reinterpret_cast<cxx::List<cxx::AST*>*>(handle);
   return std::intptr_t(list->value);
 }
 
-std::intptr_t getListNext(std::intptr_t handle) {
+auto getListNext(std::intptr_t handle) -> std::intptr_t {
   auto list = reinterpret_cast<cxx::List<cxx::AST*>*>(handle);
   return std::intptr_t(list->next);
 }
 
-std::intptr_t getASTSlot(std::intptr_t handle, int slot) {
+auto getASTSlot(std::intptr_t handle, int slot) -> std::intptr_t {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   auto [value, slotKind, slotNameIndex, slotCount] = getSlot(ast, slot);
   return value;
 }
 
-int getASTSlotKind(std::intptr_t handle, int slot) {
+auto getASTSlotKind(std::intptr_t handle, int slot) -> int {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   auto [value, slotKind, slotNameIndex, slotCount] = getSlot(ast, slot);
   return static_cast<int>(slotKind);
 }
 
-int getASTSlotName(std::intptr_t handle, int slot) {
+auto getASTSlotName(std::intptr_t handle, int slot) -> int {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   auto [value, slotKind, slotName, slotCount] = getSlot(ast, slot);
   return static_cast<int>(slotName);
 }
 
-int getASTSlotCount(std::intptr_t handle, int slot) {
+auto getASTSlotCount(std::intptr_t handle, int slot) -> int {
   auto ast = reinterpret_cast<cxx::AST*>(handle);
   auto [value, slotKind, slotNameIndex, slotCount] = getSlot(ast, slot);
   return static_cast<int>(slotCount);
 }
 
-WrappedUnit* createUnit(std::string source, std::string filename) {
+auto createUnit(std::string source, std::string filename) -> WrappedUnit* {
   auto wrapped = new WrappedUnit(std::move(source), std::move(filename));
 
   return wrapped;
@@ -196,17 +276,6 @@ auto lexerNext(cxx::Lexer& lexer) -> int {
   return static_cast<int>(lexer.next());
 }
 
-void preprocessorSetup(cxx::Preprocessor& preprocessor, val fileExistsFn,
-                       val readFileFn) {
-  preprocessor.setFileExistsFunction([fileExistsFn](std::string fileName) {
-    return fileExistsFn(fileName).as<bool>();
-  });
-
-  preprocessor.setReadFileFunction([readFileFn](std::string fileName) {
-    return readFileFn(fileName).as<std::string>();
-  });
-}
-
 auto preprocesorPreprocess(cxx::Preprocessor& preprocessor, std::string source,
                            std::string filename) -> std::string {
   std::vector<cxx::Token> tokens;
@@ -241,7 +310,6 @@ auto register_preprocessor(const char* name = "Preprocessor")
   return class_<cxx::Preprocessor>(name)
       .constructor<cxx::Control*, cxx::DiagnosticsClient*>()
       .function("preprocess", &preprocesorPreprocess)
-      .function("setup", &preprocessorSetup)
       .function("addIncludePath", &cxx::Preprocessor::addSystemIncludePath)
       .function("defineMacro", &cxx::Preprocessor::defineMacro)
       .function("undefineMacro", &cxx::Preprocessor::undefMacro)
@@ -283,7 +351,7 @@ auto register_translation_unit(const char* name = "TranslationUnit")
 
 }  // namespace
 
-EMSCRIPTEN_BINDINGS(my_module) {
+EMSCRIPTEN_BINDINGS(cxx) {
   register_control();
   register_diagnostics_client();
   register_preprocessor();

diff --git a/src/parser/cxx/cxx_fwd.h b/src/parser/cxx/cxx_fwd.h
@@ -20,7 +20,6 @@
 
 #pragma once
 
-#include <cstdint>
 #include <string>
 
 namespace cxx {

diff --git a/src/parser/cxx/literals.h b/src/parser/cxx/literals.h
@@ -20,6 +20,7 @@
 
 #include <cxx/cxx_fwd.h>
 
+#include <cstdint>
 #include <string>
 #include <string_view>