Move all modules into a single AST and reactivate most visitors.

There's now a single `AST Root` node that all modules are children of.
AST management and processing has moved into the `ASTContext` class.
The driver's units now just refer to the modules inside that single,
joint AST; they are no longer an owner of the module AST.

Most visitors are reactivated to operate on that single AST, but many
don't have any actual functionality yet.

hilti/toolchain/CMakeLists.txt

@@ -35,6 +35,7 @@ autogen_dispatchers(
     ${CMAKE_CURRENT_SOURCE_DIR}/include/hilti/ast/nodes.decl ${AUTOGEN_H}/operators.decl)
 
 set(SOURCES
+    src/ast/ast-context.cc
     src/ast/attribute.cc
     src/ast/builder/builder.cc
     # src/ast/builder/type.cc
@@ -47,16 +48,14 @@ set(SOURCES
     src/ast/expression.cc
     src/ast/function.cc
     # src/ast/expressions
-    # src/ast/expressions/id.cc
+    src/ast/expressions/name.cc
     src/ast/location.cc
     src/ast/meta.cc
     src/ast/module.cc
     src/ast/node.cc
     src/ast/statement.cc
     src/ast/type.cc
-    # src/ast/node_ref.cc
     src/ast/scope.cc
-    # src/ast/scope-lookup.cc
     src/ast/statements/switch.cc
     src/ast/statements/try.cc
     # src/ast/type.cc
@@ -71,16 +70,15 @@ set(SOURCES
     src/base/preprocessor.cc
     src/base/timing.cc
     src/base/util.cc
-    # src/compiler/codegen/codegen.cc
-    # src/compiler/codegen/coercions.cc
-    # src/compiler/codegen/ctors.cc
-    # src/compiler/codegen/expressions.cc
-    # src/compiler/codegen/expressions.cc
-    # src/compiler/codegen/operators.cc
-    # src/compiler/codegen/statements.cc
-    # src/compiler/codegen/types.cc
-    # src/compiler/codegen/unpack.cc
-    # src/compiler/coercion.cc
+    src/compiler/codegen/codegen.cc
+    src/compiler/codegen/coercions.cc
+    src/compiler/codegen/ctors.cc
+    src/compiler/codegen/expressions.cc
+    src/compiler/codegen/operators.cc
+    src/compiler/codegen/statements.cc
+    src/compiler/codegen/types.cc
+    src/compiler/codegen/unpack.cc
+    src/compiler/coercion.cc
     src/compiler/context.cc
     src/compiler/cxx/elements.cc
     src/compiler/cxx/formatter.cc
@@ -93,14 +91,14 @@ set(SOURCES
     src/compiler/parser/driver.cc
     src/compiler/plugin.cc
     src/compiler/unit.cc
-    # src/compiler/visitors/coercer.cc
-    # src/compiler/visitors/constant-folder.cc
-    # src/compiler/visitors/normalizer.cc
+    src/compiler/visitors/coercer.cc
+    src/compiler/visitors/constant-folder.cc
+    src/compiler/visitors/normalizer.cc
     src/compiler/visitors/printer.cc
     src/compiler/visitors/renderer.cc
-    # src/compiler/visitors/resolver.cc
-    # src/compiler/visitors/scope-builder.cc
-    # src/compiler/visitors/validator.cc
+    src/compiler/visitors/resolver.cc
+    src/compiler/visitors/scope-builder.cc
+    src/compiler/visitors/validator.cc
     src/global.cc
     # # Already included in hilti-rt, which we pull in.
     # # src/3rdparty/utf8proc/utf8proc.c

hilti/toolchain/include/ast/ast-context.h

@@ -2,11 +2,119 @@
 
 #pragma once
 
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <hilti/ast/forward.h>
+#include <hilti/ast/module.h>
+#include <hilti/ast/node.h>
+#include <hilti/base/logger.h>
+
 namespace hilti {
 
-// Place-holder we'll use later for maintaining AST-global state.
-class ASTContext {
+class Context;
+class ASTContext;
+struct Plugin;
+
+/** Root node for the AST inside an AST context. This will always be present exactly once. */
+class ASTRoot : public Node {
 public:
+    ~ASTRoot() override;
+
+    static auto create(ASTContext* ctx) { return NodeDerivedPtr<ASTRoot>(new ASTRoot()); }
+
+protected:
+    ASTRoot() : Node({}, Meta(Location("<root>"))) {}
+
+    std::string _render() const override;
+
+    bool isEqual(const Node& other) const override { return other.isA<ASTRoot>(); }
+
+    HILTI_NODE(ASTRoot);
+};
+
+// Container for AST-wide state.
+class ASTContext : public std::enable_shared_from_this<ASTContext> {
+public:
+    enum ASTState { Modified, NotModified };
+
+    ASTContext(Context* context);
+
+    auto root() const { return _root; }
+
+    Result<module::UID> parseSource(const hilti::rt::filesystem::path& path,
+                                    std::optional<hilti::rt::filesystem::path> process_extension = {});
+
+    Result<module::UID> importModule(const ID& id, const std::optional<ID>& scope,
+                                     const hilti::rt::filesystem::path& parse_extension,
+                                     const std::optional<hilti::rt::filesystem::path>& process_extension,
+                                     std::vector<hilti::rt::filesystem::path> search_dirs);
+
+    ModulePtr getModule(const module::UID& uid) const {
+        if ( auto m = _modules_by_uid.find(uid); m != _modules_by_uid.end() )
+            return m->second;
+        else
+            return nullptr;
+    }
+
+    Result<Nothing> processAST(); // top-level entry point for resolving/validating/optimizing AST
+
+    bool isFullyResolved() const { return _resolved; }
+
+    /**
+     * Returns direct & indirect dependencies that a module imports. This
+     * information will be available only once the AST has been resolved
+     * successfully.
+     *
+     * @param uid UID of module to return dependencies for; the module must be known, otherwise an internal error is
+     * reported
+     * @param recursive if true, return the transitive closure of all
+     * dependent units, vs just direct dependencies of the current unit
+     * @return set of dependencies, or empty vector if not yet resolved
+     */
+    std::vector<module::UID> dependencies(const module::UID& uid, bool recursive = false) const;
+
+    /**
+     * Dumps the current total AST of all modules to a debug stream.
+     *
+     * @param stream debug stream to write to
+     */
+    void dumpAST(const hilti::logging::DebugStream& stream, const std::string& prefix);
+
+private:
+    Result<module::UID> _parseSource(const hilti::rt::filesystem::path& path, const std::optional<ID>& scope,
+                                     std::optional<hilti::rt::filesystem::path> process_extension = {});
+    module::UID _addModuleToAST(ModulePtr module);
+    Result<Nothing> _buildScopes(const Plugin& plugin);
+    Result<Nothing> _resolve(const Plugin& plugin);
+    Result<Nothing> _validate(const Plugin& plugin, bool pre_resolver);
+    Result<Nothing> _optimize(const Plugin& plugin);
+
+    void _saveIterationAST(const Plugin& plugin, const std::string& prefix, int round = 0);
+    void _saveIterationAST(const Plugin& plugin, const std::string& prefix, const std::string& tag);
+
+    void _dumpAST(const hilti::logging::DebugStream& stream, const Plugin& plugin, const std::string& prefix,
+                  int round);
+    void _dumpAST(std::ostream& stream, const Plugin& plugin, const std::string& prefix, int round);
+    void _dumpDeclarations(const Plugin& plugin);
+
+    // Reports any errors recorded in the AST to stderr.
+    //
+    // @returns false if there were errors, true if the AST is all good
+    bool _reportErrors();
+
+    Context* _context;
+    NodeDerivedPtr<ASTRoot> _root;
+    bool _rebuild_scopes = true;
+    bool _resolved = false;
+
+    std::unordered_map<module::UID, ModulePtr> _modules_by_uid;
+    std::unordered_map<std::string, ModulePtr> _modules_by_path;
+    std::map<std::pair<ID, ID>, ModulePtr> _modules_by_id_and_scope;
 };
 
 } // namespace hilti

hilti/toolchain/include/ast/builder/builder.h

@@ -11,7 +11,7 @@ namespace hilti {
 
 class Builder : public builder::NodeFactory {
 public:
-    Builder(std::shared_ptr<ASTContext> context) : NodeFactory(std::move(context)) {}
+    Builder(ASTContext* ctx) : NodeFactory(ctx) {}
 };
 
 using BuilderPtr = std::shared_ptr<Builder>;

hilti/toolchain/include/ast/builder/node-factory.autogen.h

@@ -6,7 +6,7 @@ auto attribute(std::string tag, const NodePtr& v, Meta m = Meta()) {
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/attribute.h:73:5
 auto attributeSet(Attributes attrs, Meta m = Meta()) {
     return hilti::AttributeSet::create(context(), attrs, m);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/attribute.h:151:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/attribute.h:154:5
 auto ctorAddress(hilti::rt::Address v, const Meta& meta = {}) {
     return hilti::ctor::Address::create(context(), v, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/ctors/address.h:29:5
@@ -122,6 +122,9 @@ auto ctorUnion(const QualifiedTypePtr& type, const ExpressionPtr& value, const M
 auto ctorUnsignedInteger(uint64_t value, int width, const Meta& meta = {}) {
     return hilti::ctor::UnsignedInteger::create(context(), value, width, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/ctors/integer.h:70:5
+auto ctorUnsignedInteger(uint64_t value, int width, const UnqualifiedTypePtr& t, const Meta& meta = {}) {
+    return hilti::ctor::UnsignedInteger::create(context(), value, width, t, meta);
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/ctors/integer.h:76:5
 auto ctorValueReference(const ExpressionPtr& expr, const Meta& meta = {}) {
     return hilti::ctor::ValueReference::create(context(), expr, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/ctors/reference.h:64:5
@@ -319,15 +322,16 @@ auto function(const ID& id, const type::FunctionPtr& ftype, const StatementPtr&
               const AttributeSetPtr& attrs = nullptr, const Meta& meta = {}) {
     return hilti::Function::create(context(), id, ftype, body, cc, attrs, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/function.h:66:5
-auto module(ID id = {}, const Meta& meta = {}) {
-    return hilti::Module::create(context(), id, meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:89:5
-auto module(ID id, const Declarations& decls, Statements stmts, const Meta& meta = {}) {
-    return hilti::Module::create(context(), id, decls, stmts, meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:81:5
-auto module(ID id, const Declarations& decls, const Meta& meta = {}) {
-    return hilti::Module::create(context(), id, decls, meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:93:5
+auto module(const module::UID& uid, const ID& scope = {}, const Meta& meta = {}) {
+    return hilti::Module::create(context(), uid, scope, meta);
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:142:5
+auto module(const module::UID& uid, const ID& scope, const Declarations& decls, Statements stmts,
+            const Meta& meta = {}) {
+    return hilti::Module::create(context(), uid, scope, decls, stmts, meta);
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:133:5
+auto module(const module::UID& uid, const ID& scope, const Declarations& decls, const Meta& meta = {}) {
+    return hilti::Module::create(context(), uid, scope, decls, meta);
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/module.h:146:5
 auto qualifiedType(const UnqualifiedTypePtr& t, bool is_constant, Meta m = Meta()) {
     return hilti::QualifiedType::create(context(), t, is_constant, m);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/type.h:160:5
@@ -434,10 +438,10 @@ auto typeBool(Meta meta = {}) {
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/bool.h:15:5
 auto typeBytes(const Meta& meta = {}) {
     return hilti::type::Bytes::create(context(), meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/bytes.h:48:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/bytes.h:46:5
 auto typeBytesIterator(Meta meta = {}) {
     return hilti::type::bytes::Iterator::create(context(), meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/bytes.h:23:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/bytes.h:21:5
 auto typeDocOnly(std::string description, Meta meta = {}) {
     return hilti::type::DocOnly::create(context(), description, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/doc-only.h:22:5
@@ -508,6 +512,9 @@ auto typeMember(const ID& id, Meta meta = {}) {
 auto typeMember(type::Wildcard _, Meta m = Meta()) {
     return hilti::type::Member::create(context(), _, m);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/member.h:27:5
+auto typeName(ID id, Meta meta = {}) {
+    return hilti::type::Name::create(context(), id, meta);
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/unresolved-id.h:16:5
 auto typeNetwork(Meta meta = {}) {
     return hilti::type::Network::create(context(), meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/network.h:14:5
@@ -553,12 +560,12 @@ auto typeSetIterator(const QualifiedTypePtr& etype, Meta meta = {}) {
 auto typeSetIterator(type::Wildcard _, Meta m = Meta()) {
     return hilti::type::set::Iterator::create(context(), _, m);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/set.h:25:5
-auto typeSignedInteger(int width, Meta m = Meta()) {
+auto typeSignedInteger(int width, const Meta& m = Meta()) {
     return hilti::type::SignedInteger::create(context(), width, m);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:43:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:46:5
 auto typeSignedInteger(type::Wildcard _, Meta m = Meta()) {
     return hilti::type::SignedInteger::create(context(), _, m);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:47:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:48:5
 auto typeStream(const Meta& meta = {}) {
     return hilti::type::Stream::create(context(), meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/stream.h:72:5
@@ -628,15 +635,12 @@ auto typeUnion(type::Wildcard _, const Meta& meta = {}) {
 auto typeUnknown(Meta meta = {}) {
     return hilti::type::Unknown::create(context(), meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/unknown.h:14:5
-auto typeUnresolvedID(ID id, Meta meta = {}) {
-    return hilti::type::Name::create(context(), id, meta);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/unresolved-id.h:16:5
-auto typeUnsignedInteger(int width, Meta m = Meta()) {
+auto typeUnsignedInteger(int width, const Meta& m = Meta()) {
     return hilti::type::UnsignedInteger::create(context(), width, m);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:67:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:68:5
 auto typeUnsignedInteger(type::Wildcard _, Meta m = Meta()) {
     return hilti::type::UnsignedInteger::create(context(), _, m);
-} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:71:5
+} // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/integer.h:70:5
 auto typeValueReference(const QualifiedTypePtr& type, Meta meta = {}) {
     return hilti::type::ValueReference::create(context(), type, meta);
 } // /Users/robin/work/spicy/node-rewrite/hilti/toolchain/include/hilti/ast/types/reference.h:85:5

hilti/toolchain/include/ast/builder/node-factory.h

@@ -13,12 +13,12 @@ namespace hilti::builder {
 
 class NodeFactory {
 public:
-    NodeFactory(std::shared_ptr<ASTContext> context) : _context(std::move(context)) {}
+    NodeFactory(ASTContext* context) : _context(context) {}
 
-    ASTContext* context() const { return _context.get(); }
+    ASTContext* context() const { return _context; }
 
 private:
-    const std::shared_ptr<ASTContext> _context;
+    ASTContext* _context;
 
 public:
 #include <hilti/ast/builder/node-factory.autogen.h>

hilti/toolchain/include/ast/declarations/imported-module.h

@@ -9,6 +9,8 @@
 
 #include <hilti/ast/declaration.h>
 
+#include "ast/module.h"
+
 namespace hilti {
 
 class Unit;
@@ -27,14 +29,13 @@ namespace declaration {
  */
 class ImportedModule : public Declaration {
 public:
-    hilti::rt::filesystem::path parseExtension() const { return _parse_extension; }
-
     const auto& path() const { return _path; }
     const auto& scope() const { return _scope; }
     const auto& searchDirectories() const { return _dirs; }
-    auto unit() const { return _unit.lock(); }
+    const auto& parseExtension() const { return _parse_extension; }
 
-    void setUnit(const std::shared_ptr<Unit>& unit) { _unit = unit; }
+    auto uid() const { return _uid; }
+    void setUID(module::UID uid) { _uid = std::move(uid); }
 
     std::string displayName() const final { return "imported module"; }
 
@@ -44,6 +45,7 @@ class ImportedModule : public Declaration {
             {"ext", _parse_extension.native()},
             {"scope", _scope ? _scope->str() : std::string("<n/a>")},
             {"dirs", util::join(_dirs)},
+            {"uid", _uid ? _uid->str() : std::string("<n/a>")},
         };
         return Declaration::properties() + p;
     }
@@ -79,7 +81,7 @@ class ImportedModule : public Declaration {
             return false;
 
         return Declaration::isEqual(other) && _parse_extension == n->_parse_extension && _path == n->_path &&
-               _scope == n->_scope && _dirs == n->_dirs;
+               _scope == n->_scope && _dirs == n->_dirs && _uid == n->_uid;
     }
 
     HILTI_NODE(ImportedModule)
@@ -90,7 +92,7 @@ class ImportedModule : public Declaration {
     std::optional<ID> _scope;
     std::vector<hilti::rt::filesystem::path> _dirs;
 
-    std::weak_ptr<hilti::Unit> _unit;
+    std::optional<module::UID> _uid;
 };
 
 } // namespace declaration

hilti/toolchain/include/ast/expressions/all.h

@@ -8,14 +8,14 @@
 #include <hilti/ast/expressions/ctor.h>
 #include <hilti/ast/expressions/deferred.h>
 #include <hilti/ast/expressions/grouping.h>
-#include <hilti/ast/expressions/id.h>
 #include <hilti/ast/expressions/keyword.h>
 #include <hilti/ast/expressions/list-comprehension.h>
 #include <hilti/ast/expressions/logical-and.h>
 #include <hilti/ast/expressions/logical-not.h>
 #include <hilti/ast/expressions/logical-or.h>
 #include <hilti/ast/expressions/member.h>
 #include <hilti/ast/expressions/move.h>
+#include <hilti/ast/expressions/name.h>
 #include <hilti/ast/expressions/pending-coerced.h>
 #include <hilti/ast/expressions/resolved-operator.h>
 #include <hilti/ast/expressions/ternary.h>