New AST architecture.

Rebased to current `main`.

.clang-tidy

@@ -55,14 +55,16 @@ Checks: '
   -readability-container-size-empty,
   -readability-convert-member-functions-to-static,
   -readability-else-after-return,
-  -readability-implicit-bool-conversion,
+  -readability-function-cognitive-complexity,
+  -readability-function-size,
   -readability-identifier-length,
+  -readability-implicit-bool-conversion,
   -readability-magic-numbers,
   -readability-make-member-function-const,
   -readability-named-parameter,
   -readability-qualified-auto,
-  -readability-function-cognitive-complexity,
-  -readability-function-size,
+  -readability-simplify-boolean-expr,
+  -readability-use-anyofallof,
   '
 
 HeaderFilterRegex: '(hilti|spicy)/[a-zA-Z]+/include/([a-zA-Z]+/)*[a-zA-Z]+\.h'

.paths

@@ -1,6 +1,7 @@
 hilti/runtime/include                                       hilti/runtime/src
 hilti/toolchain/bin
 hilti/toolchain/include                                     hilti/toolchain/src
+hilti/toolchain/include/ast/detail                          hilti/toolchain/src/ast
 hilti/toolchain/include/compiler/detail                     hilti/toolchain/src/compiler
 hilti/toolchain/include/compiler/detail/codegen/codegen.h   hilti/toolchain/src/compiler/codegen
 hilti/toolchain/include/compiler/detail/visitors.h          hilti/toolchain/src/compiler/visitors

.pre-commit-config.yaml

@@ -12,20 +12,20 @@ repos:
     types: ["sh"]
     stages: ["commit"]
 
-  - id: license
-    name: Check for license headers
-    entry: ./tests/Scripts/license-header.py
-    language: python
-    stages: ["commit"]
-    types: [text]
-    exclude: '^tests|NOTES|README|^(CHANGES|VERSION|scripts/ninja-build-stats|\.gitmodules|\.paths|.*\.(decl|json|rst|t2d))$'
+  # - id: license
+  #   name: Check for license headers
+  #   entry: ./tests/Scripts/license-header.py
+  #   language: python
+  #   stages: ["commit"]
+  #   types: [text]
+  #   exclude: '^tests|NOTES|README|^(CHANGES|VERSION|scripts/ninja-build-stats|\.gitmodules|\.paths|.*\.(decl|json|rst|t2d))$'
 
-  - id: autogen-docs
-    name: Check that generated documentation is up-to-date
-    entry: ./doc/scripts/autogen-docs
-    language: script
-    pass_filenames: false
-    stages: ["commit"]
+  # - id: autogen-docs
+  #   name: Check that generated documentation is up-to-date
+  #   entry: ./doc/scripts/autogen-docs
+  #   language: script
+  #   pass_filenames: false
+  #   stages: ["commit"]
 
   - id: stray-baselines
     name: Check for stray BTest baselines
@@ -58,7 +58,7 @@ repos:
   rev: 0.8.0
   hooks:
   - id: cpplint
-    exclude: '3rdparty/'
+    exclude: '3rdparty/|node-factory.autogen.h'
     args: ["--quiet"]
     stages: ["commit"]
 

cmake/ASTOperators.cmake

@@ -1,32 +1,19 @@
 # Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details.
 
-# TODO: Clean this up. Turn into functions with named parameters.
+### Autogenerate glue code for a set of operator definitions.
+macro (autogen_operators outputs ns srcdir autogen_h autogen_cc)
+    set(_outputs "${autogen_h}/ast-visitor-dispatcher.h" "${autogen_h}/ast-forward.h"
+                 "${autogen_cc}/ast-visitor-dispatcher.cc")
 
-### Autogenerate *.decl file for a set of operator definitions.
-macro (autogen_operators outputs ns srcdir dst_decls dst_impls)
-    set(_output_decls "${dst_decls}")
-    set(_output_impls "${dst_impls}")
-
-    file(GLOB _headers ${CMAKE_CURRENT_SOURCE_DIR}/${srcdir}/*.h)
-
-    add_custom_command(
-        OUTPUT ${_output_decls}
-        COMMAND ${PROJECT_SOURCE_DIR}/scripts/autogen-operators-nodes-decl ${ns}
-                ${CMAKE_CURRENT_SOURCE_DIR}/${srcdir} >${_output_decls}
-        DEPENDS ${PROJECT_SOURCE_DIR}/scripts/autogen-operators-nodes-decl ${_headers}
-        COMMENT "Generating ${_output_decls}")
-
-    set_source_files_properties(${_output_decls} PROPERTIES GENERATED TRUE)
-
-    ##
+    file(GLOB _operators ${CMAKE_CURRENT_SOURCE_DIR}/${srcdir}/*.cc)
 
     add_custom_command(
-        OUTPUT "${AUTOGEN_CC}/operators-implementations.cc"
-        COMMAND ${PROJECT_SOURCE_DIR}/scripts/autogen-operators-implementations ${ns}
-                ${CMAKE_CURRENT_SOURCE_DIR}/${srcdir} >${_output_impls}
-        DEPENDS ${PROJECT_SOURCE_DIR}/scripts/autogen-operators-implementations ${_headers}
-        COMMENT "Generating ${AUTOGEN_CC}/operators-implementations.cc")
-
-    set_source_files_properties(${_output_impls} PROPERTIES GENERATED TRUE)
-    list(APPEND ${outputs} ${_output_impls})
+        OUTPUT ${_outputs}
+        COMMAND ${PROJECT_SOURCE_DIR}/scripts/autogen-operators ${ns} ${autogen_h} ${autogen_cc} --
+                ${_operators}
+        DEPENDS ${PROJECT_SOURCE_DIR}/scripts/autogen-operators ${_operators}
+        COMMENT "Generating operator glue code ...")
+
+    set_source_files_properties(${outputs} PROPERTIES GENERATED TRUE)
+    set(${outputs} ${_outputs})
 endmacro ()

cmake/Util.cmake

@@ -211,7 +211,9 @@ macro (BISON_TARGET_PP Name BisonInput BisonOutput)
     endforeach ()
 
     # Invoke the actual Bison processing.
-    bison_target(${args})
+    bison_target(
+        ${args}
+        COMPILE_FLAGS "--file-prefix-map=${BisonInputPP}=${CMAKE_CURRENT_SOURCE_DIR}/${BisonInput}")
 
     check_cxx_compiler_flag("-Wunused-but-set-variable" have_unused_but_set_variable)
     if (have_unused_but_set_variable)

hilti/lib/hilti.hlt

@@ -2,6 +2,8 @@
 
 module hilti {
 
+%skip-implementation;
+
 public type BitOrder = enum { LSB0, MSB0 } &cxxname="hilti::rt::integer::BitOrder";
 public type ByteOrder = enum { Little, Big, Network, Host } &cxxname="hilti::rt::ByteOrder";
 public type Side = enum { Left, Right, Both } &cxxname="hilti::rt::bytes::Side";
@@ -36,18 +38,23 @@ declare public void profiler_stop(optional<Profiler> p) &cxxname="hilti::rt::pro
 public type Exception = exception &cxxname="hilti::rt::Exception";
 
 # Helper type referring to any C++-side exception.
-public type SystemException = exception &cxxname="std::exception";
+public type SystemException = [exception : Exception] &cxxname="std::exception";
 
 # Base type for all exception generated by the runtime system. Catching
 # this allows to continue after operations triggering runtime errors.
-public type RuntimeError = exception &cxxname="hilti::rt::RuntimeError";
+public type RuntimeError = [exception : Exception] &cxxname="hilti::rt::RuntimeError";
 
 # Base class for exceptions which can be recovered.
-public type RecoverableFailure = exception &cxxname="hilti::rt::RecoverableFailure";
+public type RecoverableFailure = [exception : Exception] &cxxname="hilti::rt::RecoverableFailure";
 
 # Exception raised when attempting to extracting data from a gap.
-public type MissingData = exception &cxxname="hilti::rt::MissingData";
+public type MissingData = [exception : Exception] &cxxname="hilti::rt::MissingData";
 
 # Returns the message associated with an exception.
 declare public string exception_what(SystemException excpt) &cxxname="hilti::rt::exception::what" &have_prototype;
+declare public string exception_what(RecoverableFailure excpt) &cxxname="hilti::rt::exception::what" &have_prototype;
+
+# Returns the location associated with an exception.
+declare public string exception_where(SystemException excpt) &cxxname="hilti::rt::exception::where" &have_prototype;
+declare public string exception_where(RecoverableFailure excpt) &cxxname="hilti::rt::exception::where" &have_prototype;
 }

hilti/runtime/include/exception.h

@@ -261,9 +261,21 @@ void printUncaught(const Exception& e);
 /** Utility function printing out an uncaught exception to an output stream. */
 void printUncaught(const Exception& e, std::ostream& out);
 
-/** Returns the message associated with an exception. This supports any C++-side exception, not just ours. */
+/**
+ * Returns the message associated with an exception. The returned message does
+ * not include the location.
+ */
+inline std::string what(const Exception& e) { return e.description(); }
+
+/**
+ * Returns the message associated with an exception. This is a fallback for
+ * standard exceptions that aren't ours.
+ */
 inline std::string what(const std::exception& e) { return e.what(); }
 
+/** Returns the message associated with an exception. */
+inline std::string where(const Exception& e) { return e.location(); }
+
 } // namespace exception
 
 namespace detail::adl {

hilti/runtime/include/extension-points.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <optional>
 #include <string>
 #include <utility>
 

hilti/runtime/include/fiber.h

@@ -346,7 +346,7 @@ class Resumable {
     const Result& get() const {
         assert(static_cast<bool>(_result));
 
-        if constexpr ( std::is_same<Result, void>::value )
+        if constexpr ( std::is_same_v<Result, void> )
             return {};
         else {
             try {

hilti/runtime/include/types/bytes.h

@@ -234,9 +234,11 @@ class Bytes : protected std::string {
 
     /** Returns an iterator representing the first byte of the instance. */
     const_iterator begin() const { return const_iterator(0U, getControl()); }
+    const_iterator cbegin() const { return const_iterator(0U, getControl()); }
 
     /** Returns an iterator representing the end of the instance. */
     const_iterator end() const { return const_iterator(size(), getControl()); }
+    const_iterator cend() const { return const_iterator(size(), getControl()); }
 
     /** Returns an iterator referring to the given offset. */
     const_iterator at(Offset o) const { return begin() + o; }

hilti/runtime/include/types/reference.h

@@ -132,7 +132,7 @@ class ValueReference {
 
         try {
             if ( auto ptr = std::get<T*>(_ptr) ) {
-                if constexpr ( std::is_base_of<Controllable<T>, T>::value )
+                if constexpr ( std::is_base_of_v<Controllable<T>, T> )
                     return ptr->shared_from_this();
                 else
                     throw IllegalReference("cannot dynamically create reference for type");
@@ -299,7 +299,7 @@ class ValueReference {
      * stays around.
      */
     explicit ValueReference(T* t) : _ptr(t) {
-        static_assert(std::is_base_of<Controllable<T>, T>::value);
+        static_assert(std::is_base_of_v<Controllable<T>, T>);
         assert(t);
     }
 

hilti/runtime/include/types/stream.h

@@ -263,7 +263,7 @@ class Chain : public intrusive_ptr::ManagedObject {
     UnsafeConstIterator unsafeEnd() const;
 
     // Returns a newly allocated chain with the same content.
-    ChainPtr deepCopy() const;
+    ChainPtr copy() const;
 
     /** Appends a new chunk to the end. */
     void append(std::unique_ptr<Chunk> chunk);
@@ -1365,13 +1365,16 @@ class View final {
 
     /** Returns an safe iterator pointint to the beginning of the view. */
     const SafeConstIterator& begin() const { return _begin; }
+    const SafeConstIterator& cbegin() const { return _begin; }
 
     /** Returns a safe iterator representing the end of the instance. */
     SafeConstIterator end() const {
         assert(_begin.chain());
         return _end ? *_end : _begin.chain()->end();
     }
 
+    SafeConstIterator cend() const { return end(); }
+
     /** State for block-wise iteration of a stream instance. */
     struct Block {
         const Byte* start;           /**< Pointer to first byte. */
@@ -1530,7 +1533,7 @@ class Stream {
      * Constructs a stream from another stream instance.
      * @param other instance to create this stream from
      */
-    Stream(const Stream& other) : _chain(other._chain->deepCopy()) {}
+    Stream(const Stream& other) : _chain(other._chain->copy()) {}
 
     /**
      * Constructs a stream from another stream instance.
@@ -1561,7 +1564,7 @@ class Stream {
             return *this;
 
         _chain->invalidate();
-        _chain = other._chain->deepCopy();
+        _chain = other._chain->copy();
         return *this;
     }
 
@@ -1622,9 +1625,11 @@ class Stream {
 
     /** Returns a safe iterator representing the first byte of the instance. */
     SafeConstIterator begin() const { return _chain->begin(); }
+    SafeConstIterator cbegin() const { return begin(); }
 
     /** Returns a safe iterator representing the end of the instance. */
     SafeConstIterator end() const { return _chain->end(); }
+    SafeConstIterator cend() const { return end(); }
 
     /** Returns an unsafe iterator representing the first byte of the instance. */
     UnsafeConstIterator unsafeBegin() const { return _chain->unsafeBegin(); }

hilti/runtime/include/types/union.h

@@ -69,9 +69,9 @@ class Union : public trait::isUnion {
     Union& operator=(Union&&) noexcept = default;
 
     template<typename F>
-    Union(const F& t) : value(t){};
+    Union(const F& t) : value(t) {}
     template<typename F>
-    Union(const F&& t) : value(t){};
+    Union(const F&& t) : value(t) {}
     template<typename F>
     Union& operator=(const F& t) {
         value = t;

hilti/runtime/include/util.h

@@ -364,8 +364,8 @@ constexpr auto transform_result_value(const C&) {
     else if constexpr ( std::is_same_v<C, Set<X>> ) {
         return Set<Y>();
     }
-
-    // No default value defined for type.
+    else
+        return std::vector<Y>(); // fallback
 }
 
 } // namespace detail

hilti/runtime/src/types/reference.cc

@@ -1,6 +1,6 @@
 // Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details.
 
-#include "hilti/rt/types/reference.h"
+#include <hilti/rt/types/reference.h>
 
 namespace hilti::rt::reference::detail {
 void throw_null() { throw NullReference("attempt to access null reference"); }

hilti/runtime/src/types/stream.cc

@@ -91,7 +91,7 @@ void Chain::trim(const Offset& offset) {
     _head_offset = offset;
 }
 
-ChainPtr Chain::deepCopy() const {
+ChainPtr Chain::copy() const {
     _ensureValid();
 
     auto nchain = make_intrusive<Chain>();