[tools:doc] Improve Doc::Generator to more consistent structure + mor…

…e data.

`Doc::Generator` (renamed from `DocGenerator`) is the command line tool users can run to auto-generate documentation from their codebase using the `--docs` flag. Providing a directory automatically scans that directory's content for Myst source files, and merges their contents together into one large Documentation structure.

The generator also creates a more consistent and complete structure, using the new `*Doc` classes to represent each type of object (constant, method, clause, module, and type) more accurately and with relevant information. The resulting JSON is a complete picture of the codebase, for objects with and without DocComments on them.

This generator still needs testing, but the structure is likely complete for the time being.

spec/syntax/parser_spec.cr

@@ -4052,6 +4052,24 @@ describe "Parser" do
     nil
   ),                          doc("foo")
 
+  # Docs can be attached to any kind of node, but generally only apply to modules, types, methods, and constants.
+  it_parses %q(
+    #doc foo
+    defmodule Foo; end
+  ),                          doc("foo", target: ModuleDef.new("Foo"))
+  it_parses %q(
+    #doc foo
+    deftype Foo; end
+  ),                          doc("foo", target: TypeDef.new("Foo"))
+  it_parses %q(
+    #doc foo
+    def foo; end
+  ),                          doc("foo", target: Def.new("foo"))
+  it_parses %q(
+    #doc foo
+    FOO = nil
+  ),                          doc("foo", target: SimpleAssign.new(c("FOO"), l(nil)))
+
   # Doc comments do not affect parsing outside of their content. A doc comment placed
   # immediately above or below another expression should not affect that expression.
   it_parses %q(

src/myst/cli.cr

@@ -58,7 +58,7 @@ module Myst
 
 
       if generate_docs
-        DocGenerator.auto_document(docs_directory)
+        Doc::Generator.auto_document(docs_directory)
         exit
       end
 

src/myst/semantic/assertions/duplicate_param_names.cr

@@ -11,7 +11,7 @@ module Myst
 
         names_in_params.each do |name, nodes|
           if nodes.size > 1
-            original_str      = Myst::Printer.new(String::Builder.new).print(def_node).to_s
+            original_str  = Myst::Printer.print(def_node)
             resolved_str  = create_resolved_str(nodes)
 
             fail! def_node.location.not_nil!, <<-FAIL_MESSAGE
@@ -26,7 +26,7 @@ module Myst
       end
 
       private def create_resolved_str(nodes : Array(Node))
-        printer = Myst::Printer.new(String::Builder.new)
+        printer = Myst::Printer.new
         nodes[1..-1].each do |node|
           new_node = node.dup
 
@@ -44,7 +44,7 @@ module Myst
           printer.replace(node, new_node)
         end
 
-        printer.print(def_node).to_s
+        printer.print(def_node)
       end
 
       private def visit_param(param : Param)

src/myst/syntax/lexer.cr

@@ -285,7 +285,6 @@ module Myst
         end
       when '\n'
         @current_token.type = Token::Type::NEWLINE
-        reset_line_based_properties!
         read_char
       when '#'
         skip_char
@@ -601,11 +600,5 @@ module Myst
 
       @current_token.value = @reader.buffer_value
     end
-
-
-    # Reset any contextual properties that only apply for a single line.
-    private def reset_line_based_properties!
-      @hash_as_token = false
-    end
   end
 end

src/myst/syntax/parser.cr

@@ -120,39 +120,36 @@ module Myst
     end
 
     def parse_expression
-      expr_node =
-        case current_token.type
-        when Token::Type::DEF, Token::Type::DEFSTATIC
-          parse_def
-        when Token::Type::DEFMODULE
-          parse_module_def
-        when Token::Type::DEFTYPE
-          parse_type_def
-        when Token::Type::FN
-          parse_anonymous_function
-        when Token::Type::MATCH
-          parse_match
-        when Token::Type::INCLUDE
-          parse_include
-        when Token::Type::EXTEND
-          parse_extend
-        when Token::Type::REQUIRE
-          parse_require
-        when Token::Type::WHEN, Token::Type::UNLESS
-          parse_conditional
-        when Token::Type::WHILE, Token::Type::UNTIL
-          parse_loop
-        when Token::Type::AMPERSAND
-          parse_function_capture
-        when Token::Type::MAGIC_FILE, Token::Type::MAGIC_LINE, Token::Type::MAGIC_DIR
-          parse_magic_constant
-        when Token::Type::DOC_START
-          parse_doc_comment
-        else
-          parse_logical_or
-        end
-
-      expr_node
+      case current_token.type
+      when Token::Type::DEF, Token::Type::DEFSTATIC
+        parse_def
+      when Token::Type::DEFMODULE
+        parse_module_def
+      when Token::Type::DEFTYPE
+        parse_type_def
+      when Token::Type::FN
+        parse_anonymous_function
+      when Token::Type::MATCH
+        parse_match
+      when Token::Type::INCLUDE
+        parse_include
+      when Token::Type::EXTEND
+        parse_extend
+      when Token::Type::REQUIRE
+        parse_require
+      when Token::Type::WHEN, Token::Type::UNLESS
+        parse_conditional
+      when Token::Type::WHILE, Token::Type::UNTIL
+        parse_loop
+      when Token::Type::AMPERSAND
+        parse_function_capture
+      when Token::Type::MAGIC_FILE, Token::Type::MAGIC_LINE, Token::Type::MAGIC_DIR
+        parse_magic_constant
+      when Token::Type::DOC_START
+        parse_doc_comment
+      else
+        parse_logical_or
+      end
     end
 
     def parse_def

src/myst/syntax/token.cr

@@ -92,7 +92,6 @@ module Myst
       POINT         # .
       COLON         # :
       SEMI          # ;
-      HASH          # #
 
       DOC_START     # #doc
       DOC_CONTENT   # #| ...