From 94ae0e15b0b5a4716d9a89b990aab783b69ebb29 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 8 Apr 2018 21:30:56 -0400 Subject: [PATCH] [tools:doc] Improve Doc::Generator to more consistent structure + more data. `Doc::Generator` (renamed from `DocGenerator`) is the command line tool users can run to auto-generate documentation from their codebase using the `--docs` flag. Providing a directory automatically scans that directory's content for Myst source files, and merges their contents together into one large Documentation structure. The generator also creates a more consistent and complete structure, using the new `*Doc` classes to represent each type of object (constant, method, clause, module, and type) more accurately and with relevant information. The resulting JSON is a complete picture of the codebase, for objects with and without DocComments on them. This generator still needs testing, but the structure is likely complete for the time being. --- spec/syntax/parser_spec.cr | 18 ++ src/myst/cli.cr | 2 +- .../assertions/duplicate_param_names.cr | 6 +- src/myst/syntax/doc.cr | 14 -- src/myst/syntax/lexer.cr | 7 - src/myst/syntax/parser.cr | 63 +++--- src/myst/syntax/token.cr | 1 - src/myst/tools/doc.cr | 204 ++++++++++-------- src/myst/tools/doc/clause_doc.cr | 24 +++ src/myst/tools/doc/const_doc.cr | 23 ++ src/myst/tools/doc/kind.cr | 14 ++ src/myst/tools/doc/method_doc.cr | 23 ++ src/myst/tools/doc/module_doc.cr | 29 +++ src/myst/tools/doc/type_doc.cr | 33 +++ src/myst/tools/printer.cr | 33 ++- src/myst/vm.cr | 5 - 16 files changed, 339 insertions(+), 160 deletions(-) delete mode 100644 src/myst/syntax/doc.cr create mode 100644 src/myst/tools/doc/clause_doc.cr create mode 100644 src/myst/tools/doc/const_doc.cr create mode 100644 src/myst/tools/doc/kind.cr create mode 100644 src/myst/tools/doc/method_doc.cr create mode 100644 src/myst/tools/doc/module_doc.cr create mode 100644 src/myst/tools/doc/type_doc.cr diff --git a/spec/syntax/parser_spec.cr b/spec/syntax/parser_spec.cr index 65143f6..cb7b19b 100644 --- a/spec/syntax/parser_spec.cr +++ b/spec/syntax/parser_spec.cr @@ -4052,6 +4052,24 @@ describe "Parser" do nil ), doc("foo") + # Docs can be attached to any kind of node, but generally only apply to modules, types, methods, and constants. + it_parses %q( + #doc foo + defmodule Foo; end + ), doc("foo", target: ModuleDef.new("Foo")) + it_parses %q( + #doc foo + deftype Foo; end + ), doc("foo", target: TypeDef.new("Foo")) + it_parses %q( + #doc foo + def foo; end + ), doc("foo", target: Def.new("foo")) + it_parses %q( + #doc foo + FOO = nil + ), doc("foo", target: SimpleAssign.new(c("FOO"), l(nil))) + # Doc comments do not affect parsing outside of their content. A doc comment placed # immediately above or below another expression should not affect that expression. it_parses %q( diff --git a/src/myst/cli.cr b/src/myst/cli.cr index 3552349..2c1f231 100644 --- a/src/myst/cli.cr +++ b/src/myst/cli.cr @@ -58,7 +58,7 @@ module Myst if generate_docs - DocGenerator.auto_document(docs_directory) + Doc::Generator.auto_document(docs_directory) exit end diff --git a/src/myst/semantic/assertions/duplicate_param_names.cr b/src/myst/semantic/assertions/duplicate_param_names.cr index 1a51e90..a1895c1 100644 --- a/src/myst/semantic/assertions/duplicate_param_names.cr +++ b/src/myst/semantic/assertions/duplicate_param_names.cr @@ -11,7 +11,7 @@ module Myst names_in_params.each do |name, nodes| if nodes.size > 1 - original_str = Myst::Printer.new(String::Builder.new).print(def_node).to_s + original_str = Myst::Printer.print(def_node) resolved_str = create_resolved_str(nodes) fail! def_node.location.not_nil!, <<-FAIL_MESSAGE @@ -26,7 +26,7 @@ module Myst end private def create_resolved_str(nodes : Array(Node)) - printer = Myst::Printer.new(String::Builder.new) + printer = Myst::Printer.new nodes[1..-1].each do |node| new_node = node.dup @@ -44,7 +44,7 @@ module Myst printer.replace(node, new_node) end - printer.print(def_node).to_s + printer.print(def_node) end private def visit_param(param : Param) diff --git a/src/myst/syntax/doc.cr b/src/myst/syntax/doc.cr deleted file mode 100644 index da03f74..0000000 --- a/src/myst/syntax/doc.cr +++ /dev/null @@ -1,14 +0,0 @@ -module Myst - class Doc - property content : String - - def initialize(@content : String) - end - - def_equals_and_hash content - - def to_json(json : JSON::Builder) - json.scalar(content) - end - end -end diff --git a/src/myst/syntax/lexer.cr b/src/myst/syntax/lexer.cr index c290fe0..fe242b3 100644 --- a/src/myst/syntax/lexer.cr +++ b/src/myst/syntax/lexer.cr @@ -285,7 +285,6 @@ module Myst end when '\n' @current_token.type = Token::Type::NEWLINE - reset_line_based_properties! read_char when '#' skip_char @@ -601,11 +600,5 @@ module Myst @current_token.value = @reader.buffer_value end - - - # Reset any contextual properties that only apply for a single line. - private def reset_line_based_properties! - @hash_as_token = false - end end end diff --git a/src/myst/syntax/parser.cr b/src/myst/syntax/parser.cr index 64667df..df05256 100644 --- a/src/myst/syntax/parser.cr +++ b/src/myst/syntax/parser.cr @@ -120,39 +120,36 @@ module Myst end def parse_expression - expr_node = - case current_token.type - when Token::Type::DEF, Token::Type::DEFSTATIC - parse_def - when Token::Type::DEFMODULE - parse_module_def - when Token::Type::DEFTYPE - parse_type_def - when Token::Type::FN - parse_anonymous_function - when Token::Type::MATCH - parse_match - when Token::Type::INCLUDE - parse_include - when Token::Type::EXTEND - parse_extend - when Token::Type::REQUIRE - parse_require - when Token::Type::WHEN, Token::Type::UNLESS - parse_conditional - when Token::Type::WHILE, Token::Type::UNTIL - parse_loop - when Token::Type::AMPERSAND - parse_function_capture - when Token::Type::MAGIC_FILE, Token::Type::MAGIC_LINE, Token::Type::MAGIC_DIR - parse_magic_constant - when Token::Type::DOC_START - parse_doc_comment - else - parse_logical_or - end - - expr_node + case current_token.type + when Token::Type::DEF, Token::Type::DEFSTATIC + parse_def + when Token::Type::DEFMODULE + parse_module_def + when Token::Type::DEFTYPE + parse_type_def + when Token::Type::FN + parse_anonymous_function + when Token::Type::MATCH + parse_match + when Token::Type::INCLUDE + parse_include + when Token::Type::EXTEND + parse_extend + when Token::Type::REQUIRE + parse_require + when Token::Type::WHEN, Token::Type::UNLESS + parse_conditional + when Token::Type::WHILE, Token::Type::UNTIL + parse_loop + when Token::Type::AMPERSAND + parse_function_capture + when Token::Type::MAGIC_FILE, Token::Type::MAGIC_LINE, Token::Type::MAGIC_DIR + parse_magic_constant + when Token::Type::DOC_START + parse_doc_comment + else + parse_logical_or + end end def parse_def diff --git a/src/myst/syntax/token.cr b/src/myst/syntax/token.cr index 247400e..dc6dac5 100644 --- a/src/myst/syntax/token.cr +++ b/src/myst/syntax/token.cr @@ -92,7 +92,6 @@ module Myst POINT # . COLON # : SEMI # ; - HASH # # DOC_START # #doc DOC_CONTENT # #| ... diff --git a/src/myst/tools/doc.cr b/src/myst/tools/doc.cr index 7654696..1d893f4 100644 --- a/src/myst/tools/doc.cr +++ b/src/myst/tools/doc.cr @@ -1,120 +1,140 @@ require "json" +require "./doc/*" + module Myst - # An AST visitor for parsing doc comments from Myst source code and emitting - # the content in a JSON structure. - # - # Currently only operates on the given source, does not follow `require`s or - # other imports. - class DocGenerator - property printer : Printer - - enum DocType - # Types defined using `deftype` - TYPE - # Modules defined using `defmodule` - MODULE - # Any method defined using `def` - METHOD - # Any method defined using `defstatic`. Only valid within a type. - STATIC_METHOD - end + module Doc + alias DocContext = ModuleDoc | TypeDoc + + # An AST visitor for parsing doc comments from Myst source code and emitting + # the content in a JSON structure. + # + # Currently only operates on the given source, does not follow `require`s or + # other imports. + class Generator + # Automatically scan everything in the current directory to find Myst files + # that can be documented. + def self.auto_document(directory = Dir.current) + generator = self.new + Dir[directory, directory+"/*"].each do |entry| + # Only consider files that end with the `.mt` extension + if entry.ends_with?(".mt") + file_ast = Parser.for_file(entry).parse + generator.document(file_ast) + end + end - alias DocContext = Hash(String, Entry) + puts generator.json + end - struct Entry - property name : String - property doc : Doc? - property type : DocType - property children : DocContext - JSON.mapping( - name: String, - doc: { type: Doc?, emit_null: true }, - type: DocType, - children: DocContext - ) + @current_context : DocContext - def initialize(@name : String, @doc : Doc?, @type : DocType, @children = DocContext.new) + def initialize + @docs = ModuleDoc.new("Root", "Root", nil) + @current_context = @docs end - end - - # Automatically scan everything in the current directory to find Myst files - # that can be documented. - def self.auto_document(directory = Dir.current) - generator = self.new - Dir[directory, directory+"/*", directory+"/**/*"].each do |entry| - # Only consider files that end with the `.mt` extension - if entry.ends_with?(".mt") - file_ast = Parser.for_file(entry).parse - generator.document(file_ast) - end + def json + @docs.to_json end - puts generator.json - end + def document(node : Node) + visit(node) + end - def initialize - @printer = Printer.new - @docs = DocContext.new - @current_doc = @docs - end - - - def document(node : Node) - visit(node) - end - - # Return a JSON representation of the current documentation structure. - def json - @docs.to_json - end - - # Automatically recurse through all non-special nodes - def visit(node : Node) - node.accept_children(self) - end - def visit(node : ModuleDef) - entry = Entry.new(name: node.name, doc: nil, type: DocType::MODULE) - entry.children = child_context do + # Automatically recurse through all non-special nodes + def visit(node : Node, doc : String?=nil) node.accept_children(self) end - @current_doc[node.name] = entry - end + def visit(node : DocComment) + visit(node.target, node.content) + end - def visit(node : TypeDef) - entry = Entry.new(name: node.name, doc: nil, type: DocType::TYPE) - entry.children = child_context do - node.accept_children(self) + def visit(node : ModuleDef, doc : String?=nil) + @current_context.submodules[node.name] ||= ModuleDoc.new(node.name, make_full_path(node.name), doc) + module_doc = @current_context.submodules[node.name] + with_context(module_doc) do + node.accept_children(self) + end end - @current_doc[node.name] = entry - end + def visit(node : TypeDef, doc : String?=nil) + @current_context.subtypes[node.name] ||= TypeDoc.new(node.name, make_full_path(node.name), doc) + type_doc = @current_context.subtypes[node.name] - def visit(node : Def) - name = String.build{ |s| printer.print(node, s) } - entry = Entry.new( - name: name, - doc: nil, - type: node.static? ? DocType::STATIC_METHOD : DocType::METHOD - ) + with_context(type_doc) do + node.accept_children(self) + end + end - @current_doc[name] = entry - end + def visit(node : Def, doc : String?=nil) + container = + case context = @current_context + when ModuleDoc + context.methods + when TypeDoc + case + when node.static? + context.static_methods + when node.name == "initialize" + context.initializers + else + context.instance_methods + end + else + # This should never be reached, since the case covers all types in + # the union of `DocContext`. + raise "Unhandled DocContext type #{typeof(context)}." + end + + # Make sure that a method entry for this clause exists. + container[node.name] ||= MethodDoc.new(node.name, make_full_path(node.name), nil) + method_doc = container[node.name] + + clause_doc = ClauseDoc.new( + head: Printer.print(node), + arity: node.params.size, + parameters: node.params.map{ |p| Printer.print(p) }, + splat_index: node.splat_index?, + block_parameter: node.block_param?.try(&.name), + doc: doc + ) + method_doc.clauses << clause_doc + end + # Only constants that are assigned with SimpleAssigns are documentable. + # The entire SimpleAssign is visited so that the value being assigned can + # also be captured and added to the documentation. + def visit(node : SimpleAssign, doc : String?=nil) + target = node.target + # If the target isn't a Const, it can just be ignored + return unless target.is_a?(Const) + + const_doc = ConstDoc.new( + target.name, + make_full_path(target.name), + Printer.print(node.value), + doc + ) + @current_context.constants[target.name] = const_doc + end - private def child_context(&block : ->) - parent_context = @current_doc - child_context = DocContext.new - @current_doc = child_context - yield - @current_doc = parent_context - child_context + private def with_context(context : DocContext) + parent_context = @current_context + @current_context = context + yield + @current_context = parent_context + context + end + + private def make_full_path(basename : String) : String + @current_context.full_name + "." + basename + end end end end diff --git a/src/myst/tools/doc/clause_doc.cr b/src/myst/tools/doc/clause_doc.cr new file mode 100644 index 0000000..9ca7fc7 --- /dev/null +++ b/src/myst/tools/doc/clause_doc.cr @@ -0,0 +1,24 @@ +module Myst + module Doc + struct ClauseDoc + property head : String + property arity : Int32 + property parameters : Array(String) + property splat_index : Int32? + property block_parameter : String? + property doc : String? = nil + + JSON.mapping( + head: String, + arity: Int32, + parameters: Array(String), + splat_index: {type: Int32?, emit_null: true}, + block_parameter: {type: String?, emit_null: true}, + doc: {type: String?, emit_null: true} + ) + + def initialize(@head : String, @arity : Int32, @parameters : Array(String), @splat_index : Int32?, @block_parameter : String?, @doc : String?) + end + end + end +end diff --git a/src/myst/tools/doc/const_doc.cr b/src/myst/tools/doc/const_doc.cr new file mode 100644 index 0000000..f1350dd --- /dev/null +++ b/src/myst/tools/doc/const_doc.cr @@ -0,0 +1,23 @@ +module Myst + module Doc + class ConstDoc + property kind = Kind::CONSTANT + property name : String + property full_name : String + property value : String + property doc : String? = nil + + JSON.mapping( + kind: Kind, + name: String, + full_name: String, + value: String, + doc: {type: String?, emit_null: true} + ) + + + def initialize(@name : String, @full_name : String, @value : String, @doc : String?) + end + end + end +end diff --git a/src/myst/tools/doc/kind.cr b/src/myst/tools/doc/kind.cr new file mode 100644 index 0000000..4c8a2dc --- /dev/null +++ b/src/myst/tools/doc/kind.cr @@ -0,0 +1,14 @@ +module Myst + module Doc + enum Kind + CONSTANT + METHOD + MODULE + TYPE + + def to_json(builder) + builder.string(self) + end + end + end +end diff --git a/src/myst/tools/doc/method_doc.cr b/src/myst/tools/doc/method_doc.cr new file mode 100644 index 0000000..01d980e --- /dev/null +++ b/src/myst/tools/doc/method_doc.cr @@ -0,0 +1,23 @@ +module Myst + module Doc + class MethodDoc + property kind = Kind::METHOD + property name : String + property full_name : String + property clauses = [] of ClauseDoc + property doc : String? = nil + + JSON.mapping( + kind: Kind, + name: String, + full_name: String, + clauses: Array(ClauseDoc), + doc: {type: String?, emit_null: true} + ) + + + def initialize(@name : String, @full_name : String, @doc : String?) + end + end + end +end diff --git a/src/myst/tools/doc/module_doc.cr b/src/myst/tools/doc/module_doc.cr new file mode 100644 index 0000000..cde349a --- /dev/null +++ b/src/myst/tools/doc/module_doc.cr @@ -0,0 +1,29 @@ +module Myst + module Doc + class ModuleDoc + property kind = Kind::MODULE + property name : String + property full_name : String + property doc : String? = nil + property constants = {} of String => ConstDoc + property methods = {} of String => MethodDoc + property submodules = {} of String => ModuleDoc + property subtypes = {} of String => TypeDoc + + JSON.mapping( + kind: Kind, + name: String, + full_name: String, + doc: {type: String?, emit_null: true}, + constants: Hash(String, ConstDoc), + methods: Hash(String, MethodDoc), + submodules: Hash(String, ModuleDoc), + subtypes: Hash(String, TypeDoc) + ) + + + def initialize(@name : String, @full_name : String, @doc : String?) + end + end + end +end diff --git a/src/myst/tools/doc/type_doc.cr b/src/myst/tools/doc/type_doc.cr new file mode 100644 index 0000000..0f6efe1 --- /dev/null +++ b/src/myst/tools/doc/type_doc.cr @@ -0,0 +1,33 @@ +module Myst + module Doc + class TypeDoc + property kind = Kind::TYPE + property name : String + property full_name : String + property doc : String? = nil + property constants = {} of String => ConstDoc + property instance_methods = {} of String => MethodDoc + property static_methods = {} of String => MethodDoc + property initializers = {} of String => MethodDoc + property submodules = {} of String => ModuleDoc + property subtypes = {} of String => TypeDoc + + JSON.mapping( + kind: Kind, + name: String, + full_name: String, + doc: {type: String?, emit_null: true}, + constants: Hash(String, ConstDoc), + instance_methods: Hash(String, MethodDoc), + static_methods: Hash(String, MethodDoc), + initializers: Hash(String, MethodDoc), + submodules: Hash(String, ModuleDoc), + subtypes: Hash(String, TypeDoc) + ) + + + def initialize(@name : String, @full_name : String, @doc : String?) + end + end + end +end diff --git a/src/myst/tools/printer.cr b/src/myst/tools/printer.cr index 55ad825..593d55e 100644 --- a/src/myst/tools/printer.cr +++ b/src/myst/tools/printer.cr @@ -1,13 +1,23 @@ module Myst class Printer - property output : IO + def self.print(node, io : IO) + Printer.new.print(node, io) + end + + def self.print(node) + String.build do |str| + print(node, str) + end + end + # `replacements` is a Hash containing mappings from Node instances # to other Nodes that should be used in their place when recursing through # a program tree. This is useful for performing code rewrites # programmatically without having to modify the original program. property replacements : Hash(UInt64, Node) - def initialize(@output : IO=STDOUT) + + def initialize @replacements = {} of UInt64 => Node end @@ -15,10 +25,14 @@ module Myst replacements[node.object_id] = new_node end - def print(node, io : IO = @output) + def print(node, io : IO) visit(node, io) end + def print(node) + String.build{ |str| print(node, str) } + end + macro make_visitor(node_type) def visit(node : {{node_type}}, io : IO) @@ -249,6 +263,18 @@ module Myst end + make_visitor ModuleDef do + io << "defmodule" + io << " " + io << node.name + end + + make_visitor TypeDef do + io << "deftype" + io << " " + io << node.name + end + make_visitor Def do io << (node.static? ? "defstatic" : "def") io << " " @@ -272,7 +298,6 @@ module Myst end - # Catch all for unimplemented nodes make_visitor Node do STDERR.puts "Attempting to print unknown node type: #{node.class.name}" diff --git a/src/myst/vm.cr b/src/myst/vm.cr index 3c1a44d..8ffe912 100644 --- a/src/myst/vm.cr +++ b/src/myst/vm.cr @@ -57,11 +57,6 @@ module Myst @program.not_nil!.accept(visitor) end - def generate_docs(io : IO = STDOUT) - visitor = DocGenerator.new - docs = visitor.document(@program.not_nil!) - end - # Tries to run the provided string as a myst program def eval(program : String) run(Parser.for_content(program).parse)