Document Symbols support (#652)

* Document Symbols support

Added document symbols, which supports the following symbols:

  * Modules
  * Functions, both private and public
  * Typespecs
  * Module Attributes
  * ExUnit describe / setup / tests

Fixes #382

* Added support for block ranges and detail ranges

For document symbols, we need to provide support for block ranges for
things like modules, functions and tests, so that the editor can
understand if it's inside the given symbol.
The LSP also would like to have selection ranges, which are more
specific, and would, say highlight the function definition.

* Upgraded sourceror

Sourceror had a bug calculating end lines, which was causing responses
not to be emitted, but only when unicode was present.

It was emitting the ending several characters beyond where the `end`
keyword was, and this would fail during conversion as being out of bounds.

apps/protocol/lib/generated/lexical/protocol/types/document/symbol.ex

@@ -0,0 +1,15 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.Document.Symbol do
+  alias Lexical.Proto
+  alias Lexical.Protocol.Types
+  use Proto
+
+  deftype children: optional(list_of(Types.Document.Symbol)),
+          deprecated: optional(boolean()),
+          detail: optional(string()),
+          kind: Types.Symbol.Kind,
+          name: string(),
+          range: Types.Range,
+          selection_range: Types.Range,
+          tags: optional(list_of(Types.Symbol.Tag))
+end

apps/protocol/lib/generated/lexical/protocol/types/document/symbol/params.ex

@@ -0,0 +1,10 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.Document.Symbol.Params do
+  alias Lexical.Proto
+  alias Lexical.Protocol.Types
+  use Proto
+
+  deftype partial_result_token: optional(Types.Progress.Token),
+          text_document: Types.TextDocument.Identifier,
+          work_done_token: optional(Types.Progress.Token)
+end

apps/protocol/lib/lexical/protocol/requests.ex

@@ -76,6 +76,12 @@ defmodule Lexical.Protocol.Requests do
     defrequest "workspace/executeCommand", Types.ExecuteCommand.Params
   end
 
+  defmodule DocumentSymbols do
+    use Proto
+
+    defrequest "textDocument/documentSymbol", Types.Document.Symbol.Params
+  end
+
   # Server -> Client requests
 
   defmodule RegisterCapability do

apps/protocol/lib/lexical/protocol/responses.ex

@@ -50,6 +50,12 @@ defmodule Lexical.Protocol.Responses do
     defresponse optional(list_of(one_of([list_of(Types.Completion.Item), Types.Completion.List])))
   end
 
+  defmodule DocumentSymbols do
+    use Proto
+
+    defresponse optional(list_of(Types.Document.Symbol))
+  end
+
   defmodule Shutdown do
     use Proto
     # yeah, this is odd... it has no params

apps/remote_control/lib/lexical/remote_control/api.ex

@@ -130,4 +130,8 @@ defmodule Lexical.RemoteControl.Api do
   def struct_definitions(%Project{} = project) do
     RemoteControl.call(project, CodeIntelligence.Structs, :for_project, [])
   end
+
+  def document_symbols(%Project{} = project, %Document{} = document) do
+    RemoteControl.call(project, CodeIntelligence.Symbols, :for_document, [document])
+  end
 end

apps/remote_control/lib/lexical/remote_control/code_intelligence/symbols.ex

@@ -0,0 +1,70 @@
+defmodule Lexical.RemoteControl.CodeIntelligence.Symbols do
+  alias Lexical.Document
+  alias Lexical.RemoteControl.CodeIntelligence.Symbols
+  alias Lexical.RemoteControl.Search.Indexer
+  alias Lexical.RemoteControl.Search.Indexer.Entry
+  alias Lexical.RemoteControl.Search.Indexer.Extractors
+
+  @block_types [
+    :ex_unit_describe,
+    :ex_unit_setup,
+    :ex_unit_setup_all,
+    :ex_unit_test,
+    :module,
+    :private_function,
+    :public_function
+  ]
+
+  @symbol_extractors [
+    Extractors.FunctionDefinition,
+    Extractors.Module,
+    Extractors.ModuleAttribute,
+    Extractors.StructDefinition,
+    Extractors.ExUnit
+  ]
+
+  def for_document(%Document{} = document) do
+    {:ok, entries} = Indexer.Source.index_document(document, @symbol_extractors)
+
+    definitions = Enum.filter(entries, &(&1.subtype == :definition))
+    to_symbols(document, definitions)
+  end
+
+  defp to_symbols(%Document{} = document, entries) do
+    entries_by_block_id = Enum.group_by(entries, & &1.block_id)
+    rebuild_structure(entries_by_block_id, document, :root)
+  end
+
+  defp rebuild_structure(entries_by_block_id, %Document{} = document, block_id) do
+    block_entries = Map.get(entries_by_block_id, block_id, [])
+
+    Enum.flat_map(block_entries, fn
+      %Entry{type: type, subtype: :definition} = entry when type in @block_types ->
+        result =
+          if Map.has_key?(entries_by_block_id, entry.id) do
+            children =
+              entries_by_block_id
+              |> rebuild_structure(document, entry.id)
+              |> Enum.sort_by(fn %Symbols.Document{} = symbol ->
+                start = symbol.range.start
+                {start.line, start.character}
+              end)
+
+            Symbols.Document.from(document, entry, children)
+          else
+            Symbols.Document.from(document, entry)
+          end
+
+        case result do
+          {:ok, symbol} -> [symbol]
+          _ -> []
+        end
+
+      %Entry{} = entry ->
+        case Symbols.Document.from(document, entry) do
+          {:ok, symbol} -> [symbol]
+          _ -> []
+        end
+    end)
+  end
+end

apps/remote_control/lib/lexical/remote_control/code_intelligence/symbols/document.ex

@@ -0,0 +1,89 @@
+defmodule Lexical.RemoteControl.CodeIntelligence.Symbols.Document do
+  alias Lexical.Document
+  alias Lexical.Formats
+  alias Lexical.RemoteControl.Search.Indexer.Entry
+
+  defstruct [:name, :type, :range, :detail_range, :detail, children: []]
+
+  def from(%Document{} = document, %Entry{} = entry, children \\ []) do
+    case name_and_type(entry.type, entry, document) do
+      {name, type} ->
+        range = entry.block_range || entry.range
+
+        {:ok,
+         %__MODULE__{
+           name: name,
+           type: type,
+           range: range,
+           detail_range: entry.range,
+           children: children
+         }}
+
+      _ ->
+        :error
+    end
+  end
+
+  @def_regex ~r/def\w*\s+/
+  @do_regex ~r/\s*do\s*$/
+
+  defp name_and_type(function, %Entry{} = entry, %Document{} = document)
+       when function in [:public_function, :private_function] do
+    fragment = Document.fragment(document, entry.range.start, entry.range.end)
+
+    name =
+      fragment
+      |> String.replace(@def_regex, "")
+      |> String.replace(@do_regex, "")
+
+    {name, function}
+  end
+
+  @ignored_attributes ~w[spec doc moduledoc derive impl tag]
+  @type_name_regex ~r/@type\s+[^\s]+/
+
+  defp name_and_type(:module_attribute, %Entry{} = entry, document) do
+    case String.split(entry.subject, "@") do
+      [_, name] when name in @ignored_attributes ->
+        nil
+
+      [_, "type"] ->
+        type_text = Document.fragment(document, entry.range.start, entry.range.end)
+
+        name =
+          case Regex.scan(@type_name_regex, type_text) do
+            [[match]] -> match
+            _ -> "@type ??"
+          end
+
+        {name, :type}
+
+      [_, name] ->
+        {"@#{name}", :module_attribute}
+    end
+  end
+
+  defp name_and_type(ex_unit, %Entry{} = entry, document)
+       when ex_unit in [:ex_unit_describe, :ex_unit_setup, :ex_unit_test] do
+    name =
+      document
+      |> Document.fragment(entry.range.start, entry.range.end)
+      |> String.trim()
+      |> String.replace(@do_regex, "")
+
+    {name, ex_unit}
+  end
+
+  defp name_and_type(:struct, %Entry{} = entry, _document) do
+    module_name = Formats.module(entry.subject)
+    {"%#{module_name}{}", :struct}
+  end
+
+  defp name_and_type(type, %Entry{subject: name}, _document) when is_atom(name) do
+    {Formats.module(name), type}
+  end
+
+  defp name_and_type(type, %Entry{} = entry, _document) do
+    {to_string(entry.subject), type}
+  end
+end

apps/remote_control/lib/lexical/remote_control/search/indexer/entry.ex

@@ -10,6 +10,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
     :application,
     :id,
     :block_id,
+    :block_range,
     :path,
     :range,
     :subject,
@@ -21,6 +22,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
           application: module(),
           subject: subject(),
           block_id: block_id(),
+          block_range: Lexical.Document.Range.t() | nil,
           path: Path.t(),
           range: Lexical.Document.Range.t(),
           subtype: entry_subtype(),
@@ -55,16 +57,27 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
     new(path, Identifier.next_global!(), block.id, subject, type, :definition, range, application)
   end
 
-  def block_definition(path, %Block{} = block, subject, type, range, application) do
-    definition(
-      path,
-      block.id,
-      block.parent_id,
-      subject,
-      type,
-      range,
-      application
-    )
+  def block_definition(
+        path,
+        %Block{} = block,
+        subject,
+        type,
+        block_range,
+        detail_range,
+        application
+      ) do
+    definition =
+      definition(
+        path,
+        block.id,
+        block.parent_id,
+        subject,
+        type,
+        detail_range,
+        application
+      )
+
+    %__MODULE__{definition | block_range: block_range}
   end
 
   defp definition(path, id, block_id, subject, type, range, application) do