Merge pull request #69 from Shopify/speed

Speed up

Gemfile.lock

@@ -2,7 +2,7 @@ PATH
   remote: .
   specs:
     smart_todo (1.6.0)
-      rexml
+      prism
 
 GEM
   remote: https://rubygems.org/
@@ -19,6 +19,7 @@ GEM
     parser (3.2.2.3)
       ast (~> 2.4.1)
       racc
+    prism (0.17.0)
     public_suffix (4.0.6)
     racc (1.7.0)
     rainbow (3.1.1)

bin/profile

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "smart_todo"
+
+class NullDispatcher < SmartTodo::Dispatchers::Base
+  class << self
+    def validate_options!(_); end
+  end
+
+  def dispatch
+  end
+end
+
+exit SmartTodo::CLI.new(NullDispatcher).run

lib/smart_todo.rb

@@ -1,25 +1,14 @@
 # frozen_string_literal: true
 
+require "prism"
 require "smart_todo/version"
 require "smart_todo/events"
 
 module SmartTodo
   autoload :SlackClient,              "smart_todo/slack_client"
   autoload :CLI,                      "smart_todo/cli"
-
-  module Parser
-    autoload :CommentParser,          "smart_todo/parser/comment_parser"
-    autoload :TodoNode,               "smart_todo/parser/todo_node"
-    autoload :MetadataParser,         "smart_todo/parser/metadata_parser"
-  end
-
-  module Events
-    autoload :Date,                   "smart_todo/events/date"
-    autoload :GemBump,                "smart_todo/events/gem_bump"
-    autoload :GemRelease,             "smart_todo/events/gem_release"
-    autoload :IssueClose,             "smart_todo/events/issue_close"
-    autoload :RubyVersion,            "smart_todo/events/ruby_version"
-  end
+  autoload :Todo,                     "smart_todo/todo"
+  autoload :CommentParser,            "smart_todo/comment_parser"
 
   module Dispatchers
     autoload :Base,                   "smart_todo/dispatchers/base"

lib/smart_todo/cli.rb

@@ -1,15 +1,17 @@
 # frozen_string_literal: true
 
 require "optionparser"
+require "etc"
 
 module SmartTodo
   # This class is the entrypoint of the SmartTodo library and is responsible
   # to retrieve the command line options as well as iterating over each files/directories
   # to run the +CommentParser+ on.
   class CLI
-    def initialize
+    def initialize(dispatcher = nil)
       @options = {}
       @errors = []
+      @dispatcher = dispatcher
     end
 
     # @param args [Array<String>]
@@ -19,15 +21,18 @@ def run(args = ARGV)
 
       paths << "." if paths.empty?
 
+      comment_parser = CommentParser.new
       paths.each do |path|
-        normalize_path(path).each do |file|
-          parse_file(file)
+        normalize_path(path).each do |filepath|
+          comment_parser.parse_file(filepath)
 
           $stdout.print(".")
           $stdout.flush
         end
       end
 
+      process_dispatches(process_todos(comment_parser.todos))
+
       if @errors.empty?
         0
       else
@@ -79,25 +84,54 @@ def normalize_path(path)
       end
     end
 
-    # @param file [String] a path to a file
-    def parse_file(file)
-      Parser::CommentParser.new(File.read(file, encoding: "UTF-8")).parse.each do |todo_node|
+    def process_todos(todos)
+      events = Events.new
+      dispatches = []
+
+      todos.each do |todo|
         event_message = nil
-        event_met = todo_node.metadata.events.find do |event|
-          event_message = Events.public_send(event.method_name, *event.arguments)
+        event_met = todo.events.find do |event|
+          event_message = events.public_send(event.method_name, *event.arguments)
         rescue => e
-          message = "Error while parsing #{file} on event `#{event.method_name}` with arguments #{event.arguments}: " \
+          message = "Error while parsing #{todo.filepath} on event `#{event.method_name}` " \
+            "with arguments #{event.arguments.map(&:inspect)}: " \
             "#{e.message}"
 
           @errors << message
 
           nil
         end
 
-        @errors.concat(todo_node.metadata.errors)
-
-        dispatcher.new(event_message, todo_node, file, @options).dispatch if event_met
+        @errors.concat(todo.errors)
+        dispatches << [event_message, todo] if event_met
       end
+
+      dispatches
+    end
+
+    def process_dispatches(dispatches)
+      queue = Queue.new
+      dispatches.each { |dispatch| queue << dispatch }
+
+      thread_count = Etc.nprocessors
+      thread_count.times { queue << nil }
+
+      threads =
+        thread_count.times.map do
+          Thread.new do
+            Thread.current.abort_on_exception = true
+
+            loop do
+              dispatch = queue.pop
+              break if dispatch.nil?
+
+              (event_message, todo) = dispatch
+              dispatcher.new(event_message, todo, todo.filepath, @options).dispatch
+            end
+          end
+        end
+
+      threads.each(&:join)
     end
   end
 end

lib/smart_todo/comment_parser.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SmartTodo
+  class CommentParser
+    attr_reader :todos
+
+    def initialize
+      @todos = []
+    end
+
+    def parse(source, filepath = "-e")
+      parse_comments(Prism.parse_comments(source), filepath)
+    end
+
+    def parse_file(filepath)
+      parse_comments(Prism.parse_file_comments(filepath), filepath)
+    end
+
+    class << self
+      def parse(source)
+        parser = new
+        parser.parse(source)
+        parser.todos
+      end
+    end
+
+    private
+
+    def parse_comments(comments, filepath)
+      current_todo = nil
+
+      comments.each do |comment|
+        next unless comment.is_a?(Prism::InlineComment)
+
+        source = comment.location.slice
+
+        if source.match?(/^#\sTODO\(/)
+          todos << current_todo if current_todo
+          current_todo = Todo.new(source, filepath)
+        elsif current_todo && (indent = source[/^#(\s*)/, 1].length) && (indent - current_todo.indent == 2)
+          current_todo << "#{source[(indent + 1)..]}\n"
+        else
+          todos << current_todo if current_todo
+          current_todo = nil
+        end
+      end
+
+      todos << current_todo if current_todo
+    end
+  end
+end

lib/smart_todo/dispatchers/base.rb

@@ -40,7 +40,7 @@ def initialize(event_message, todo_node, file, options)
         @todo_node = todo_node
         @options = options
         @file = file
-        @assignees = @todo_node.metadata.assignees
+        @assignees = @todo_node.assignees
       end
 
       # This method gets called when a TODO reminder is expired and needs to be delivered.