Begin outlining the design for multiple scanners

Refs #163

lib/i18n/tasks/scanners/base_scanner.rb

@@ -1,5 +1,6 @@
 require 'i18n/tasks/key_pattern_matching'
 require 'i18n/tasks/scanners/relative_keys'
+require 'i18n/tasks/scanners/file_provider'
 
 module I18n::Tasks::Scanners
   class BaseScanner
@@ -35,6 +36,7 @@ def initialize(config = {})
         }
         @ignore_lines_res = conf[:ignore_lines].inject({}) { |h, (ext, re)| h.update(ext => Regexp.new(re)) }
         @key_filter = nil
+        @file_provider = FileProvider.new(search_paths: conf[:paths], include: conf[:include], exclude: conf[:exclude])
       end
     end
 
@@ -59,37 +61,16 @@ def keys(opts = {})
     end
 
     def read_file(path)
-      result = nil
-      File.open(path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
-      result
+      @file_provider.read_file(path)
     end
 
     # @return [Array<Key>] keys found in file
     def scan_file(path, opts = {})
       raise 'Unimplemented'
     end
 
-    # Run given block for every relevant file, according to config
-    # @return [Array] Results of block calls
     def traverse_files
-      result = []
-      paths  = config[:paths].select { |p| File.exist?(p) }
-      if paths.empty?
-        log_warn "search.paths #{config[:paths].inspect} do not exist"
-        return result
-      end
-      Find.find(*paths) do |path|
-        is_dir   = File.directory?(path)
-        hidden   = File.basename(path).start_with?('.')
-        not_incl = config[:include] && !path_fnmatch_any?(path, config[:include])
-        excl     = path_fnmatch_any?(path, config[:exclude])
-        if is_dir || hidden || not_incl || excl
-          Find.prune if is_dir && (hidden || excl)
-        else
-          result << yield(path)
-        end
-      end
-      result
+      @file_provider.traverse_files { |path| yield path }
     end
 
     def with_key_filter(key_filter = nil)
@@ -102,10 +83,6 @@ def with_key_filter(key_filter = nil)
 
     protected
 
-    def path_fnmatch_any?(path, globs)
-      globs.any? { |glob| File.fnmatch(glob, path) }
-    end
-
     def src_location(path, text, src_pos, position = true)
       data = {src_path: path}
       if position

lib/i18n/tasks/scanners/file_provider.rb

@@ -0,0 +1,84 @@
+module I18n::Tasks::Scanners
+  # Finds the files and provides their source.
+  #
+  # @note This class is thread-safe. All methods are cached.
+  # @since 0.9.0
+  class FileProvider
+    include I18n::Tasks::Logging
+
+    # @param search_paths [Array<String>] {Find.find}-compatible paths to traverse,
+    #     absolute or relative to the working directory.
+    # @param include [Array<String>, nil] {File.fnmatch}-compatible patterns files to include.
+    #     Files not matching any of the inclusion patterns will be excluded.
+    # @param exclude [Arry<String>] {File.fnmatch}-compatible patterns of files to exclude.
+    #     Files matching any of the exclusion patterns will be excluded even if they match an inclusion pattern.
+    def initialize(search_paths: [], include: nil, exclude: [])
+      @search_paths = search_paths
+      @include      = include
+      @exclude      = exclude
+
+      @file_cache = {}
+      @mutex      = Mutex.new
+    end
+
+    # Traverse the paths and yield the matching ones.
+    #
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @yield [path]
+    # @yieldparam path [String] the path of the found file.
+    # @return [Array<of block results>]
+    def traverse_files
+      find_paths.map { |path| yield path }
+    end
+
+    # Return the contents of the file at the given path.
+    # The file is read in the 'rb' mode and assuming UTF-8 encoding.
+    #
+    # @note This method is cached, it will only access the filesystem on the first invocation.
+    # @param [String] path Path to the file, absolute or relative to the working directory.
+    def read_file(path)
+      absolute_path = File.expand_path(path)
+      contents      = @file_cache[absolute_path]
+      return contents if contents
+      @mutex.synchronize do
+        @file_cache[absolute_path] ||= begin
+          result = nil
+          File.open(absolute_path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
+          result
+        end
+      end
+    end
+
+    private
+
+    def path_fnmatch_any?(path, globs)
+      globs.any? { |glob| File.fnmatch(glob, path) }
+    end
+
+    def find_paths
+      return @paths if @paths
+      @mutex.synchronize do
+        @paths ||= begin
+          paths        = []
+          search_paths = @search_paths.select { |p| File.exist?(p) }
+          if search_paths.empty?
+            log_warn "None of the search.paths exist #{@search_paths.inspect}"
+          else
+            Find.find(*search_paths) do |path|
+              is_dir   = File.directory?(path)
+              hidden   = File.basename(path).start_with?('.')
+              not_incl = @include && !path_fnmatch_any?(path, @include)
+              excl     = path_fnmatch_any?(path, @exclude)
+              if is_dir || hidden || not_incl || excl
+                Find.prune if is_dir && (hidden || excl)
+              else
+                paths << path
+              end
+            end
+          end
+          paths
+        end
+      end
+    end
+  end
+end

lib/i18n/tasks/scanners/scanner.rb

@@ -0,0 +1,58 @@
+module I18n::Tasks::Scanners
+  # Describes the API of a scanner.
+  #
+  # @abstract
+  # @since 0.9.0
+  class Scanner
+    # @abstract
+    # @return [Array<KeyOccurrences>] the keys found by this scanner and their occurrences.
+    def keys
+      raise 'Unimplemented'
+    end
+  end
+
+  # A scanned key and all its occurrences.
+  class KeyOccurrences
+    # @return [String] the key.
+    attr_reader :key
+
+    # @return [Array<Occurrence>] the key's occurrences.
+    attr_reader :occurrences
+
+    def initialize(key:, occurrences:)
+      @key         = key
+      @occurrences = occurrences
+    end
+  end
+
+  # The occurrence of some key in a file.
+  class Occurrence
+    # @return [String] source path relative to the current working directory.
+    attr_reader :path
+
+    # @return [Fixnum] count of characters in the file before the occurrence.
+    attr_reader :pos
+
+    # @return [Fixnum] line number of the occurrence, counting from 1.
+    attr_reader :line_num
+
+    # @return [Fixnum] position of the start of the occurrence in the line, counting from 1.
+    attr_reader :line_pos
+
+    # @return [String] the line of the occurrence, excluding the last LF or CRLF.
+    attr_reader :line
+
+    # @param path     [String]
+    # @param pos      [Fixnum]
+    # @param line_num [Fixnum]
+    # @param line_pos [Fixnum]
+    # @param line     [String
+    def initialize(path:, pos:, line_num:, line_pos:, line:)
+      @path     = path
+      @pos      = pos
+      @line_num = line_num
+      @line_pos = line_pos
+      @line     = line
+    end
+  end
+end

lib/i18n/tasks/scanners/scanner_multiplexer.rb

@@ -0,0 +1,37 @@
+module I18n::Tasks::Scanners
+  # Run multiple {Scanner Scanners} and merge their results.
+  # @note The scanners are run concurrently. A thread is spawned per each scanner.
+  # @since 0.9.0
+  class ScannerMultiplexer < Scanner
+    # @param scanners [Array<Scanner>]
+    def initialize(scanners:)
+      @scanners = scanners
+    end
+
+    # Collect the results of all the scanners. Occurrences of a key from multiple scanners are merged.
+    #
+    # @note The scanners are run concurrently. A thread is spawned per each scanner.
+    # @return (see Scanner#keys)
+    def keys
+      collect_results.inject({}) { |results_by_key, key_occurences|
+        key_occurences.each do |key_occurrence|
+          (results_by_key[key_occurrence.key] ||= []) << key_occurrence.occurrences
+        end
+        results_by_key
+      }.map { |key, all_occurrences|
+        KeyOccurrences.new(key: key, occurrences: all_occurrences.flatten(1))
+      }
+    end
+
+    private
+
+    # @return Array<Array<KeyOccurrences>>
+    def collect_results
+      Array.new(@scanners.length).tap do |results|
+        @scanners.map.with_index { |scanner, i|
+          Thread.start { results[i] = scanner.keys }
+        }.each(&:join)
+      end
+    end
+  end
+end