AO3-6210 Load the tags in the mass wrangling bins from Elasticsearch

app/controllers/tag_wranglers_controller.rb

@@ -1,5 +1,6 @@
 class TagWranglersController < ApplicationController
   include ExportsHelper
+  include WranglingHelper
 
   before_action :check_user_status
   before_action :check_permission_to_wrangle, except: [:report_csv]
@@ -41,15 +42,7 @@ def show
     @wrangler = User.find_by!(login: params[:id])
     @page_subtitle = @wrangler.login
     @fandoms = @wrangler.fandoms.by_name
-    @counts = {}
-    [Fandom, Character, Relationship, Freeform].each do |klass|
-      @counts[klass.to_s.downcase.pluralize.to_sym] = Rails.cache.fetch("/wrangler/counts/sidebar/#{klass}", race_condition_ttl: 10, expires_in: 1.hour) do
-        klass.unwrangled.in_use.count
-      end
-    end
-    @counts[:UnsortedTag] = Rails.cache.fetch("/wrangler/counts/sidebar/UnsortedTag", race_condition_ttl: 10, expires_in: 1.hour) do
-      UnsortedTag.count
-    end
+    @counts = tag_counts_per_category
   end
 
   def report_csv

app/controllers/tag_wranglings_controller.rb

@@ -1,34 +1,33 @@
 class TagWranglingsController < ApplicationController
   include TagWrangling
+  include WranglingHelper
 
   before_action :check_user_status
   before_action :check_permission_to_wrangle
   around_action :record_wrangling_activity, only: [:wrangle]
 
   def index
-    @counts = {}
-    [Fandom, Character, Relationship, Freeform].each do |klass|
-      @counts[klass.to_s.downcase.pluralize.to_sym] = Rails.cache.fetch("/wrangler/counts/sidebar/#{klass}", race_condition_ttl: 10, expires_in: 1.hour) do
-        klass.unwrangled.in_use.count
-      end
-    end
-    @counts[:UnsortedTag] = Rails.cache.fetch("/wrangler/counts/sidebar/UnsortedTag", race_condition_ttl: 10, expires_in: 1.hour) do
-      UnsortedTag.count
-    end
+    @counts = tag_counts_per_category
     unless params[:show].blank?
+      raise "Redshirt: Attempted to constantize invalid class initialize tag_wranglings_controller_index #{params[:show].classify}" unless Tag::USER_DEFINED.include?(params[:show].classify)
+
       params[:sort_column] = 'created_at' if !valid_sort_column(params[:sort_column], 'tag')
       params[:sort_direction] = 'ASC' if !valid_sort_direction(params[:sort_direction])
-      sort = params[:sort_column] + " " + params[:sort_direction]
-      sort = sort + ", name ASC" if sort.include?('taggings_count_cache')
+
       if params[:show] == "fandoms"
         @media_names = Media.by_name.pluck(:name)
         @page_subtitle = ts("fandoms")
-        @tags = Fandom.unwrangled.in_use.order(sort).paginate(page: params[:page], per_page: ArchiveConfig.ITEMS_PER_PAGE)
-      else # by fandom
-        raise "Redshirt: Attempted to constantize invalid class initialize tag_wranglings_controller_index #{params[:show].classify}" unless Tag::USER_DEFINED.include?(params[:show].classify)
-        klass = params[:show].classify.constantize
-        @tags = klass.unwrangled.in_use.order(sort).paginate(page: params[:page], per_page: ArchiveConfig.ITEMS_PER_PAGE)
       end
+
+      type = params[:show].singularize.capitalize
+      @tags = TagQuery.new({
+                             type: type,
+                             to_wrangle: true,
+                             sort_column: params[:sort_column],
+                             sort_direction: params[:sort_direction],
+                             page: params[:page],
+                             per_page: ArchiveConfig.ITEMS_PER_PAGE
+                           }).search_results
     end
   end
 

app/controllers/unsorted_tags_controller.rb

@@ -1,19 +1,12 @@
 class UnsortedTagsController < ApplicationController
+  include WranglingHelper
 
   before_action :check_user_status
   before_action :check_permission_to_wrangle
 
   def index
     @tags = UnsortedTag.page(params[:page])
-    @counts = {}
-    [Fandom, Character, Relationship, Freeform].each do |klass|
-      @counts[klass.to_s.downcase.pluralize.to_sym] = Rails.cache.fetch("/wrangler/counts/sidebar/#{klass}", race_condition_ttl: 10, expires_in: 1.hour) do
-        klass.unwrangled.in_use.count
-      end
-    end
-    @counts[:UnsortedTag] = Rails.cache.fetch("/wrangler/counts/sidebar/UnsortedTag", race_condition_ttl: 10, expires_in: 1.hour) do 
-      UnsortedTag.count 
-    end
+    @counts = tag_counts_per_category
   end
 
   def mass_update

app/helpers/wrangling_helper.rb

@@ -0,0 +1,14 @@
+module WranglingHelper
+  def tag_counts_per_category
+    counts = {}
+    [Fandom, Character, Relationship, Freeform].each do |klass|
+      counts[klass.to_s.downcase.pluralize.to_sym] = Rails.cache.fetch("/wrangler/counts/sidebar/#{klass}", race_condition_ttl: 10, expires_in: 1.hour) do
+        TagQuery.new(type: klass.to_s, to_wrangle: true).count
+      end
+    end
+    counts[:UnsortedTag] = Rails.cache.fetch("/wrangler/counts/sidebar/UnsortedTag", race_condition_ttl: 10, expires_in: 1.hour) do
+      UnsortedTag.count
+    end
+    counts
+  end
+end

app/models/fandom.rb

@@ -14,9 +14,8 @@ class Fandom < Tag
 
   scope :by_media, lambda {|media| where(media_id: media.id)}
 
-  def self.unwrangled
-    joins(:common_taggings).
-    where("unwrangleable = 0 AND common_taggings.filterable_id = ? AND common_taggings.filterable_type = 'Tag'", Media.uncategorized.try(:id))
+  def unwrangled?
+    Media.uncategorized.fandoms.include?(self)
   end
 
   # An association callback to add the default media if all others have been removed

app/models/search/tag_indexer.rb

@@ -23,7 +23,8 @@ def self.mapping
         },
         tag_type: { type: "keyword" },
         sortable_name: { type: "keyword" },
-        uses: { type: "integer" }
+        uses: { type: "integer" },
+        unwrangled: { type: "boolean" }
       }
     }
   end
@@ -67,7 +68,8 @@ def document(object)
     ).merge(
       has_posted_works: object.has_posted_works?,
       tag_type: object.type,
-      uses: object.taggings_count_cache
+      uses: object.taggings_count_cache,
+      unwrangled: object.unwrangled?
     ).merge(parent_data(object))
   end
 

app/models/search/tag_query.rb

@@ -22,7 +22,8 @@ def filters
       fandom_filter,
       character_filter,
       suggested_fandom_filter,
-      suggested_character_filter
+      suggested_character_filter,
+      *to_wrangle_filter
     ].flatten.compact
   end
 
@@ -60,7 +61,11 @@ def sort
       sort_hash[column][:unmapped_type] = "date"
     end
 
-    [sort_hash, { id: { order: direction } }]
+    sort_by_id = { id: { order: direction } }
+
+    return [sort_hash, { "name.keyword" => { order: "asc" } }, sort_by_id] if column == "uses"
+
+    [sort_hash, sort_by_id]
   end
 
   ################
@@ -103,6 +108,16 @@ def suggested_character_filter
     terms_filter(:pre_character_ids, options[:pre_character_ids]) if options[:pre_character_ids]
   end
 
+  def to_wrangle_filter
+    return [] unless options[:to_wrangle]
+
+    [
+      { bool: { should: [{ range: { uses: { gt: 0 } } }, term_filter(:canonical, true)] } }, # Check if used OR canonical
+      term_filter(:unwrangleable, false),
+      term_filter(:unwrangled, true)
+    ]
+  end
+
   # Filter to only include tags that have no assigned fandom_ids. Checks that
   # the fandom exists, because this particular filter is included in the
   # exclusion_filters section.

app/models/tag.rb

@@ -271,9 +271,6 @@ def destroy_common_tagging(parent)
   scope :unfilterable, -> { nonsynonymous.where(unwrangleable: false) }
   scope :unwrangleable, -> { where(unwrangleable: true) }
 
-  # we need to manually specify a LEFT JOIN instead of just joins(:common_taggings or :meta_taggings) here because
-  # what we actually need are the empty rows in the results
-  scope :unwrangled, -> { joins("LEFT JOIN `common_taggings` ON common_taggings.common_tag_id = tags.id").where("unwrangleable = 0 AND common_taggings.id IS NULL") }
   scope :in_use, -> { where("canonical = 1 OR taggings_count_cache > 0") }
   scope :first_class, -> { joins("LEFT JOIN `meta_taggings` ON meta_taggings.sub_tag_id = tags.id").where("meta_taggings.id IS NULL") }
 
@@ -1033,6 +1030,10 @@ def merger_string=(tag_string)
     end
   end
 
+  def unwrangled?
+    common_taggings.empty?
+  end
+
   #################################
   ## SEARCH #######################
   #################################

app/views/tag_wranglings/index.html.erb

@@ -104,32 +104,34 @@
 
             <tbody>
               <% @tags.each do |tag| %>
-                <tr>
-                  <th scope="row" title="<%= ts('tag') %>">
-                    <%= check_box_tag 'selected_tags[]', tag.id, nil, id: "selected_tags_#{tag.id}" %>
-                    <%= label_tag "selected_tags_#{tag.id}", "#{tag.name}" %>
-                  </th>
-
-                  <td title="<%= ts('created') %>"><%= l(tag.created_at.to_date) %></td>
-
-                  <td title="<%= ts('canonical?') %>">
-                    <% if tag.canonical? %>
-                      <%= ts('Yes') %>
-                    <% else %>
-                      <%= check_box_tag 'canonicals[]', tag.id, tag.canonical?, id: "canonicals_#{tag.id}" %>
-                    <% end %>
-                  </td>
-
-                  <td title="<%= ts('taggings') %>"><%= tag.taggings_count %></td>
-
-                  <td>
-                    <ul class="actions" role="navigation">
-                      <li><%= link_to ts('Edit'), {controller: :tags, action: :edit, id: tag} %></li>
-                      <li><%= link_to ts('Wrangle'), {controller: :tags, action: :wrangle, id: tag} %></li>
-                      <li><%= link_to ts('Works'), {controller: :works, action: :index, tag_id: tag} %></li>
-                    </ul>
-                  </td>
-                </tr>
+                <% if tag.unwrangled? %>
+                  <tr>
+                    <th scope="row" title="<%= ts("tag") %>">
+                      <%= check_box_tag "selected_tags[]", tag.id, nil, id: "selected_tags_#{tag.id}" %>
+                      <%= label_tag "selected_tags_#{tag.id}", tag.name.to_s %>
+                    </th>
+
+                    <td title="<%= ts("created") %>"><%= l(tag.created_at.to_date) %></td>
+
+                    <td title="<%= ts("canonical?") %>">
+                      <% if tag.canonical? %>
+                        <%= ts("Yes") %>
+                      <% else %>
+                        <%= check_box_tag "canonicals[]", tag.id, tag.canonical?, id: "canonicals_#{tag.id}" %>
+                      <% end %>
+                    </td>
+
+                    <td title="<%= ts("taggings") %>"><%= tag.taggings_count_cache %></td>
+
+                    <td>
+                      <ul class="actions" role="navigation">
+                        <li><%= link_to ts("Edit"), { controller: :tags, action: :edit, id: tag } %></li>
+                        <li><%= link_to ts("Wrangle"), { controller: :tags, action: :wrangle, id: tag } %></li>
+                        <li><%= link_to ts("Works"), { controller: :works, action: :index, tag_id: tag } %></li>
+                      </ul>
+                    </td>
+                  </tr>
+                <% end %>
               <% end %>
             </tbody>
           </table>

features/step_definitions/generic_steps.rb

@@ -267,3 +267,7 @@ def assure_xpath_not_present(tag, attribute, value, selector)
   Time.zone = zone
   page.body.should =~ /#{Regexp.escape(Time.zone.now.zone)}/
 end
+
+When "I clear Rails cache" do
+  Rails.cache.clear
+end

features/step_definitions/tag_steps.rb

@@ -192,6 +192,7 @@
   step %{I am logged in as a random user}
   step %{I post the work "Revenge of the Sith 2" with fandom "Star Wars, Stargate SG-1" with character "Daniel Jackson" with second character "Jack O'Neil" with rating "Not Rated" with relationship "JackDaniel"}
   step %{The periodic tag count task is run}
+  step %{all indexing jobs have been run}
   step %{I flush the wrangling sidebar caches}
 end
 
@@ -443,3 +444,7 @@
   tag = Tag.find_by(name: tagname)
   puts tag.inspect
 end
+
+Then "no tag is scheduled for count update from now on" do
+  Tag.any_instance.should_not_receive(:write_taggings_to_redis)
+end

features/tags_and_wrangling/brand_new_fandoms.feature

@@ -12,6 +12,7 @@ Feature: Brand new fandoms
     Given I am logged in as a random user
       And I post a work "My New Work" with fandom "My Brand New Fandom"
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Uncategorized Fandoms" within "#header"
     Then I should see "My Brand New Fandom"
 
@@ -21,13 +22,15 @@ Feature: Brand new fandoms
       And I fill in "Fandoms" with "My Brand New Fandom"
       And I submit
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Uncategorized Fandoms" within "#header"
     Then I should see "My Brand New Fandom"
 
   Scenario: When the only work with a brand new fandom is destroyed, the fandom should not be visible on the Uncategorized Fandoms page.
     Given I am logged in as a random user
       And I post a work "My New Work" with fandom "My Brand New Fandom"
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Edit"
       And I follow "Delete Work"
       And I press "Yes"
@@ -43,6 +46,7 @@ Feature: Brand new fandoms
       And I fill in "Fandoms" with "My Brand New Fandom"
       And I submit
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I am logged in as a "policy_and_abuse" admin
       And I view the external work "External Work To Be Deleted"
       And I follow "Delete External Work"
@@ -55,6 +59,7 @@ Feature: Brand new fandoms
     Given I am logged in as a tag wrangler
       And I post a work "My New Work" with fandom "My Brand New Fandom"
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Tag Wrangling" within "#header"
       And I follow "Fandoms by media"
     Then I should see "My Brand New Fandom"
@@ -65,6 +70,7 @@ Feature: Brand new fandoms
       And I fill in "Fandoms" with "My Brand New Fandom"
       And I submit
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Tag Wrangling" within "#header"
       And I follow "Fandoms by media"
     Then I should see "My Brand New Fandom"
@@ -73,11 +79,13 @@ Feature: Brand new fandoms
     Given I am logged in as a tag wrangler
       And I post a work "My New Work" with fandom "My Brand New Fandom"
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I follow "Edit"
       And I follow "Delete Work"
       And I press "Yes"
     Then I should see "Your work My New Work was deleted."
     When the periodic tag count task is run
+      And all indexing jobs have been run
       And I follow "Tag Wrangling" within "#header"
       And I follow "Fandoms by media"
     Then I should not see "My Brand New Fandom"
@@ -89,11 +97,13 @@ Feature: Brand new fandoms
       And I fill in "Fandoms" with "My Brand New Fandom"
       And I submit
       And the periodic tag count task is run
+      And all indexing jobs have been run
     When I am logged in as a "policy_and_abuse" admin
       And I view the external work "External Work To Be Deleted"
       And I follow "Delete External Work"
     Then I should see "Item was successfully deleted."
     When the periodic tag count task is run
+      And all indexing jobs have been run
       And I am logged in as a tag wrangler
       And I follow "Tag Wrangling" within "#header"
       And I follow "Fandoms by media"

features/tags_and_wrangling/tag_wrangling.feature

@@ -335,7 +335,8 @@ Feature: Tag wrangling
       And I should see "Grandparent" within "#parent_MetaTag_associations_to_remove_checkboxes"
       But I should not see "Parent" within "#parent_MetaTag_associations_to_remove_checkboxes"
 
-    When I view the tag "Child"
+    When I clear Rails cache
+      And I view the tag "Child"
     Then I should see "Grandparent" within ".meta"
       But I should not see "Parent" within ".meta"
 
@@ -353,3 +354,13 @@ Feature: Tag wrangling
     Then I should see "Youngest"
       But I should not see "Oldest"
       And I should not see "Middle"
+
+  Scenario: No call to Redis when no action is taken
+    Given the tag wrangling setup
+      And I am logged in as a tag wrangler
+    Then no tag is scheduled for count update from now on
+    When I go to my wrangling page
+    Then I should see "Wrangling Home"
+      And I should see "Characters by fandom (2)"
+    When I follow "Characters by fandom (2)"
+    Then I should see "Mass Wrangle New/Unwrangled Tags"