Merge pull request #1279 from edpaget/import-update

Import and store legacy project counts by workflow

app/controllers/api/events_controller.rb

@@ -2,8 +2,6 @@ module Api
 
   class EventsController < ApplicationController
 
-    skip_before_action :verify_authenticity_token, if: :json_request?
-
     KNOWN_EVENTS = %w( activity workflow_activity )
 
     def self.resource_name
@@ -33,16 +31,16 @@ def require_basic_authentication
 
     def process_incoming_event
       response_status = :unprocessable_entity
-      if event_params_sanity_check
+      if event_params_check
         if upp = user_project_preference
-          upp.activity_count = create_params[:count]
+          upp.legacy_count[create_params[:workflow]] = create_params[:count]
           response_status = :ok if upp.save
         end
       end
       render status: response_status, nothing: true
     end
 
-    def event_params_sanity_check
+    def event_params_check
       required_params && known_event? && legacy_zoo_project_exists?
     end
 
@@ -85,7 +83,7 @@ def resource_sym
     def legacy_zoo_project
       @legacy_zoo_project ||=
         Project.where(migrated: true)
-        .where("configuration ->> 'zoo_home_project_id' = ?", create_params[:project_id])
+        .where("configuration ->> 'zoo_home_project_id' = ?", create_params[:project_id].to_s)
         .first
     end
 

app/serializers/user_project_preference_serializer.rb

@@ -8,7 +8,11 @@ def self.key
   end
 
   def activity_count
-    @model.activity_count || user_project_activity
+    if !@model.legacy_count.blank?
+      @model.legacy_count.values.reduce(:+)
+    else
+      @model.activity_count || user_project_activity
+    end
   end
 
   def user_project_activity

db/migrate/20150817145756_add_legacy_count_to_project_preferences.rb

@@ -0,0 +1,5 @@
+class AddLegacyCountToProjectPreferences < ActiveRecord::Migration
+  def change
+    add_column :user_project_preferences, :legacy_count, :jsonb, default: {}
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150811202500) do
+ActiveRecord::Schema.define(version: 20150817145756) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -298,6 +298,7 @@
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer  "activity_count"
+    t.jsonb    "legacy_count",        default: {}
   end
 
   create_table "user_seen_subjects", force: :cascade do |t|

lib/zooniverse_user_subscription.rb

@@ -11,65 +11,32 @@ class MissingLegacyProject < StandardError; end
   serialize :summary, Hash
 
   def self.import_zoo_user_subscriptions(user_logins=nil)
-    setup_caches
-    zoo_user_scope = zoo_users_to_import(user_logins)
-    total_batches = zoo_user_scope.count / ZOO_USER_BATCH_SIZE
-    zoo_user_scope.find_in_batches.with_index do |zoo_users, batch|
-      puts "Processing zoo user batch: #{ batch + 1} out of #{total_batches}"
-      import_batch(zoo_users)
-    end
-  end
-
-  def self.setup_caches
-    @cached_projects = {}
-    @cached_users = {}
+    import_ids = zoo_users_to_import(user_logins)
+    subs = zoo_project_subscriptions(import_ids)
+    subs.find_each.with_index(&:import)
   end
 
   def self.zoo_users_to_import(user_logins)
     if user_logins.blank?
-      ZooniverseUser.all
+      nil
     else
-      ZooniverseUser.where(users: { login: user_logins })
-    end.select(:id)
-  end
-
-  def self.import_batch(zoo_users)
-    #reset the cache for each batch
-    @cached_users = {}
-    import_user_scope = zoo_project_subscriptions(zoo_users.map(&:id))
-    total = import_user_scope.count.keys.count
-    import_user_scope.find_each.with_index do |user_project_sub, index|
-      puts "#{ index } / #{ total }" if index % 1_000 == 0
-      unless user_project_sub.empty_summary?
-        user_project_sub.import
-      end
+      ZooniverseUser.where(users: { login: user_logins }).pluck(:id)
     end
   end
 
-  def self.cached_users
-    @cached_users
-  end
-
-  def self.cached_projects
-    @cached_projects
-  end
-
   def self.zoo_project_subscriptions(user_ids)
-    self.where.not(user_id: nil, project_id: nil)
-      includes(:zooniverse_user)
-      .joins(:zooniverse_user)
-      .where(users: { id: user_ids })
-      .distinct
-      .group(:user_id, :project_id)
+    query = self.where.not(user_id: nil, project_id: nil)
+    query = query.where(user_id: user_ids) if user_ids
+    query
   end
 
-  def import
+  def import(index)
     return if user_id.nil? || empty_summary?
     zoo_project = find_legacy_migrated_project
     if panoptes_user = find_migrated_user
       migrated_upp = UserProjectPreference.find_or_initialize_by(project_id: zoo_project.id,
                                                                  user_id: panoptes_user.id)
-      migrated_upp.activity_count = summate_activity_counts
+      migrated_upp.legacy_count = summate_activity_counts
       migrated_upp.email_communication = (notifications || true)
       migrated_upp.save! if migrated_upp.changed?
     end
@@ -82,41 +49,34 @@ def empty_summary?
   private
 
   def find_legacy_migrated_project
-    return self.class.cached_projects[project_id] if self.class.cached_projects.has_key?(project_id)
     legacy_project = Project.where(migrated: true)
       .where("configuration ->> 'zoo_home_project_id' = '#{project_id}'")
       .first
     unless legacy_project
       raise MissingLegacyProject.new("Legacy project missing make sure it has been migrated!")
     end
-    self.class.cached_projects[project_id] = legacy_project
+    legacy_project
   end
 
   def find_migrated_user
-    if self.class.cached_users.has_key?(zooniverse_user.id)
-      self.class.cached_users[zooniverse_user.id]
-    else
-      migrated_user = User.where(zooniverse_id: zooniverse_user.id)
-        .or(User.where(email: zooniverse_user.email))
-        .first
-      unless migrated_user
-        p "Skipping subscription for non-migrated user account: #{zooniverse_user.login}"
-      end
-      self.class.cached_users[zooniverse_user.id] = migrated_user
+    unless migrated_user = User.find_by(zooniverse_id: user_id)
+      p "Skipping subscription for non-migrated user account: #{zooniverse_user.login}"
     end
+    migrated_user
   end
 
   # Summate all the activity counts for the user : project id combo
   # original DB has no validations and there are duplicates with different values
   def summate_activity_counts
-    activity_count = 0
-    all_user_projects_subscriptions.find_each do |subscription|
+    all_user_projects_subscriptions.reduce({}) do |counts, subscription|
       next if subscription.empty_summary?
-      if count = subscription.summary.values.first[:count].to_i
-        activity_count += count
+      summary = subscription.summary
+      summary.each do |workflow, value|
+        counts[workflow] ||= 0
+        counts[workflow] += value[:count].to_i
       end
+      counts
     end
-    activity_count
   end
 
   def all_user_projects_subscriptions

spec/controllers/api/events_controller_spec.rb

@@ -26,16 +26,17 @@ def overridden_params(new_params)
         user.update_column(:zooniverse_id, 1)
         user
       end
-      let(:event_count) { 10 }
+      let(:event_count) { {workflow.display_name => 10} }
       let(:created_at) { project.created_at.to_s }
       let(:event_kind) { "workflow_activity" }
       let(:event_params) do
         {
           event: {
             kind: event_kind, project_id: zoo_home_project_id, project: project.name,
             zooniverse_user_id: user.zooniverse_id, workflow: workflow.display_name,
-            count: event_count, created_at: created_at
-          }
+            count: event_count.values.first, created_at: created_at
+          },
+          format: :json
         }
       end
       let(:user_project_pref) do
@@ -205,9 +206,9 @@ def overridden_params(new_params)
           end.to change { UserProjectPreference.count }.from(0).to(1)
         end
 
-        it "should update the upp activity_count to the correct value" do
+        it "should update the upp legacy_count to the correct value" do
           post :create, event_params
-          expect(user_project_pref.activity_count).to eq(event_count)
+          expect(user_project_pref.legacy_count).to eq(event_count)
         end
 
         context "with an panoptes-formatted user zooniverse_id" do
@@ -221,7 +222,7 @@ def overridden_params(new_params)
 
         context "when the user project preference already exists" do
           let!(:upp) do
-            create(:user_project_preference, project: project, user: user, activity_count: 100)
+            create(:user_project_preference, project: project, user: user, legacy_count: {workflow.display_name => 100})
           end
 
           it "should update the model only" do
@@ -230,9 +231,9 @@ def overridden_params(new_params)
             end.to_not change { UserProjectPreference.count }.from(1)
           end
 
-          it "should overwrite the upp activity_count to the correct value" do
+          it "should overwrite the upp legacy_count to the correct value" do
             post :create, event_params
-            expect(upp.reload.activity_count).to eq(event_count)
+            expect(upp.reload.legacy_count).to eq(event_count)
           end
         end
       end