diff --git a/app/controllers/api/events_controller.rb b/app/controllers/api/events_controller.rb index d957a0412..8d953386c 100644 --- a/app/controllers/api/events_controller.rb +++ b/app/controllers/api/events_controller.rb @@ -2,8 +2,6 @@ module Api class EventsController < ApplicationController - skip_before_action :verify_authenticity_token, if: :json_request? - KNOWN_EVENTS = %w( activity workflow_activity ) def self.resource_name @@ -33,16 +31,16 @@ def require_basic_authentication def process_incoming_event response_status = :unprocessable_entity - if event_params_sanity_check + if event_params_check if upp = user_project_preference - upp.activity_count = create_params[:count] + upp.legacy_count[create_params[:workflow]] = create_params[:count] response_status = :ok if upp.save end end render status: response_status, nothing: true end - def event_params_sanity_check + def event_params_check required_params && known_event? && legacy_zoo_project_exists? end @@ -85,7 +83,7 @@ def resource_sym def legacy_zoo_project @legacy_zoo_project ||= Project.where(migrated: true) - .where("configuration ->> 'zoo_home_project_id' = ?", create_params[:project_id]) + .where("configuration ->> 'zoo_home_project_id' = ?", create_params[:project_id].to_s) .first end diff --git a/app/serializers/user_project_preference_serializer.rb b/app/serializers/user_project_preference_serializer.rb index cd36119c8..4add4199e 100644 --- a/app/serializers/user_project_preference_serializer.rb +++ b/app/serializers/user_project_preference_serializer.rb @@ -8,7 +8,11 @@ def self.key end def activity_count - @model.activity_count || user_project_activity + if !@model.legacy_count.blank? + @model.legacy_count.values.reduce(:+) + else + @model.activity_count || user_project_activity + end end def user_project_activity diff --git a/db/migrate/20150817145756_add_legacy_count_to_project_preferences.rb b/db/migrate/20150817145756_add_legacy_count_to_project_preferences.rb new file mode 100644 index 000000000..4bbbda971 --- /dev/null +++ b/db/migrate/20150817145756_add_legacy_count_to_project_preferences.rb @@ -0,0 +1,5 @@ +class AddLegacyCountToProjectPreferences < ActiveRecord::Migration + def change + add_column :user_project_preferences, :legacy_count, :jsonb, default: {} + end +end diff --git a/db/schema.rb b/db/schema.rb index 2c84991eb..29d055bc4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150811202500) do +ActiveRecord::Schema.define(version: 20150817145756) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -298,6 +298,7 @@ t.datetime "created_at" t.datetime "updated_at" t.integer "activity_count" + t.jsonb "legacy_count", default: {} end create_table "user_seen_subjects", force: :cascade do |t| diff --git a/lib/zooniverse_user_subscription.rb b/lib/zooniverse_user_subscription.rb index 938bf54e4..bdca82d2e 100644 --- a/lib/zooniverse_user_subscription.rb +++ b/lib/zooniverse_user_subscription.rb @@ -11,65 +11,32 @@ class MissingLegacyProject < StandardError; end serialize :summary, Hash def self.import_zoo_user_subscriptions(user_logins=nil) - setup_caches - zoo_user_scope = zoo_users_to_import(user_logins) - total_batches = zoo_user_scope.count / ZOO_USER_BATCH_SIZE - zoo_user_scope.find_in_batches.with_index do |zoo_users, batch| - puts "Processing zoo user batch: #{ batch + 1} out of #{total_batches}" - import_batch(zoo_users) - end - end - - def self.setup_caches - @cached_projects = {} - @cached_users = {} + import_ids = zoo_users_to_import(user_logins) + subs = zoo_project_subscriptions(import_ids) + subs.find_each.with_index(&:import) end def self.zoo_users_to_import(user_logins) if user_logins.blank? - ZooniverseUser.all + nil else - ZooniverseUser.where(users: { login: user_logins }) - end.select(:id) - end - - def self.import_batch(zoo_users) - #reset the cache for each batch - @cached_users = {} - import_user_scope = zoo_project_subscriptions(zoo_users.map(&:id)) - total = import_user_scope.count.keys.count - import_user_scope.find_each.with_index do |user_project_sub, index| - puts "#{ index } / #{ total }" if index % 1_000 == 0 - unless user_project_sub.empty_summary? - user_project_sub.import - end + ZooniverseUser.where(users: { login: user_logins }).pluck(:id) end end - def self.cached_users - @cached_users - end - - def self.cached_projects - @cached_projects - end - def self.zoo_project_subscriptions(user_ids) - self.where.not(user_id: nil, project_id: nil) - includes(:zooniverse_user) - .joins(:zooniverse_user) - .where(users: { id: user_ids }) - .distinct - .group(:user_id, :project_id) + query = self.where.not(user_id: nil, project_id: nil) + query = query.where(user_id: user_ids) if user_ids + query end - def import + def import(index) return if user_id.nil? || empty_summary? zoo_project = find_legacy_migrated_project if panoptes_user = find_migrated_user migrated_upp = UserProjectPreference.find_or_initialize_by(project_id: zoo_project.id, user_id: panoptes_user.id) - migrated_upp.activity_count = summate_activity_counts + migrated_upp.legacy_count = summate_activity_counts migrated_upp.email_communication = (notifications || true) migrated_upp.save! if migrated_upp.changed? end @@ -82,41 +49,34 @@ def empty_summary? private def find_legacy_migrated_project - return self.class.cached_projects[project_id] if self.class.cached_projects.has_key?(project_id) legacy_project = Project.where(migrated: true) .where("configuration ->> 'zoo_home_project_id' = '#{project_id}'") .first unless legacy_project raise MissingLegacyProject.new("Legacy project missing make sure it has been migrated!") end - self.class.cached_projects[project_id] = legacy_project + legacy_project end def find_migrated_user - if self.class.cached_users.has_key?(zooniverse_user.id) - self.class.cached_users[zooniverse_user.id] - else - migrated_user = User.where(zooniverse_id: zooniverse_user.id) - .or(User.where(email: zooniverse_user.email)) - .first - unless migrated_user - p "Skipping subscription for non-migrated user account: #{zooniverse_user.login}" - end - self.class.cached_users[zooniverse_user.id] = migrated_user + unless migrated_user = User.find_by(zooniverse_id: user_id) + p "Skipping subscription for non-migrated user account: #{zooniverse_user.login}" end + migrated_user end # Summate all the activity counts for the user : project id combo # original DB has no validations and there are duplicates with different values def summate_activity_counts - activity_count = 0 - all_user_projects_subscriptions.find_each do |subscription| + all_user_projects_subscriptions.reduce({}) do |counts, subscription| next if subscription.empty_summary? - if count = subscription.summary.values.first[:count].to_i - activity_count += count + summary = subscription.summary + summary.each do |workflow, value| + counts[workflow] ||= 0 + counts[workflow] += value[:count].to_i end + counts end - activity_count end def all_user_projects_subscriptions diff --git a/spec/controllers/api/events_controller_spec.rb b/spec/controllers/api/events_controller_spec.rb index 7242bd330..dc0bc75f4 100644 --- a/spec/controllers/api/events_controller_spec.rb +++ b/spec/controllers/api/events_controller_spec.rb @@ -26,7 +26,7 @@ def overridden_params(new_params) user.update_column(:zooniverse_id, 1) user end - let(:event_count) { 10 } + let(:event_count) { {workflow.display_name => 10} } let(:created_at) { project.created_at.to_s } let(:event_kind) { "workflow_activity" } let(:event_params) do @@ -34,8 +34,9 @@ def overridden_params(new_params) event: { kind: event_kind, project_id: zoo_home_project_id, project: project.name, zooniverse_user_id: user.zooniverse_id, workflow: workflow.display_name, - count: event_count, created_at: created_at - } + count: event_count.values.first, created_at: created_at + }, + format: :json } end let(:user_project_pref) do @@ -205,9 +206,9 @@ def overridden_params(new_params) end.to change { UserProjectPreference.count }.from(0).to(1) end - it "should update the upp activity_count to the correct value" do + it "should update the upp legacy_count to the correct value" do post :create, event_params - expect(user_project_pref.activity_count).to eq(event_count) + expect(user_project_pref.legacy_count).to eq(event_count) end context "with an panoptes-formatted user zooniverse_id" do @@ -221,7 +222,7 @@ def overridden_params(new_params) context "when the user project preference already exists" do let!(:upp) do - create(:user_project_preference, project: project, user: user, activity_count: 100) + create(:user_project_preference, project: project, user: user, legacy_count: {workflow.display_name => 100}) end it "should update the model only" do @@ -230,9 +231,9 @@ def overridden_params(new_params) end.to_not change { UserProjectPreference.count }.from(1) end - it "should overwrite the upp activity_count to the correct value" do + it "should overwrite the upp legacy_count to the correct value" do post :create, event_params - expect(upp.reload.activity_count).to eq(event_count) + expect(upp.reload.legacy_count).to eq(event_count) end end end