Skip to content

Commit

Permalink
Merge pull request #244 from mashirozx/dev
Browse files Browse the repository at this point in the history
merge upstream (bump version to 3.2.1)
  • Loading branch information
mashirozx authored Oct 26, 2020
2 parents be70c76 + dd7560e commit a1632c7
Show file tree
Hide file tree
Showing 28 changed files with 995 additions and 517 deletions.
2 changes: 1 addition & 1 deletion .codeclimate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ plugins:
channel: eslint-7
rubocop:
enabled: true
channel: rubocop-0-88
channel: rubocop-0-92
sass-lint:
enabled: true
exclude_patterns:
Expand Down
4 changes: 3 additions & 1 deletion Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ group :test do
end

group :development do
gem 'active_record_query_trace', '~> 1.7'
gem 'active_record_query_trace', '~> 1.8'
gem 'annotate', '~> 3.1'
gem 'better_errors', '~> 2.8'
gem 'binding_of_caller', '~> 0.7'
Expand Down Expand Up @@ -164,3 +164,5 @@ gem 'concurrent-ruby', require: false
gem 'connection_pool', require: false

gem "sidekiq-statistic", "~> 1.4"
gem 'xorcist', '~> 1.1'
gem 'pluck_each', '~> 0.1.3'
10 changes: 8 additions & 2 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ GEM
activemodel (>= 4.1, < 6.1)
case_transform (>= 0.2)
jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
active_record_query_trace (1.7)
active_record_query_trace (1.8)
activejob (5.2.4.4)
activesupport (= 5.2.4.4)
globalid (>= 0.3.6)
Expand Down Expand Up @@ -407,6 +407,9 @@ GEM
pghero (2.7.2)
activerecord (>= 5)
pkg-config (1.4.4)
pluck_each (0.1.3)
activerecord (> 3.2.0)
activesupport (> 3.0.0)
posix-spawn (0.3.15)
premailer (1.14.2)
addressable
Expand Down Expand Up @@ -670,6 +673,7 @@ GEM
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
wisper (2.0.1)
xorcist (1.1.2)
xpath (3.2.0)
nokogiri (~> 1.8)

Expand All @@ -678,7 +682,7 @@ PLATFORMS

DEPENDENCIES
active_model_serializers (~> 0.10)
active_record_query_trace (~> 1.7)
active_record_query_trace (~> 1.8)
addressable (~> 2.7)
annotate (~> 3.1)
aws-sdk-s3 (~> 1.83)
Expand Down Expand Up @@ -758,6 +762,7 @@ DEPENDENCIES
pg (~> 1.2)
pghero (~> 2.7)
pkg-config (~> 1.4)
pluck_each (~> 0.1.3)
posix-spawn
premailer-rails
private_address_check (~> 0.5)
Expand Down Expand Up @@ -808,6 +813,7 @@ DEPENDENCIES
webmock (~> 3.9)
webpacker (~> 5.2)
webpush
xorcist (~> 1.1)

RUBY VERSION
ruby 2.6.6p146
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# frozen_string_literal: true

class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseController
include SignatureVerification
include AccountOwnedConcern

before_action :require_signature!
before_action :set_items
before_action :set_cache_headers

def show
expires_in 0, public: false
render json: collection_presenter,
serializer: ActivityPub::CollectionSerializer,
adapter: ActivityPub::Adapter,
content_type: 'application/activity+json'
end

private

def uri_prefix
signed_request_account.uri[/http(s?):\/\/[^\/]+\//]
end

def set_items
@items = @account.followers.where(Account.arel_table[:uri].matches(uri_prefix + '%', false, true)).pluck(:uri)
end

def collection_presenter
ActivityPub::CollectionPresenter.new(
id: account_followers_synchronization_url(@account),
type: :ordered,
items: @items
)
end
end
14 changes: 14 additions & 0 deletions app/controllers/activitypub/inboxes_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController

def create
upgrade_account
process_collection_synchronization
process_payload
head 202
end
Expand Down Expand Up @@ -52,6 +53,19 @@ def upgrade_account
DeliveryFailureTracker.reset!(signed_request_account.inbox_url)
end

def process_collection_synchronization
raw_params = request.headers['Collection-Synchronization']
return if raw_params.blank? || ENV['DISABLE_FOLLOWERS_SYNCHRONIZATION'] == 'true'

# Re-using the syntax for signature parameters
tree = SignatureParamsParser.new.parse(raw_params)
params = SignatureParamsTransformer.new.apply(tree)

ActivityPub::PrepareFollowersSynchronizationService.new.call(signed_request_account, params)
rescue Parslet::ParseFailed
Rails.logger.warn 'Error parsing Collection-Synchronization header'
end

def process_payload
ActivityPub::ProcessingWorker.perform_async(signed_request_account.id, body, @account&.id)
end
Expand Down
4 changes: 4 additions & 0 deletions app/lib/activitypub/tag_manager.rb
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ def uri_for(target)
end
end

def uri_for_username(username)
account_url(username: username)
end

def generate_uri_for(_target)
URI.join(root_url, 'payloads', SecureRandom.uuid)
end
Expand Down
1 change: 1 addition & 0 deletions app/lib/sanitize_config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ module Config
gopher
xmpp
magnet
gemini
).freeze

CLASS_WHITELIST_TRANSFORMER = lambda do |env|
Expand Down
2 changes: 0 additions & 2 deletions app/lib/settings/scoped_settings.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ def initialize(object)
@object = object
end

# rubocop:disable Style/MethodMissingSuper
def method_missing(method, *args)
method_name = method.to_s
# set a value for a variable
Expand All @@ -24,7 +23,6 @@ def method_missing(method, *args)
self[method_name]
end
end
# rubocop:enable Style/MethodMissingSuper

def respond_to_missing?(*)
true
Expand Down
6 changes: 6 additions & 0 deletions app/models/account.rb
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,12 @@ def preferred_inbox_url
shared_inbox_url.presence || inbox_url
end

def synchronization_uri_prefix
return 'local' if local?

@synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
end

class Field < ActiveModelSerializers::Model
attributes :name, :value, :verified_at, :account, :errors

Expand Down
20 changes: 20 additions & 0 deletions app/models/concerns/account_interactions.rb
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,26 @@ def lists_for_local_distribution
.where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
end

def remote_followers_hash(url_prefix)
Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
digest = "\x00" * 32
followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
Xorcist.xor!(digest, Digest::SHA256.digest(uri))
end
digest.unpack('H*')[0]
end
end

def local_followers_hash
Rails.cache.fetch("followers_hash:#{id}:local") do
digest = "\x00" * 32
followers.where(domain: nil).pluck_each(:username) do |username|
Xorcist.xor!(digest, Digest::SHA256.digest(ActivityPub::TagManager.instance.uri_for_username(username)))
end
digest.unpack('H*')[0]
end
end

private

def remove_potential_friendship(other_account, mutual = false)
Expand Down
8 changes: 8 additions & 0 deletions app/models/follow.rb
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,10 @@ def revoke_request!

before_validation :set_uri, only: :create
after_create :increment_cache_counters
after_create :invalidate_hash_cache
after_destroy :remove_endorsements
after_destroy :decrement_cache_counters
after_destroy :invalidate_hash_cache

private

Expand All @@ -63,4 +65,10 @@ def decrement_cache_counters
account&.decrement_count!(:following_count)
target_account&.decrement_count!(:followers_count)
end

def invalidate_hash_cache
return if account.local? && target_account.local?

Rails.cache.delete("followers_hash:#{target_account_id}:#{account.synchronization_uri_prefix}")
end
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# frozen_string_literal: true

class ActivityPub::PrepareFollowersSynchronizationService < BaseService
include JsonLdHelper

def call(account, params)
@account = account

return if params['collectionId'] != @account.followers_url || invalid_origin?(params['url']) || @account.local_followers_hash == params['digest']

ActivityPub::FollowersSynchronizationWorker.perform_async(@account.id, params['url'])
end
end
2 changes: 1 addition & 1 deletion app/services/activitypub/process_account_service.rb
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ def collection_info(type)
total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
has_first_page = collection.is_a?(Hash) && collection['first'].present?
@collections[type] = [total_items, has_first_page]
rescue HTTP::Error, OpenSSL::SSL::SSLError
rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::LengthValidationError
@collections[type] = [nil, nil]
end

Expand Down
74 changes: 74 additions & 0 deletions app/services/activitypub/synchronize_followers_service.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# frozen_string_literal: true

class ActivityPub::SynchronizeFollowersService < BaseService
include JsonLdHelper
include Payloadable

def call(account, partial_collection_url)
@account = account

items = collection_items(partial_collection_url)
return if items.nil?

# There could be unresolved accounts (hence the call to .compact) but this
# should never happen in practice, since in almost all cases we keep an
# Account record, and should we not do that, we should have sent a Delete.
# In any case there is not much we can do if that occurs.
@expected_followers = items.map { |uri| ActivityPub::TagManager.instance.uri_to_resource(uri, Account) }.compact

remove_unexpected_local_followers!
handle_unexpected_outgoing_follows!
end

private

def remove_unexpected_local_followers!
@account.followers.local.where.not(id: @expected_followers.map(&:id)).each do |unexpected_follower|
UnfollowService.new.call(unexpected_follower, @account)
end
end

def handle_unexpected_outgoing_follows!
@expected_followers.each do |expected_follower|
next if expected_follower.following?(@account)

if expected_follower.requested?(@account)
# For some reason the follow request went through but we missed it
expected_follower.follow_requests.find_by(target_account: @account)&.authorize!
else
# Since we were not aware of the follow from our side, we do not have an
# ID for it that we can include in the Undo activity. For this reason,
# the Undo may not work with software that relies exclusively on
# matching activity IDs and not the actor and target
follow = Follow.new(account: expected_follower, target_account: @account)
ActivityPub::DeliveryWorker.perform_async(build_undo_follow_json(follow), follow.account_id, follow.target_account.inbox_url)
end
end
end

def build_undo_follow_json(follow)
Oj.dump(serialize_payload(follow, ActivityPub::UndoFollowSerializer))
end

def collection_items(collection_or_uri)
collection = fetch_collection(collection_or_uri)
return unless collection.is_a?(Hash)

collection = fetch_collection(collection['first']) if collection['first'].present?
return unless collection.is_a?(Hash)

case collection['type']
when 'Collection', 'CollectionPage'
collection['items']
when 'OrderedCollection', 'OrderedCollectionPage'
collection['orderedItems']
end
end

def fetch_collection(collection_or_uri)
return collection_or_uri if collection_or_uri.is_a?(Hash)
return if invalid_origin?(collection_or_uri)

fetch_resource_without_id_validation(collection_or_uri, nil, true)
end
end
2 changes: 1 addition & 1 deletion app/services/process_mentions_service.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def create_notification(mention)
if mentioned_account.local?
LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name, :mention)
elsif mentioned_account.activitypub? && !@status.local_only?
ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url, { synchronize_followers: !mention.status.distributable? })
end
end

Expand Down
10 changes: 10 additions & 0 deletions app/workers/activitypub/delivery_worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

class ActivityPub::DeliveryWorker
include Sidekiq::Worker
include RoutingHelper
include JsonLdHelper

STOPLIGHT_FAILURE_THRESHOLD = 10
Expand Down Expand Up @@ -38,9 +39,18 @@ def build_request(http_client)
Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request|
request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
request.add_headers(HEADERS)
request.add_headers({ 'Collection-Synchronization' => synchronization_header }) if ENV['DISABLE_FOLLOWERS_SYNCHRONIZATION'] != 'true' && @options[:synchronize_followers]
end
end

def synchronization_header
"collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
end

def inbox_url_prefix
@inbox_url[/http(s?):\/\/[^\/]+\//]
end

def perform_request
light = Stoplight(@inbox_url) do
request_pool.with(@host) do |http_client|
Expand Down
2 changes: 1 addition & 1 deletion app/workers/activitypub/distribution_worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def perform(status_id)
return if skip_distribution?

ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
[payload, @account.id, inbox_url]
[payload, @account.id, inbox_url, { synchronize_followers: !@status.distributable? }]
end

relay! if relayable?
Expand Down
14 changes: 14 additions & 0 deletions app/workers/activitypub/followers_synchronization_worker.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# frozen_string_literal: true

class ActivityPub::FollowersSynchronizationWorker
include Sidekiq::Worker

sidekiq_options queue: 'push', lock: :until_executed

def perform(account_id, url)
@account = Account.find_by(id: account_id)
return true if @account.nil?

ActivityPub::SynchronizeFollowersService.new.call(@account, url)
end
end
Loading

0 comments on commit a1632c7

Please sign in to comment.