change sidekiq queueing to bulk push (#3536)

This commit is contained in:
takayamaki 2017-06-04 07:11:15 +09:00 committed by Eugen Rochko
parent ce7c0def88
commit 3eedad2737
6 changed files with 23 additions and 9 deletions

View File

@ -52,6 +52,7 @@ gem 'sanitize', '~> 4.4'
gem 'sidekiq', '~> 5.0'
gem 'sidekiq-scheduler', '~> 2.1'
gem 'sidekiq-unique-jobs', '~> 5.0'
gem 'sidekiq-bulk', '~>0.1.1'
gem 'simple-navigation', '~> 4.0'
gem 'simple_form', '~> 3.4'
gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'

View File

@ -402,6 +402,9 @@ GEM
connection_pool (~> 2.2, >= 2.2.0)
rack-protection (>= 1.5.0)
redis (~> 3.3, >= 3.3.3)
sidekiq-bulk (0.1.1)
activesupport
sidekiq
sidekiq-scheduler (2.1.4)
redis (~> 3)
rufus-scheduler (~> 3.2)
@ -543,6 +546,7 @@ DEPENDENCIES
sanitize (~> 4.4)
scss_lint (~> 0.53)
sidekiq (~> 5.0)
sidekiq-bulk (~> 0.1.1)
sidekiq-scheduler (~> 2.1)
sidekiq-unique-jobs (~> 5.0)
simple-navigation (~> 4.0)

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class Settings::FollowerDomainsController < ApplicationController
layout 'admin'
@ -13,8 +15,8 @@ class Settings::FollowerDomainsController < ApplicationController
def update
domains = bulk_params[:select] || []
domains.each do |domain|
SoftBlockDomainFollowersWorker.perform_async(current_account.id, domain)
SoftBlockDomainFollowersWorker.push_bulk(domains) do |domain|
[current_account.id, domain]
end
redirect_to settings_follower_domains_path, notice: I18n.t('followers.success', count: domains.size)

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class FanOutOnWriteService < BaseService
# Push a status into home and mentions feeds
# @param [Status] status
@ -34,8 +36,10 @@ class FanOutOnWriteService < BaseService
def deliver_to_followers(status)
Rails.logger.debug "Delivering status #{status.id} to followers"
status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_each do |follower|
FeedInsertWorker.perform_async(status.id, follower.id)
status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers|
FeedInsertWorker.push_bulk(followers) do |follower|
[status.id, follower.id]
end
end
end

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
require 'sidekiq-scheduler'
require 'sidekiq-bulk'
class Scheduler::SubscriptionsScheduler
include Sidekiq::Worker
@ -7,9 +9,7 @@ class Scheduler::SubscriptionsScheduler
def perform
logger.info 'Queueing PuSH re-subscriptions'
expiring_accounts.pluck(:id).each do |id|
Pubsubhubbub::SubscribeWorker.perform_async(id)
end
Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
end
private

View File

@ -1,13 +1,16 @@
# frozen_string_literal: true
require 'sidekiq-bulk'
class SoftBlockDomainFollowersWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(account_id, domain)
Account.find(account_id).followers.where(domain: domain).pluck(:id).each do |follower_id|
SoftBlockWorker.perform_async(account_id, follower_id)
followers_id = Account.find(account_id).followers.where(domain: domain).pluck(:id)
SoftBlockWorker.push_bulk(followers_id) do |follower_id|
[account_id, follower_id]
end
end
end