mirror of
https://framagit.org/tykayn/mastodon.git
synced 2023-08-25 08:33:12 +02:00
Add trending statuses (#17431)
* Add trending statuses * Fix dangling items with stale scores in localized sets * Various fixes and improvements - Change approve_all/reject_all to approve_accounts/reject_accounts - Change Trends::Query methods to not mutate the original query - Change Trends::Query#skip to offset - Change follow recommendations to be refreshed in a transaction * Add tests for trending statuses filtering behaviour * Fix not applying filtering scope in controller
This commit is contained in:
parent
a29a982eaa
commit
27965ce5ed
@ -32,10 +32,11 @@ Layout/EmptyLineAfterGuardClause:
|
||||
Layout/EmptyLinesAroundAttributeAccessor:
|
||||
Enabled: true
|
||||
|
||||
Layout/FirstHashElementIndentation:
|
||||
EnforcedStyle: consistent
|
||||
|
||||
Layout/HashAlignment:
|
||||
Enabled: false
|
||||
# EnforcedHashRocketStyle: table
|
||||
# EnforcedColonStyle: table
|
||||
|
||||
Layout/SpaceAroundMethodCallOperator:
|
||||
Enabled: true
|
||||
|
@ -5,11 +5,11 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll
|
||||
authorize :preview_card_provider, :index?
|
||||
|
||||
@preview_card_providers = filtered_preview_card_providers.page(params[:page])
|
||||
@form = Form::PreviewCardProviderBatch.new
|
||||
@form = Trends::PreviewCardProviderBatch.new
|
||||
end
|
||||
|
||||
def batch
|
||||
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form = Trends::PreviewCardProviderBatch.new(trends_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form.save
|
||||
rescue ActionController::ParameterMissing
|
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
|
||||
@ -20,15 +20,15 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll
|
||||
private
|
||||
|
||||
def filtered_preview_card_providers
|
||||
PreviewCardProviderFilter.new(filter_params).results
|
||||
Trends::PreviewCardProviderFilter.new(filter_params).results
|
||||
end
|
||||
|
||||
def filter_params
|
||||
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
|
||||
params.slice(:page, *Trends::PreviewCardProviderFilter::KEYS).permit(:page, *Trends::PreviewCardProviderFilter::KEYS)
|
||||
end
|
||||
|
||||
def form_preview_card_provider_batch_params
|
||||
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
|
||||
def trends_preview_card_provider_batch_params
|
||||
params.require(:trends_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
|
||||
end
|
||||
|
||||
def action_from_button
|
||||
|
@ -5,11 +5,11 @@ class Admin::Trends::LinksController < Admin::BaseController
|
||||
authorize :preview_card, :index?
|
||||
|
||||
@preview_cards = filtered_preview_cards.page(params[:page])
|
||||
@form = Form::PreviewCardBatch.new
|
||||
@form = Trends::PreviewCardBatch.new
|
||||
end
|
||||
|
||||
def batch
|
||||
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form = Trends::PreviewCardBatch.new(trends_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form.save
|
||||
rescue ActionController::ParameterMissing
|
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
|
||||
@ -20,26 +20,26 @@ class Admin::Trends::LinksController < Admin::BaseController
|
||||
private
|
||||
|
||||
def filtered_preview_cards
|
||||
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
|
||||
Trends::PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
|
||||
end
|
||||
|
||||
def filter_params
|
||||
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
|
||||
params.slice(:page, *Trends::PreviewCardFilter::KEYS).permit(:page, *Trends::PreviewCardFilter::KEYS)
|
||||
end
|
||||
|
||||
def form_preview_card_batch_params
|
||||
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
|
||||
def trends_preview_card_batch_params
|
||||
params.require(:trends_preview_card_batch).permit(:action, preview_card_ids: [])
|
||||
end
|
||||
|
||||
def action_from_button
|
||||
if params[:approve]
|
||||
'approve'
|
||||
elsif params[:approve_all]
|
||||
'approve_all'
|
||||
elsif params[:approve_providers]
|
||||
'approve_providers'
|
||||
elsif params[:reject]
|
||||
'reject'
|
||||
elsif params[:reject_all]
|
||||
'reject_all'
|
||||
elsif params[:reject_providers]
|
||||
'reject_providers'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
45
app/controllers/admin/trends/statuses_controller.rb
Normal file
45
app/controllers/admin/trends/statuses_controller.rb
Normal file
@ -0,0 +1,45 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Admin::Trends::StatusesController < Admin::BaseController
|
||||
def index
|
||||
authorize :status, :index?
|
||||
|
||||
@statuses = filtered_statuses.page(params[:page])
|
||||
@form = Trends::StatusBatch.new
|
||||
end
|
||||
|
||||
def batch
|
||||
@form = Trends::StatusBatch.new(trends_status_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form.save
|
||||
rescue ActionController::ParameterMissing
|
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
|
||||
ensure
|
||||
redirect_to admin_trends_statuses_path(filter_params)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def filtered_statuses
|
||||
Trends::StatusFilter.new(filter_params.with_defaults(trending: 'all')).results.includes(:account, :media_attachments, :active_mentions)
|
||||
end
|
||||
|
||||
def filter_params
|
||||
params.slice(:page, *Trends::StatusFilter::KEYS).permit(:page, *Trends::StatusFilter::KEYS)
|
||||
end
|
||||
|
||||
def trends_status_batch_params
|
||||
params.require(:trends_status_batch).permit(:action, status_ids: [])
|
||||
end
|
||||
|
||||
def action_from_button
|
||||
if params[:approve]
|
||||
'approve'
|
||||
elsif params[:approve_accounts]
|
||||
'approve_accounts'
|
||||
elsif params[:reject]
|
||||
'reject'
|
||||
elsif params[:reject_accounts]
|
||||
'reject_accounts'
|
||||
end
|
||||
end
|
||||
end
|
@ -5,11 +5,11 @@ class Admin::Trends::TagsController < Admin::BaseController
|
||||
authorize :tag, :index?
|
||||
|
||||
@tags = filtered_tags.page(params[:page])
|
||||
@form = Form::TagBatch.new
|
||||
@form = Trends::TagBatch.new
|
||||
end
|
||||
|
||||
def batch
|
||||
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form = Trends::TagBatch.new(trends_tag_batch_params.merge(current_account: current_account, action: action_from_button))
|
||||
@form.save
|
||||
rescue ActionController::ParameterMissing
|
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
|
||||
@ -20,15 +20,15 @@ class Admin::Trends::TagsController < Admin::BaseController
|
||||
private
|
||||
|
||||
def filtered_tags
|
||||
TagFilter.new(filter_params).results
|
||||
Trends::TagFilter.new(filter_params).results
|
||||
end
|
||||
|
||||
def filter_params
|
||||
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
|
||||
params.slice(:page, *Trends::TagFilter::KEYS).permit(:page, *Trends::TagFilter::KEYS)
|
||||
end
|
||||
|
||||
def form_tag_batch_params
|
||||
params.require(:form_tag_batch).permit(:action, tag_ids: [])
|
||||
def trends_tag_batch_params
|
||||
params.require(:trends_tag_batch).permit(:action, tag_ids: [])
|
||||
end
|
||||
|
||||
def action_from_button
|
||||
|
19
app/controllers/api/v1/admin/trends/links_controller.rb
Normal file
19
app/controllers/api/v1/admin/trends/links_controller.rb
Normal file
@ -0,0 +1,19 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Api::V1::Admin::Trends::LinksController < Api::BaseController
|
||||
protect_from_forgery with: :exception
|
||||
|
||||
before_action -> { authorize_if_got_token! :'admin:read' }
|
||||
before_action :require_staff!
|
||||
before_action :set_links
|
||||
|
||||
def index
|
||||
render json: @links, each_serializer: REST::Trends::LinkSerializer
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_links
|
||||
@links = Trends.links.query.limit(limit_param(10))
|
||||
end
|
||||
end
|
19
app/controllers/api/v1/admin/trends/statuses_controller.rb
Normal file
19
app/controllers/api/v1/admin/trends/statuses_controller.rb
Normal file
@ -0,0 +1,19 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Api::V1::Admin::Trends::StatusesController < Api::BaseController
|
||||
protect_from_forgery with: :exception
|
||||
|
||||
before_action -> { authorize_if_got_token! :'admin:read' }
|
||||
before_action :require_staff!
|
||||
before_action :set_statuses
|
||||
|
||||
def index
|
||||
render json: @statuses, each_serializer: REST::StatusSerializer
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_statuses
|
||||
@statuses = cache_collection(Trends.statuses.query.limit(limit_param(DEFAULT_STATUSES_LIMIT)), Status)
|
||||
end
|
||||
end
|
@ -14,6 +14,6 @@ class Api::V1::Admin::Trends::TagsController < Api::BaseController
|
||||
private
|
||||
|
||||
def set_tags
|
||||
@tags = Trends.tags.get(false, limit_param(10))
|
||||
@tags = Trends.tags.query.limit(limit_param(10))
|
||||
end
|
||||
end
|
||||
|
@ -12,10 +12,14 @@ class Api::V1::Trends::LinksController < Api::BaseController
|
||||
def set_links
|
||||
@links = begin
|
||||
if Setting.trends
|
||||
Trends.links.get(true, limit_param(10))
|
||||
links_from_trends
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def links_from_trends
|
||||
Trends.links.query.allowed.in_locale(content_locale).limit(limit_param(10))
|
||||
end
|
||||
end
|
||||
|
27
app/controllers/api/v1/trends/statuses_controller.rb
Normal file
27
app/controllers/api/v1/trends/statuses_controller.rb
Normal file
@ -0,0 +1,27 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Api::V1::Trends::StatusesController < Api::BaseController
|
||||
before_action :set_statuses
|
||||
|
||||
def index
|
||||
render json: @statuses, each_serializer: REST::StatusSerializer
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_statuses
|
||||
@statuses = begin
|
||||
if Setting.trends
|
||||
cache_collection(statuses_from_trends, Status)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def statuses_from_trends
|
||||
scope = Trends.statuses.query.allowed.in_locale(content_locale)
|
||||
scope = scope.filtered_for(current_account) if user_signed_in?
|
||||
scope.limit(limit_param(DEFAULT_STATUSES_LIMIT))
|
||||
end
|
||||
end
|
@ -12,7 +12,7 @@ class Api::V1::Trends::TagsController < Api::BaseController
|
||||
def set_tags
|
||||
@tags = begin
|
||||
if Setting.trends
|
||||
Trends.tags.get(true, limit_param(10))
|
||||
Trends.tags.query.allowed.limit(limit_param(10))
|
||||
else
|
||||
[]
|
||||
end
|
||||
|
@ -27,4 +27,8 @@ module Localized
|
||||
def available_locale_or_nil(locale_name)
|
||||
locale_name.to_sym if locale_name.present? && I18n.available_locales.include?(locale_name.to_sym)
|
||||
end
|
||||
|
||||
def content_locale
|
||||
@content_locale ||= I18n.locale.to_s.split(/[_-]/).first
|
||||
end
|
||||
end
|
||||
|
@ -5,9 +5,10 @@ module Admin::FilterHelper
|
||||
AccountFilter::KEYS,
|
||||
CustomEmojiFilter::KEYS,
|
||||
ReportFilter::KEYS,
|
||||
TagFilter::KEYS,
|
||||
PreviewCardProviderFilter::KEYS,
|
||||
PreviewCardFilter::KEYS,
|
||||
Trends::TagFilter::KEYS,
|
||||
Trends::PreviewCardProviderFilter::KEYS,
|
||||
Trends::PreviewCardFilter::KEYS,
|
||||
Trends::StatusFilter::KEYS,
|
||||
InstanceFilter::KEYS,
|
||||
InviteFilter::KEYS,
|
||||
RelationshipFilter::KEYS,
|
||||
|
@ -242,6 +242,6 @@ module LanguagesHelper
|
||||
end
|
||||
|
||||
def valid_locale?(locale)
|
||||
SUPPORTED_LOCALES.key?(locale.to_sym)
|
||||
locale.present? && SUPPORTED_LOCALES.key?(locale.to_sym)
|
||||
end
|
||||
end
|
||||
|
@ -331,7 +331,8 @@
|
||||
}
|
||||
|
||||
.batch-table__row--muted .pending-account__header,
|
||||
.batch-table__row--muted .accounts-table {
|
||||
.batch-table__row--muted .accounts-table,
|
||||
.batch-table__row--muted .name-tag {
|
||||
&,
|
||||
a,
|
||||
strong {
|
||||
@ -339,6 +340,10 @@
|
||||
}
|
||||
}
|
||||
|
||||
.batch-table__row--muted .name-tag .avatar {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.batch-table__row--muted .accounts-table {
|
||||
tbody td.accounts-table__extra,
|
||||
&__count,
|
||||
@ -352,7 +357,8 @@
|
||||
}
|
||||
|
||||
.batch-table__row--attention .pending-account__header,
|
||||
.batch-table__row--attention .accounts-table {
|
||||
.batch-table__row--attention .accounts-table,
|
||||
.batch-table__row--attention .name-tag {
|
||||
&,
|
||||
a,
|
||||
strong {
|
||||
|
@ -210,6 +210,7 @@ a.table-action-link {
|
||||
&__content {
|
||||
padding-top: 12px;
|
||||
padding-bottom: 16px;
|
||||
overflow: hidden;
|
||||
|
||||
&--unpadded {
|
||||
padding: 0;
|
||||
@ -296,3 +297,9 @@ a.table-action-link {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.one-liner {
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
@ -23,8 +23,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
|
||||
visibility: visibility_from_audience
|
||||
)
|
||||
|
||||
Trends.tags.register(@status)
|
||||
Trends.links.register(@status)
|
||||
Trends.register!(@status)
|
||||
|
||||
distribute
|
||||
end
|
||||
|
@ -7,6 +7,8 @@ class ActivityPub::Activity::Like < ActivityPub::Activity
|
||||
return if original_status.nil? || !original_status.account.local? || delete_arrived_first?(@json['id']) || @account.favourited?(original_status)
|
||||
|
||||
favourite = original_status.favourites.create!(account: @account)
|
||||
|
||||
NotifyService.new.call(original_status.account, :favourite, favourite)
|
||||
Trends.statuses.register(original_status)
|
||||
end
|
||||
end
|
||||
|
@ -35,25 +35,18 @@ class AdminMailer < ApplicationMailer
|
||||
end
|
||||
end
|
||||
|
||||
def new_trending_tags(recipient, tags)
|
||||
@tags = tags
|
||||
@me = recipient
|
||||
@instance = Rails.configuration.x.local_domain
|
||||
@lowest_trending_tag = Trends.tags.get(true, Trends.tags.options[:review_threshold]).last
|
||||
def new_trends(recipient, links, tags, statuses)
|
||||
@links = links
|
||||
@lowest_trending_link = Trends.links.query.allowed.limit(Trends.links.options[:review_threshold]).last
|
||||
@tags = tags
|
||||
@lowest_trending_tag = Trends.tags.query.allowed.limit(Trends.tags.options[:review_threshold]).last
|
||||
@statuses = statuses
|
||||
@lowest_trending_status = Trends.statuses.query.allowed.limit(Trends.statuses.options[:review_threshold]).last
|
||||
@me = recipient
|
||||
@instance = Rails.configuration.x.local_domain
|
||||
|
||||
locale_for_account(@me) do
|
||||
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
|
||||
end
|
||||
end
|
||||
|
||||
def new_trending_links(recipient, links)
|
||||
@links = links
|
||||
@me = recipient
|
||||
@instance = Rails.configuration.x.local_domain
|
||||
@lowest_trending_link = Trends.links.get(true, Trends.links.options[:review_threshold]).last
|
||||
|
||||
locale_for_account(@me) do
|
||||
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
|
||||
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -40,13 +40,15 @@
|
||||
# also_known_as :string is an Array
|
||||
# silenced_at :datetime
|
||||
# suspended_at :datetime
|
||||
# trust_level :integer
|
||||
# hide_collections :boolean
|
||||
# avatar_storage_schema_version :integer
|
||||
# header_storage_schema_version :integer
|
||||
# devices_url :string
|
||||
# suspension_origin :integer
|
||||
# sensitized_at :datetime
|
||||
# trendable :boolean
|
||||
# reviewed_at :datetime
|
||||
# requested_review_at :datetime
|
||||
#
|
||||
|
||||
class Account < ApplicationRecord
|
||||
@ -56,6 +58,7 @@ class Account < ApplicationRecord
|
||||
remote_url
|
||||
salmon_url
|
||||
hub_url
|
||||
trust_level
|
||||
)
|
||||
|
||||
USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
|
||||
@ -74,11 +77,6 @@ class Account < ApplicationRecord
|
||||
include DomainMaterializable
|
||||
include AccountMerging
|
||||
|
||||
TRUST_LEVELS = {
|
||||
untrusted: 0,
|
||||
trusted: 1,
|
||||
}.freeze
|
||||
|
||||
enum protocol: [:ostatus, :activitypub]
|
||||
enum suspension_origin: [:local, :remote], _prefix: true
|
||||
|
||||
@ -202,10 +200,6 @@ class Account < ApplicationRecord
|
||||
last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
|
||||
end
|
||||
|
||||
def trust_level
|
||||
self[:trust_level] || 0
|
||||
end
|
||||
|
||||
def refresh!
|
||||
ResolveAccountService.new.call(acct) unless local?
|
||||
end
|
||||
@ -388,6 +382,22 @@ class Account < ApplicationRecord
|
||||
@synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
|
||||
end
|
||||
|
||||
def requires_review?
|
||||
reviewed_at.nil?
|
||||
end
|
||||
|
||||
def reviewed?
|
||||
reviewed_at.present?
|
||||
end
|
||||
|
||||
def requested_review?
|
||||
requested_review_at.present?
|
||||
end
|
||||
|
||||
def requires_review_notification?
|
||||
requires_review? && !requested_review?
|
||||
end
|
||||
|
||||
class Field < ActiveModelSerializers::Model
|
||||
attributes :name, :value, :verified_at, :account
|
||||
|
||||
|
@ -268,6 +268,18 @@ class Status < ApplicationRecord
|
||||
update_status_stat!(key => [public_send(key) - 1, 0].max)
|
||||
end
|
||||
|
||||
def trendable?
|
||||
if attributes['trendable'].nil?
|
||||
account.trendable?
|
||||
else
|
||||
attributes['trendable']
|
||||
end
|
||||
end
|
||||
|
||||
def requires_review_notification?
|
||||
attributes['trendable'].nil? && account.requires_review_notification?
|
||||
end
|
||||
|
||||
after_create_commit :increment_counter_caches
|
||||
after_destroy_commit :decrement_counter_caches
|
||||
|
||||
|
@ -13,15 +13,37 @@ module Trends
|
||||
@tags ||= Trends::Tags.new
|
||||
end
|
||||
|
||||
def self.statuses
|
||||
@statuses ||= Trends::Statuses.new
|
||||
end
|
||||
|
||||
def self.register!(status)
|
||||
[links, tags, statuses].each { |trend_type| trend_type.register(status) }
|
||||
end
|
||||
|
||||
def self.refresh!
|
||||
[links, tags].each(&:refresh)
|
||||
[links, tags, statuses].each(&:refresh)
|
||||
end
|
||||
|
||||
def self.request_review!
|
||||
[links, tags].each(&:request_review) if enabled?
|
||||
return unless enabled?
|
||||
|
||||
links_requiring_review = links.request_review
|
||||
tags_requiring_review = tags.request_review
|
||||
statuses_requiring_review = statuses.request_review
|
||||
|
||||
return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty?
|
||||
|
||||
User.staff.includes(:account).find_each do |user|
|
||||
AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails?
|
||||
end
|
||||
end
|
||||
|
||||
def self.enabled?
|
||||
Setting.trends
|
||||
end
|
||||
|
||||
def self.available_locales
|
||||
@available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq
|
||||
end
|
||||
end
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
class Trends::Base
|
||||
include Redisable
|
||||
include LanguagesHelper
|
||||
|
||||
class_attribute :default_options
|
||||
|
||||
@ -32,8 +33,8 @@ class Trends::Base
|
||||
raise NotImplementedError
|
||||
end
|
||||
|
||||
def get(*)
|
||||
raise NotImplementedError
|
||||
def query
|
||||
Trends::Query.new(key_prefix, klass)
|
||||
end
|
||||
|
||||
def score(id)
|
||||
@ -72,6 +73,21 @@ class Trends::Base
|
||||
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
|
||||
end
|
||||
|
||||
# @param [Integer] id
|
||||
# @param [Float] score
|
||||
# @param [Hash<String, Boolean>] subsets
|
||||
def add_to_and_remove_from_subsets(id, score, subsets = {})
|
||||
subsets.each_key do |subset|
|
||||
key = [key_prefix, subset].compact.join(':')
|
||||
|
||||
if score.positive? && subsets[subset]
|
||||
redis.zadd(key, score, id)
|
||||
else
|
||||
redis.zrem(key, id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def used_key(at_time)
|
||||
|
@ -4,8 +4,8 @@ class Trends::Links < Trends::Base
|
||||
PREFIX = 'trending_links'
|
||||
|
||||
self.default_options = {
|
||||
threshold: 15,
|
||||
review_threshold: 10,
|
||||
threshold: 5,
|
||||
review_threshold: 3,
|
||||
max_score_cooldown: 2.days.freeze,
|
||||
max_score_halflife: 8.hours.freeze,
|
||||
}
|
||||
@ -27,12 +27,6 @@ class Trends::Links < Trends::Base
|
||||
record_used_id(preview_card.id, at_time)
|
||||
end
|
||||
|
||||
def get(allowed, limit)
|
||||
preview_card_ids = currently_trending_ids(allowed, limit)
|
||||
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
|
||||
preview_card_ids.map { |id| preview_cards[id] }.compact
|
||||
end
|
||||
|
||||
def refresh(at_time = Time.now.utc)
|
||||
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
|
||||
calculate_scores(preview_cards, at_time)
|
||||
@ -42,7 +36,7 @@ class Trends::Links < Trends::Base
|
||||
def request_review
|
||||
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
|
||||
|
||||
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
|
||||
preview_cards.filter_map do |preview_card|
|
||||
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
|
||||
|
||||
if preview_card.provider.nil?
|
||||
@ -53,12 +47,6 @@ class Trends::Links < Trends::Base
|
||||
|
||||
preview_card
|
||||
end
|
||||
|
||||
return if preview_cards_requiring_review.empty?
|
||||
|
||||
User.staff.includes(:account).find_each do |user|
|
||||
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
@ -67,6 +55,10 @@ class Trends::Links < Trends::Base
|
||||
PREFIX
|
||||
end
|
||||
|
||||
def klass
|
||||
PreviewCard
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def calculate_scores(preview_cards, at_time)
|
||||
@ -96,17 +88,27 @@ class Trends::Links < Trends::Base
|
||||
|
||||
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
|
||||
|
||||
if decaying_score.zero?
|
||||
redis.zrem("#{PREFIX}:all", preview_card.id)
|
||||
redis.zrem("#{PREFIX}:allowed", preview_card.id)
|
||||
else
|
||||
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
|
||||
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
|
||||
all: true,
|
||||
allowed: preview_card.trendable?,
|
||||
})
|
||||
|
||||
if preview_card.trendable?
|
||||
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
|
||||
else
|
||||
redis.zrem("#{PREFIX}:allowed", preview_card.id)
|
||||
end
|
||||
next unless valid_locale?(preview_card.language)
|
||||
|
||||
add_to_and_remove_from_subsets(preview_card.id, decaying_score, {
|
||||
"all:#{preview_card.language}" => true,
|
||||
"allowed:#{preview_card.language}" => preview_card.trendable?,
|
||||
})
|
||||
end
|
||||
|
||||
# Clean up localized sets by calculating the intersection with the main
|
||||
# set. We do this instead of just deleting the localized sets to avoid
|
||||
# having moments where the API returns empty results
|
||||
|
||||
redis.pipelined do
|
||||
Trends.available_locales.each do |locale|
|
||||
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
|
||||
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -1,6 +1,6 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Form::PreviewCardBatch
|
||||
class Trends::PreviewCardBatch
|
||||
include ActiveModel::Model
|
||||
include Authorization
|
||||
|
||||
@ -10,12 +10,12 @@ class Form::PreviewCardBatch
|
||||
case action
|
||||
when 'approve'
|
||||
approve!
|
||||
when 'approve_all'
|
||||
approve_all!
|
||||
when 'approve_providers'
|
||||
approve_providers!
|
||||
when 'reject'
|
||||
reject!
|
||||
when 'reject_all'
|
||||
reject_all!
|
||||
when 'reject_providers'
|
||||
reject_providers!
|
||||
end
|
||||
end
|
||||
|
||||
@ -30,13 +30,13 @@ class Form::PreviewCardBatch
|
||||
end
|
||||
|
||||
def approve!
|
||||
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
|
||||
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
|
||||
preview_cards.update_all(trendable: true)
|
||||
end
|
||||
|
||||
def approve_all!
|
||||
def approve_providers!
|
||||
preview_card_providers.each do |provider|
|
||||
authorize(provider, :update?)
|
||||
authorize(provider, :review?)
|
||||
provider.update(trendable: true, reviewed_at: action_time)
|
||||
end
|
||||
|
||||
@ -45,13 +45,13 @@ class Form::PreviewCardBatch
|
||||
end
|
||||
|
||||
def reject!
|
||||
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
|
||||
preview_cards.each { |preview_card| authorize(preview_card, :review?) }
|
||||
preview_cards.update_all(trendable: false)
|
||||
end
|
||||
|
||||
def reject_all!
|
||||
def reject_providers!
|
||||
preview_card_providers.each do |provider|
|
||||
authorize(provider, :update?)
|
||||
authorize(provider, :review?)
|
||||
provider.update(trendable: false, reviewed_at: action_time)
|
||||
end
|
||||
|
@ -1,8 +1,9 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class PreviewCardFilter
|
||||
class Trends::PreviewCardFilter
|
||||
KEYS = %i(
|
||||
trending
|
||||
locale
|
||||
).freeze
|
||||
|
||||
attr_reader :params
|
||||
@ -15,7 +16,7 @@ class PreviewCardFilter
|
||||
scope = PreviewCard.unscoped
|
||||
|
||||
params.each do |key, value|
|
||||
next if key.to_s == 'page'
|
||||
next if %w(page locale).include?(key.to_s)
|
||||
|
||||
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
|
||||
end
|
||||
@ -35,19 +36,11 @@ class PreviewCardFilter
|
||||
end
|
||||
|
||||
def trending_scope(value)
|
||||
ids = begin
|
||||
case value.to_s
|
||||
when 'allowed'
|
||||
Trends.links.currently_trending_ids(true, -1)
|
||||
else
|
||||
Trends.links.currently_trending_ids(false, -1)
|
||||
end
|
||||
end
|
||||
scope = Trends.links.query
|
||||
|
||||
if ids.empty?
|
||||
PreviewCard.none
|
||||
else
|
||||
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
|
||||
end
|
||||
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
|
||||
scope = scope.allowed if value == 'allowed'
|
||||
|
||||
scope.to_arel
|
||||
end
|
||||
end
|
@ -1,6 +1,6 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Form::PreviewCardProviderBatch
|
||||
class Trends::PreviewCardProviderBatch
|
||||
include ActiveModel::Model
|
||||
include Authorization
|
||||
|
||||
@ -22,12 +22,12 @@ class Form::PreviewCardProviderBatch
|
||||
end
|
||||
|
||||
def approve!
|
||||
preview_card_providers.each { |provider| authorize(provider, :update?) }
|
||||
preview_card_providers.each { |provider| authorize(provider, :review?) }
|
||||
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
|
||||
end
|
||||
|
||||
def reject!
|
||||
preview_card_providers.each { |provider| authorize(provider, :update?) }
|
||||
preview_card_providers.each { |provider| authorize(provider, :review?) }
|
||||
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
|
||||
end
|
||||
end
|
@ -1,6 +1,6 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class PreviewCardProviderFilter
|
||||
class Trends::PreviewCardProviderFilter
|
||||
KEYS = %i(
|
||||
status
|
||||
).freeze
|
106
app/models/trends/query.rb
Normal file
106
app/models/trends/query.rb
Normal file
@ -0,0 +1,106 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Trends::Query
|
||||
include Redisable
|
||||
include Enumerable
|
||||
|
||||
attr_reader :prefix, :klass, :loaded
|
||||
|
||||
alias loaded? loaded
|
||||
|
||||
def initialize(prefix, klass)
|
||||
@prefix = prefix
|
||||
@klass = klass
|
||||
@records = []
|
||||
@loaded = false
|
||||
@allowed = false
|
||||
@limit = -1
|
||||
@offset = 0
|
||||
end
|
||||
|
||||
def allowed!
|
||||
@allowed = true
|
||||
self
|
||||
end
|
||||
|
||||
def allowed
|
||||
clone.allowed!
|
||||
end
|
||||
|
||||
def in_locale!(value)
|
||||
@locale = value
|
||||
self
|
||||
end
|
||||
|
||||
def in_locale(value)
|
||||
clone.in_locale!(value)
|
||||
end
|
||||
|
||||
def offset!(value)
|
||||
@offset = value
|
||||
self
|
||||
end
|
||||
|
||||
def offset(value)
|
||||
clone.offset!(value)
|
||||
end
|
||||
|
||||
def limit!(value)
|
||||
@limit = value
|
||||
self
|
||||
end
|
||||
|
||||
def limit(value)
|
||||
clone.limit!(value)
|
||||
end
|
||||
|
||||
def records
|
||||
load
|
||||
@records
|
||||
end
|
||||
|
||||
delegate :each, :empty?, :first, :last, to: :records
|
||||
|
||||
def to_ary
|
||||
records.dup
|
||||
end
|
||||
|
||||
alias to_a to_ary
|
||||
|
||||
def to_arel
|
||||
tmp_ids = ids
|
||||
|
||||
if tmp_ids.empty?
|
||||
klass.none
|
||||
else
|
||||
klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering')
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def key
|
||||
[@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':')
|
||||
end
|
||||
|
||||
def load
|
||||
unless loaded?
|
||||
@records = perform_queries
|
||||
@loaded = true
|
||||
end
|
||||
|
||||
self
|
||||
end
|
||||
|
||||
def ids
|
||||
redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i)
|
||||
end
|
||||
|
||||
def perform_queries
|
||||
apply_scopes(to_arel).to_a
|
||||
end
|
||||
|
||||
def apply_scopes(scope)
|
||||
scope
|
||||
end
|
||||
end
|
65
app/models/trends/status_batch.rb
Normal file
65
app/models/trends/status_batch.rb
Normal file
@ -0,0 +1,65 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Trends::StatusBatch
|
||||
include ActiveModel::Model
|
||||
include Authorization
|
||||
|
||||
attr_accessor :status_ids, :action, :current_account
|
||||
|
||||
def save
|
||||
case action
|
||||
when 'approve'
|
||||
approve!
|
||||
when 'approve_accounts'
|
||||
approve_accounts!
|
||||
when 'reject'
|
||||
reject!
|
||||
when 'reject_accounts'
|
||||
reject_accounts!
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def statuses
|
||||
@statuses ||= Status.where(id: status_ids)
|
||||
end
|
||||
|
||||
def status_accounts
|
||||
@status_accounts ||= Account.where(id: statuses.map(&:account_id).uniq)
|
||||
end
|
||||
|
||||
def approve!
|
||||
statuses.each { |status| authorize(status, :review?) }
|
||||
statuses.update_all(trendable: true)
|
||||
end
|
||||
|
||||
def approve_accounts!
|
||||
status_accounts.each do |account|
|
||||
authorize(account, :review?)
|
||||
account.update(trendable: true, reviewed_at: action_time)
|
||||
end
|
||||
|
||||
# Reset any individual overrides
|
||||
statuses.update_all(trendable: nil)
|
||||
end
|
||||
|
||||
def reject!
|
||||
statuses.each { |status| authorize(status, :review?) }
|
||||
statuses.update_all(trendable: false)
|
||||
end
|
||||
|
||||
def reject_accounts!
|
||||
status_accounts.each do |account|
|
||||
authorize(account, :review?)
|
||||
account.update(trendable: false, reviewed_at: action_time)
|
||||
end
|
||||
|
||||
# Reset any individual overrides
|
||||
statuses.update_all(trendable: nil)
|
||||
end
|
||||
|
||||
def action_time
|
||||
@action_time ||= Time.now.utc
|
||||
end
|
||||
end
|
46
app/models/trends/status_filter.rb
Normal file
46
app/models/trends/status_filter.rb
Normal file
@ -0,0 +1,46 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Trends::StatusFilter
|
||||
KEYS = %i(
|
||||
trending
|
||||
locale
|
||||
).freeze
|
||||
|
||||
attr_reader :params
|
||||
|
||||
def initialize(params)
|
||||
@params = params
|
||||
end
|
||||
|
||||
def results
|
||||
scope = Status.unscoped.kept
|
||||
|
||||
params.each do |key, value|
|
||||
next if %w(page locale).include?(key.to_s)
|
||||
|
||||
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
|
||||
end
|
||||
|
||||
scope
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def scope_for(key, value)
|
||||
case key.to_s
|
||||
when 'trending'
|
||||
trending_scope(value)
|
||||
else
|
||||
raise "Unknown filter: #{key}"
|
||||
end
|
||||
end
|
||||
|
||||
def trending_scope(value)
|
||||
scope = Trends.statuses.query
|
||||
|
||||
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
|
||||
scope = scope.allowed if value == 'allowed'
|
||||
|
||||
scope.to_arel
|
||||
end
|
||||
end
|
142
app/models/trends/statuses.rb
Normal file
142
app/models/trends/statuses.rb
Normal file
@ -0,0 +1,142 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Trends::Statuses < Trends::Base
|
||||
PREFIX = 'trending_statuses'
|
||||
|
||||
self.default_options = {
|
||||
threshold: 5,
|
||||
review_threshold: 3,
|
||||
score_halflife: 2.hours.freeze,
|
||||
}
|
||||
|
||||
class Query < Trends::Query
|
||||
def filtered_for!(account)
|
||||
@account = account
|
||||
self
|
||||
end
|
||||
|
||||
def filtered_for(account)
|
||||
clone.filtered_for!(account)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def apply_scopes(scope)
|
||||
scope.includes(:account)
|
||||
end
|
||||
|
||||
def perform_queries
|
||||
return super if @account.nil?
|
||||
|
||||
statuses = super
|
||||
account_ids = statuses.map(&:account_id)
|
||||
account_domains = statuses.map(&:account_domain)
|
||||
|
||||
preloaded_relations = {
|
||||
blocking: Account.blocking_map(account_ids, @account.id),
|
||||
blocked_by: Account.blocked_by_map(account_ids, @account.id),
|
||||
muting: Account.muting_map(account_ids, @account.id),
|
||||
following: Account.following_map(account_ids, @account.id),
|
||||
domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id),
|
||||
}
|
||||
|
||||
statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
|
||||
end
|
||||
end
|
||||
|
||||
def register(status, at_time = Time.now.utc)
|
||||
add(status.proper, status.account_id, at_time) if eligible?(status)
|
||||
end
|
||||
|
||||
def add(status, _account_id, at_time = Time.now.utc)
|
||||
# We rely on the total reblogs and favourites count, so we
|
||||
# don't record which account did the what and when here
|
||||
|
||||
record_used_id(status.id, at_time)
|
||||
end
|
||||
|
||||
def query
|
||||
Query.new(key_prefix, klass)
|
||||
end
|
||||
|
||||
def refresh(at_time = Time.now.utc)
|
||||
statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
|
||||
calculate_scores(statuses, at_time)
|
||||
trim_older_items
|
||||
end
|
||||
|
||||
def request_review
|
||||
statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
|
||||
|
||||
statuses.filter_map do |status|
|
||||
next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
|
||||
|
||||
status.account.touch(:requested_review_at)
|
||||
status
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def key_prefix
|
||||
PREFIX
|
||||
end
|
||||
|
||||
def klass
|
||||
Status
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def eligible?(status)
|
||||
original_status = status.proper
|
||||
|
||||
original_status.public_visibility? &&
|
||||
original_status.account.discoverable? && !original_status.account.silenced? &&
|
||||
original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply?
|
||||
end
|
||||
|
||||
def calculate_scores(statuses, at_time)
|
||||
redis.pipelined do
|
||||
statuses.each do |status|
|
||||
expected = 1.0
|
||||
observed = (status.reblogs_count + status.favourites_count).to_f
|
||||
|
||||
score = begin
|
||||
if expected > observed || observed < options[:threshold]
|
||||
0
|
||||
else
|
||||
((observed - expected)**2) / expected
|
||||
end
|
||||
end
|
||||
|
||||
decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
|
||||
|
||||
add_to_and_remove_from_subsets(status.id, decaying_score, {
|
||||
all: true,
|
||||
allowed: status.trendable? && status.account.discoverable?,
|
||||
})
|
||||
|
||||
next unless valid_locale?(status.language)
|
||||
|
||||
add_to_and_remove_from_subsets(status.id, decaying_score, {
|
||||
"all:#{status.language}" => true,
|
||||
"allowed:#{status.language}" => status.trendable? && status.account.discoverable?,
|
||||
})
|
||||
end
|
||||
|
||||
# Clean up localized sets by calculating the intersection with the main
|
||||
# set. We do this instead of just deleting the localized sets to avoid
|
||||
# having moments where the API returns empty results
|
||||
|
||||
Trends.available_locales.each do |locale|
|
||||
redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
|
||||
redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def would_be_trending?(id)
|
||||
score(id) > score_at_rank(options[:review_threshold] - 1)
|
||||
end
|
||||
end
|
@ -1,6 +1,6 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Form::TagBatch
|
||||
class Trends::TagBatch
|
||||
include ActiveModel::Model
|
||||
include Authorization
|
||||
|
||||
@ -22,12 +22,12 @@ class Form::TagBatch
|
||||
end
|
||||
|
||||
def approve!
|
||||
tags.each { |tag| authorize(tag, :update?) }
|
||||
tags.each { |tag| authorize(tag, :review?) }
|
||||
tags.update_all(trendable: true, reviewed_at: action_time)
|
||||
end
|
||||
|
||||
def reject!
|
||||
tags.each { |tag| authorize(tag, :update?) }
|
||||
tags.each { |tag| authorize(tag, :review?) }
|
||||
tags.update_all(trendable: false, reviewed_at: action_time)
|
||||
end
|
||||
|
@ -1,6 +1,6 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class TagFilter
|
||||
class Trends::TagFilter
|
||||
KEYS = %i(
|
||||
trending
|
||||
status
|
||||
@ -42,13 +42,7 @@ class TagFilter
|
||||
end
|
||||
|
||||
def trending_scope
|
||||
ids = Trends.tags.currently_trending_ids(false, -1)
|
||||
|
||||
if ids.empty?
|
||||
Tag.none
|
||||
else
|
||||
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
|
||||
end
|
||||
Trends.tags.query.to_arel
|
||||
end
|
||||
|
||||
def status_scope(value)
|
@ -5,7 +5,7 @@ class Trends::Tags < Trends::Base
|
||||
|
||||
self.default_options = {
|
||||
threshold: 5,
|
||||
review_threshold: 10,
|
||||
review_threshold: 3,
|
||||
max_score_cooldown: 2.days.freeze,
|
||||
max_score_halflife: 4.hours.freeze,
|
||||
}
|
||||
@ -29,27 +29,15 @@ class Trends::Tags < Trends::Base
|
||||
trim_older_items
|
||||
end
|
||||
|
||||
def get(allowed, limit)
|
||||
tag_ids = currently_trending_ids(allowed, limit)
|
||||
tags = Tag.where(id: tag_ids).index_by(&:id)
|
||||
tag_ids.map { |id| tags[id] }.compact
|
||||
end
|
||||
|
||||
def request_review
|
||||
tags = Tag.where(id: currently_trending_ids(false, -1))
|
||||
|
||||
tags_requiring_review = tags.filter_map do |tag|
|
||||
tags.filter_map do |tag|
|
||||
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
|
||||
|
||||
tag.touch(:requested_review_at)
|
||||
tag
|
||||
end
|
||||
|
||||
return if tags_requiring_review.empty?
|
||||
|
||||
User.staff.includes(:account).find_each do |user|
|
||||
AdminMailer.new_trending_tags(user.account, |