tykayn
/
mastodon
mirror of https://framagit.org/tykayn/mastodon
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add trending links (#16917) 

* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
master
Eugen Rochko 2021-11-25 13:07:38 +01:00 committed by GitHub
parent 46e62fc4b3
commit 6e50134a42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
97 changed files with 2071 additions and 722 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/chewy/tags_index.rb
View File

@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
end
field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
end
end

2
app/controllers/admin/dashboard_controller.rb
View File

@ -4,7 +4,7 @@ module Admin
class DashboardController < BaseController
def index
@system_checks = Admin::SystemCheck.perform
@time_period = (1.month.ago.to_date...Time.now.utc.to_date)
@time_period = (29.days.ago.to_date...Time.now.utc.to_date)
@pending_users_count = User.pending.count
@pending_reports_count = Report.unresolved.count
@pending_tags_count = Tag.pending_review.count

76
app/controllers/admin/tags_controller.rb
View File

@ -2,38 +2,12 @@
module Admin
class TagsController < BaseController
before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_tags_path(filter_params)
end
def approve_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
redirect_to admin_tags_path(filter_params)
end
def reject_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
redirect_to admin_tags_path(filter_params)
end
before_action :set_tag
def show
authorize @tag, :show?
@time_period = (6.days.ago.to_date...Time.now.utc.to_date)
end
def update
@ -52,52 +26,8 @@ module Admin
@tag = Tag.find(params[:id])
end
def set_usage_by_domain
@usage_by_domain = @tag.statuses
.with_public_visibility
.excluding_silenced_accounts
.where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
.joins(:account)
.group('accounts.domain')
.reorder(statuses_count: :desc)
.pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
end
def set_counters
@accounts_today = @tag.history.first[:accounts]
@accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
end
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def tag_params
params.require(:tag).permit(:name, :trendable, :usable, :listable)
end
def current_week_days
now = Time.now.utc.beginning_of_day.to_date
(Date.commercial(now.cwyear, now.cweek)..now).map do |date|
date.to_time(:utc).beginning_of_day.to_i
end
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end
end

41
app/controllers/admin/trends/links/preview_card_providers_controller.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
def index
authorize :preview_card_provider, :index?
@preview_card_providers = filtered_preview_card_providers.page(params[:page])
@form = Form::PreviewCardProviderBatch.new
end
def batch
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_preview_card_providers_path(filter_params)
end
private
def filtered_preview_card_providers
PreviewCardProviderFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
end
def form_preview_card_provider_batch_params
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

45
app/controllers/admin/trends/links_controller.rb Normal file
View File

@ -0,0 +1,45 @@
# frozen_string_literal: true
class Admin::Trends::LinksController < Admin::BaseController
def index
authorize :preview_card, :index?
@preview_cards = filtered_preview_cards.page(params[:page])
@form = Form::PreviewCardBatch.new
end
def batch
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_path(filter_params)
end
private
def filtered_preview_cards
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
end
def filter_params
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
end
def form_preview_card_batch_params
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:approve_all]
'approve_all'
elsif params[:reject]
'reject'
elsif params[:reject_all]
'reject_all'
end
end
end

41
app/controllers/admin/trends/tags_controller.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Trends::TagsController < Admin::BaseController
def index
authorize :tag, :index?
@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end
def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_tags_path(filter_params)
end
private
def filtered_tags
TagFilter.new(filter_params).results
end
def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end
def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end
def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end

3
app/controllers/api/v1/admin/dimensions_controller.rb
View File

@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController
params[:keys],
params[:start_at],
params[:end_at],
params[:limit]
params[:limit],
params
)
end
end

3
app/controllers/api/v1/admin/measures_controller.rb
View File

@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController
@measures = Admin::Metrics::Measure.retrieve(
params[:keys],
params[:start_at],
params[:end_at]
params[:end_at],
params
)
end
end

16
app/controllers/api/v1/admin/trends/tags_controller.rb Normal file
View File

@ -0,0 +1,16 @@
# frozen_string_literal: true
class Api::V1::Admin::Trends::TagsController < Api::BaseController
before_action :require_staff!
before_action :set_tags
def index
render json: @tags, each_serializer: REST::Admin::TagSerializer
end
private
def set_tags
@tags = Trends.tags.get(false, limit_param(10))
end
end

16
app/controllers/api/v1/admin/trends_controller.rb
View File

@ -1,16 +0,0 @@
# frozen_string_literal: true
class Api::V1::Admin::TrendsController < Api::BaseController
before_action :require_staff!
before_action :set_trends
def index
render json: @trends, each_serializer: REST::Admin::TagSerializer
end
private
def set_trends
@trends = TrendingTags.get(10, filtered: false)
end
end

21
app/controllers/api/v1/trends/links_controller.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::LinksController < Api::BaseController
before_action :set_links
def index
render json: @links, each_serializer: REST::Trends::LinkSerializer
end
private
def set_links
@links = begin
if Setting.trends
Trends.links.get(true, limit_param(10))
else
[]
end
end
end
end

21
app/controllers/api/v1/trends/tags_controller.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Api::V1::Trends::TagsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = begin
if Setting.trends
Trends.tags.get(true, limit_param(10))
else
[]
end
end
end
end

15
app/controllers/api/v1/trends_controller.rb
View File

@ -1,15 +0,0 @@
# frozen_string_literal: true
class Api::V1::TrendsController < Api::BaseController
before_action :set_tags
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = TrendingTags.get(limit_param(10))
end
end

2
app/helpers/admin/filter_helper.rb
View File

@ -6,6 +6,8 @@ module Admin::FilterHelper
CustomEmojiFilter::KEYS,
ReportFilter::KEYS,
TagFilter::KEYS,
PreviewCardProviderFilter::KEYS,
PreviewCardFilter::KEYS,
InstanceFilter::KEYS,
InviteFilter::KEYS,
RelationshipFilter::KEYS,

94
app/helpers/languages_helper.rb Normal file
View File

@ -0,0 +1,94 @@
# frozen_string_literal: true
module LanguagesHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文（香港）',
'zh-TW': '繁體中文（臺灣）',
zh: '中文',
}.freeze
def human_locale(locale)
if locale == 'und'
I18n.t('generic.none')
else
HUMAN_LOCALES[locale.to_sym] || locale
end
end
end

89
app/helpers/settings_helper.rb
View File

@ -1,95 +1,8 @@
# frozen_string_literal: true
module SettingsHelper
HUMAN_LOCALES = {
af: 'Afrikaans',
ar: 'العربية',
ast: 'Asturianu',
bg: 'Български',
bn: 'বাংলা',
br: 'Breton',
ca: 'Català',
co: 'Corsu',
cs: 'Čeština',
cy: 'Cymraeg',
da: 'Dansk',
de: 'Deutsch',
el: 'Ελληνικά',
en: 'English',
eo: 'Esperanto',
'es-AR': 'Español (Argentina)',
'es-MX': 'Español (México)',
es: 'Español',
et: 'Eesti',
eu: 'Euskara',
fa: 'فارسی',
fi: 'Suomi',
fr: 'Français',
ga: 'Gaeilge',
gd: 'Gàidhlig',
gl: 'Galego',
he: 'עברית',
hi: 'हिन्दी',
hr: 'Hrvatski',
hu: 'Magyar',
hy: 'Հայերեն',
id: 'Bahasa Indonesia',
io: 'Ido',
is: 'Íslenska',
it: 'Italiano',
ja: '日本語',
ka: 'ქართული',
kab: 'Taqbaylit',
kk: 'Қазақша',
kmr: 'Kurmancî',
kn: 'ಕನ್ನಡ',
ko: '한국어',
ku: 'سۆرانی',
lt: 'Lietuvių',
lv: 'Latviešu',
mk: 'Македонски',
ml: 'മലയാളം',
mr: 'मराठी',
ms: 'Bahasa Melayu',
nl: 'Nederlands',
nn: 'Nynorsk',
no: 'Norsk',
oc: 'Occitan',
pl: 'Polski',
'pt-BR': 'Português (Brasil)',
'pt-PT': 'Português (Portugal)',
pt: 'Português',
ro: 'Română',
ru: 'Русский',
sa: 'संस्कृतम्',
sc: 'Sardu',
si: 'සිංහල',
sk: 'Slovenčina',
sl: 'Slovenščina',
sq: 'Shqip',
'sr-Latn': 'Srpski (latinica)',
sr: 'Српски',
sv: 'Svenska',
ta: 'தமிழ்',
te: 'తెలుగు',
th: 'ไทย',
tr: 'Türkçe',
uk: 'Українська',
ur: 'اُردُو',
vi: 'Tiếng Việt',
zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ',
'zh-CN': '简体中文',
'zh-HK': '繁體中文（香港）',
'zh-TW': '繁體中文（臺灣）',
zh: '中文',
}.freeze
def human_locale(locale)
HUMAN_LOCALES[locale]
end
def filterable_languages
LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?))
LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?))
end
def hash_to_object(hash)

5
app/javascript/mastodon/components/admin/Counter.js
View File

@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent {
end_at: PropTypes.string.isRequired,
label: PropTypes.string.isRequired,
href: PropTypes.string,
params: PropTypes.object,
};
state = {
@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent {
};
componentDidMount () {
const { measure, start_at, end_at } = this.props;
const { measure, start_at, end_at, params } = this.props;
api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => {
api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => {
this.setState({
loading: false,
data: res.data,

5
app/javascript/mastodon/components/admin/Dimension.js
View File

@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent {
end_at: PropTypes.string.isRequired,
limit: PropTypes.number.isRequired,
label: PropTypes.string.isRequired,
params: PropTypes.object,
};
state = {
@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent {
};
componentDidMount () {
const { start_at, end_at, dimension, limit } = this.props;
const { start_at, end_at, dimension, limit, params } = this.props;
api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => {
api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => {
this.setState({
loading: false,
data: res.data,

2
app/javascript/mastodon/components/admin/Trends.js
View File

@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent {
componentDidMount () {
const { limit } = this.props;
api().get('/api/v1/admin/trends', { params: { limit } }).then(res => {
api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => {
this.setState({
loading: false,
data: res.data,

16
app/javascript/styles/mastodon/accounts.scss
View File

@ -325,3 +325,19 @@
margin-top: 10px;
}
}
.batch-table__row--muted .pending-account__header {
&,
a,
strong {
color: lighten($ui-base-color, 26%);
}
}
.batch-table__row--attention .pending-account__header {
&,
a,
strong {
color: $gold-star;
}
}

10
app/javascript/styles/mastodon/dashboard.scss
View File

@ -100,6 +100,16 @@
transition: all 200ms ease-out;
}
&.positive {
background: lighten($ui-base-color, 4%);
color: $valid-value-color;
}
&.negative {
background: lighten($ui-base-color, 4%);
color: $error-value-color;
}
span {
flex: 1 1 auto;
}

2
app/lib/activitypub/activity.rb
View File

@ -129,8 +129,6 @@ class ActivityPub::Activity
end
def crawl_links(status)
return if status.spoiler_text?
# Spread out crawling randomly to avoid DDoSing the link
LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id)
end

5
app/lib/activitypub/activity/announce.rb
View File

@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
visibility: visibility_from_audience
)
original_status.tags.each do |tag|
tag.use!(@account)
end
Trends.tags.register(@status)
Trends.links.register(@status)
distribute(@status)
end

7
app/lib/activitypub/activity/create.rb
View File

@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def attach_tags(status)
@tags.each do |tag|
status.tags << tag
tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago)
end
# If we're processing an old status, this may register tags as being used now
# as opposed to when the status was really published, but this is probably
# not a big deal
Trends.tags.register(status)
@mentions.each do |mention|
mention.status = status
mention.save

9
app/lib/admin/metrics/dimension.rb
View File

@ -7,9 +7,14 @@ class Admin::Metrics::Dimension
servers: Admin::Metrics::Dimension::ServersDimension,
space_usage: Admin::Metrics::Dimension::SpaceUsageDimension,
software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension,
tag_servers: Admin::Metrics::Dimension::TagServersDimension,
tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension,
}.freeze
def self.retrieve(dimension_keys, start_at, end_at, limit)
Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact
def self.retrieve(dimension_keys, start_at, end_at, limit, params)
Array(dimension_keys).map do |key|
klass = DIMENSIONS[key.to_sym]
klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

13
app/lib/admin/metrics/dimension/base_dimension.rb
View File

@ -1,10 +1,15 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::BaseDimension
def initialize(start_at, end_at, limit)
def self.with_params?
false
end
def initialize(start_at, end_at, limit, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@limit = limit&.to_i
@params = params
end
def key
@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def params
raise NotImplementedError
end
end

4
app/lib/admin/metrics/dimension/languages_dimension.rb
View File

@ -1,6 +1,8 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def key
'languages'
end
@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]])
rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } }
rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } }
end
end

36
app/lib/admin/metrics/dimension/tag_languages_dimension.rb Normal file
View File

@ -0,0 +1,36 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension
include LanguagesHelper
def self.with_params?
true
end
def key
'tag_languages'
end
def data
sql = <<-SQL.squish
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
FROM statuses
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY COALESCE(statuses.language, 'und')
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

35
app/lib/admin/metrics/dimension/tag_servers_dimension.rb Normal file
View File

@ -0,0 +1,35 @@
# frozen_string_literal: true
class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension
def self.with_params?
true
end
def key
'tag_servers'
end
def data
sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value
FROM statuses
INNER JOIN accounts ON accounts.id = statuses.account_id
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id
WHERE statuses_tags.tag_id = $1
AND statuses.id BETWEEN $2 AND $3
GROUP BY accounts.domain
ORDER BY count(*) DESC
LIMIT $4
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]])
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
end
private
def params
@params.permit(:id)
end
end

10
app/lib/admin/metrics/measure.rb
View File

@ -7,9 +7,15 @@ class Admin::Metrics::Measure
interactions: Admin::Metrics::Measure::InteractionsMeasure,
opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure,
resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure,
tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure,
tag_uses: Admin::Metrics::Measure::TagUsesMeasure,
tag_servers: Admin::Metrics::Measure::TagServersMeasure,
}.freeze
def self.retrieve(measure_keys, start_at, end_at)
Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact
def self.retrieve(measure_keys, start_at, end_at, params)
Array(measure_keys).map do |key|
klass = MEASURES[key.to_sym]
klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil)
end.compact
end
end

4
app/lib/admin/metrics/measure/active_users_measure.rb
View File

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

15
app/lib/admin/metrics/measure/base_measure.rb
View File

@ -1,9 +1,14 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::BaseMeasure
def initialize(start_at, end_at)
def self.with_params?
false
end
def initialize(start_at, end_at, params)
@start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime
@params = params
end
def key
@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure
protected
def time_period
(@start_at...@end_at)
(@start_at..@end_at)
end
def previous_time_period
((@start_at - length_of_period)...(@end_at - length_of_period))
((@start_at - length_of_period)..(@end_at - length_of_period))
end
def length_of_period
@length_of_period ||= @end_at - @start_at
end
def params
raise NotImplementedError
end
end

4
app/lib/admin/metrics/measure/interactions_measure.rb
View File

@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
end
def time_period
(@start_at.to_date...@end_at.to_date)
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period))
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
end

41
app/lib/admin/metrics/measure/tag_accounts_measure.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_accounts'
end
def total
tag.history.aggregate(time_period).accounts
end
def previous_total
tag.history.aggregate(previous_time_period).accounts
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

47
app/lib/admin/metrics/measure/tag_servers_measure.rb Normal file
View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_servers'
end
def total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def previous_total
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
end
def data
sql = <<-SQL.squish
SELECT axis.*, (
SELECT count(*) AS value
FROM statuses
WHERE statuses.id BETWEEN $1 AND $2
AND date_trunc('day', statuses.created_at)::date = axis.day
)
FROM (
SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day
) as axis
SQL
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]])
rows.map { |row| { date: row['day'], value: row['value'].to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def params
@params.permit(:id)
end
end

41
app/lib/admin/metrics/measure/tag_uses_measure.rb Normal file
View File

@ -0,0 +1,41 @@
# frozen_string_literal: true
class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure
def self.with_params?
true
end
def key
'tag_uses'
end
def total
tag.history.aggregate(time_period).uses
end
def previous_total
tag.history.aggregate(previous_time_period).uses
end
def data
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
end
protected
def tag
@tag ||= Tag.find(params[:id])
end
def time_period
(@start_at.to_date..@end_at.to_date)
end
def previous_time_period
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period))
end
def params
@params.permit(:id)
end
end

49
app/lib/link_details_extractor.rb
View File

@ -4,6 +4,11 @@ class LinkDetailsExtractor
include ActionView::Helpers::TagHelper
class StructuredData
SUPPORTED_TYPES = %w(
NewsArticle
WebPage
).freeze
def initialize(data)
@data = data
end
@ -16,6 +21,14 @@ class LinkDetailsExtractor
json['description']
end
def language
json['inLanguage']
end
def type
json['@type']
end
def image
obj = first_of_value(json['image'])
@ -44,6 +57,10 @@ class LinkDetailsExtractor
publisher['name']
end
def publisher_logo
publisher.dig('logo', 'url')
end
private
def author
@ -58,8 +75,12 @@ class LinkDetailsExtractor
arr.is_a?(Array) ? arr.first : arr
end
def root_array(root)
root.is_a?(Array) ? root : [root]
end
def json
@json ||= first_of_value(Oj.load(@data))
@json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {}
end
end
@ -75,6 +96,7 @@ class LinkDetailsExtractor
description: description || '',
image_remote_url: image,
type: type,
link_type: link_type,
width: width || 0,
height: height || 0,
html: html || '',
@ -83,6 +105,7 @@ class LinkDetailsExtractor
author_name: author_name || '',
author_url: author_url || '',
embed_url: embed_url || '',
language: language,
}
end
@ -90,6 +113,14 @@ class LinkDetailsExtractor
player_url.present? ? :video : :link
end
def link_type
if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article'
:article
else
:unknown
end
end
def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end
@ -138,6 +169,14 @@ class LinkDetailsExtractor
valid_url_or_nil(opengraph_tag('twitter:player:stream'))
end
def language
valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first)
end
def icon
valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon'))
end
private
def player_url
@ -162,6 +201,14 @@ class LinkDetailsExtractor
nil
end
def valid_locale_or_nil(str)
return nil if str.blank?
code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA
locale = ISO_639.find(code)
locale&.alpha2
end
def link_tag(name)
document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
end

22
app/mailers/admin_mailer.rb
View File

@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer
end
end
def new_trending_tag(recipient, tag)
@tag = tag
@me = recipient
@instance = Rails.configuration.x.local_domain
def new_trending_tags(recipient, tags)
@tags = tags
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance)
end
end
def new_trending_links(recipient, links)
@links = links
@me = recipient
@instance = Rails.configuration.x.local_domain
@lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last
locale_for_account(@me) do
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance)
end
end
end

4
app/models/account_statuses_cleanup_policy.rb
View File

@ -4,8 +4,8 @@
#
# Table name: account_statuses_cleanup_policies
#
# id :bigint not null, primary key
# account_id :bigint not null
# id :bigint(8) not null, primary key
# account_id :bigint(8) not null
# enabled :boolean default(TRUE), not null
# min_status_age :integer default(1209600), not null
# keep_direct :boolean default(TRUE), not null

65
app/models/form/preview_card_batch.rb Normal file
View File

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_ids, :action, :current_account, :precision
def save
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'reject'
reject!
when 'reject_all'
reject_all!
end
end
private
def preview_cards
@preview_cards ||= PreviewCard.where(id: preview_card_ids)
end
def preview_card_providers
@preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

33
app/models/form/preview_card_provider_batch.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization