Add caching layer to metrics (#17617)

This commit is contained in:
Eugen Rochko 2022-02-22 15:27:08 +01:00 committed by GitHub
parent 8338826963
commit b377022cf9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 165 additions and 55 deletions

View File

@ -1,23 +1,34 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Dimension::BaseDimension class Admin::Metrics::Dimension::BaseDimension
CACHE_TTL = 5.minutes.freeze
def self.with_params? def self.with_params?
false false
end end
attr_reader :loaded
alias loaded? loaded
def initialize(start_at, end_at, limit, params) def initialize(start_at, end_at, limit, params)
@start_at = start_at&.to_datetime @start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime @end_at = end_at&.to_datetime
@limit = limit&.to_i @limit = limit&.to_i
@params = params @params = params
@loaded = false
end end
def key def key
raise NotImplementedError raise NotImplementedError
end end
def cache_key
["metrics/dimension/#{key}", @start_at, @end_at, @limit, canonicalized_params].join(';')
end
def data def data
raise NotImplementedError load
end end
def self.model_name def self.model_name
@ -30,11 +41,28 @@ class Admin::Metrics::Dimension::BaseDimension
protected protected
def load
unless loaded?
@values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query }
@loaded = true
end
@values
end
def perform_query
raise NotImplementedError
end
def time_period def time_period
(@start_at..@end_at) (@start_at..@end_at)
end end
def params def params
raise NotImplementedError {}
end
def canonicalized_params
params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';')
end end
end end

View File

@ -7,7 +7,9 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension:
'languages' 'languages'
end end
def data protected
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT locale, count(*) AS value SELECT locale, count(*) AS value
FROM users FROM users

View File

@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::ServersDimension < Admin::Metrics::Dimension::B
'servers' 'servers'
end end
def data protected
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value SELECT accounts.domain, count(*) AS value
FROM statuses FROM statuses

View File

@ -7,12 +7,12 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
'software_versions' 'software_versions'
end end
def data protected
def perform_query
[mastodon_version, ruby_version, postgresql_version, redis_version] [mastodon_version, ruby_version, postgresql_version, redis_version]
end end
private
def mastodon_version def mastodon_version
value = Mastodon::Version.to_s value = Mastodon::Version.to_s

View File

@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::SourcesDimension < Admin::Metrics::Dimension::B
'sources' 'sources'
end end
def data protected
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT oauth_applications.name, count(*) AS value SELECT oauth_applications.name, count(*) AS value
FROM users FROM users

View File

@ -8,12 +8,12 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
'space_usage' 'space_usage'
end end
def data protected
def perform_query
[postgresql_size, redis_size, media_size] [postgresql_size, redis_size, media_size]
end end
private
def postgresql_size def postgresql_size
value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size'] value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size']

View File

@ -11,7 +11,9 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi
'tag_languages' 'tag_languages'
end end
def data protected
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value
FROM statuses FROM statuses
@ -28,8 +30,6 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi
rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } } rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } }
end end
private
def params def params
@params.permit(:id) @params.permit(:id)
end end

View File

@ -9,7 +9,9 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension
'tag_servers' 'tag_servers'
end end
def data protected
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT accounts.domain, count(*) AS value SELECT accounts.domain, count(*) AS value
FROM statuses FROM statuses
@ -27,8 +29,6 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } }
end end
private
def params def params
@params.permit(:id) @params.permit(:id)
end end

View File

@ -5,20 +5,20 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas
'active_users' 'active_users'
end end
def total protected
def perform_total_query
activity_tracker.sum(time_period.first, time_period.last) activity_tracker.sum(time_period.first, time_period.last)
end end
def previous_total def perform_previous_total_query
activity_tracker.sum(previous_time_period.first, previous_time_period.last) activity_tracker.sum(previous_time_period.first, previous_time_period.last)
end end
def data def perform_data_query
activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } }
end end
protected
def activity_tracker def activity_tracker
@activity_tracker ||= ActivityTracker.new('activity:logins', :unique) @activity_tracker ||= ActivityTracker.new('activity:logins', :unique)
end end

View File

@ -1,14 +1,25 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Measure::BaseMeasure class Admin::Metrics::Measure::BaseMeasure
CACHE_TTL = 5.minutes.freeze
def self.with_params? def self.with_params?
false false
end end
attr_reader :loaded
alias loaded? loaded
def initialize(start_at, end_at, params) def initialize(start_at, end_at, params)
@start_at = start_at&.to_datetime @start_at = start_at&.to_datetime
@end_at = end_at&.to_datetime @end_at = end_at&.to_datetime
@params = params @params = params
@loaded = false
end
def cache_key
["metrics/measure/#{key}", @start_at, @end_at, canonicalized_params].join(';')
end end
def key def key
@ -16,15 +27,15 @@ class Admin::Metrics::Measure::BaseMeasure
end end
def total def total
raise NotImplementedError load[:total]
end end
def previous_total def previous_total
raise NotImplementedError load[:previous_total]
end end
def data def data
raise NotImplementedError load[:data]
end end
def self.model_name def self.model_name
@ -37,6 +48,35 @@ class Admin::Metrics::Measure::BaseMeasure
protected protected
def load
unless loaded?
@values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_queries }.with_indifferent_access
@loaded = true
end
@values
end
def perform_queries
{
total: perform_total_query,
previous_total: perform_previous_total_query,
data: perform_data_query,
}
end
def perform_total_query
raise NotImplementedError
end
def perform_previous_total_query
raise NotImplementedError
end
def perform_data_query
raise NotImplementedError
end
def time_period def time_period
(@start_at..@end_at) (@start_at..@end_at)
end end
@ -50,6 +90,10 @@ class Admin::Metrics::Measure::BaseMeasure
end end
def params def params
raise NotImplementedError {}
end
def canonicalized_params
params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';')
end end
end end

View File

@ -5,20 +5,20 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba
'interactions' 'interactions'
end end
def total protected
def perform_total_query
activity_tracker.sum(time_period.first, time_period.last) activity_tracker.sum(time_period.first, time_period.last)
end end
def previous_total def perform_previous_total_query
activity_tracker.sum(previous_time_period.first, previous_time_period.last) activity_tracker.sum(previous_time_period.first, previous_time_period.last)
end end
def data def perform_data_query
activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } }
end end
protected
def activity_tracker def activity_tracker
@activity_tracker ||= ActivityTracker.new('activity:interactions', :basic) @activity_tracker ||= ActivityTracker.new('activity:interactions', :basic)
end end

View File

@ -5,15 +5,17 @@ class Admin::Metrics::Measure::NewUsersMeasure < Admin::Metrics::Measure::BaseMe
'new_users' 'new_users'
end end
def total protected
def perform_total_query
User.where(created_at: time_period).count User.where(created_at: time_period).count
end end
def previous_total def perform_previous_total_query
User.where(created_at: previous_time_period).count User.where(created_at: previous_time_period).count
end end
def data def perform_data_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT axis.*, ( SELECT axis.*, (
WITH new_users AS ( WITH new_users AS (

View File

@ -5,15 +5,17 @@ class Admin::Metrics::Measure::OpenedReportsMeasure < Admin::Metrics::Measure::B
'opened_reports' 'opened_reports'
end end
def total protected
def perform_total_query
Report.where(created_at: time_period).count Report.where(created_at: time_period).count
end end
def previous_total def perform_previous_total_query
Report.where(created_at: previous_time_period).count Report.where(created_at: previous_time_period).count
end end
def data def perform_data_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT axis.*, ( SELECT axis.*, (
WITH new_reports AS ( WITH new_reports AS (

View File

@ -5,15 +5,17 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure:
'resolved_reports' 'resolved_reports'
end end
def total protected
def perform_total_query
Report.resolved.where(action_taken_at: time_period).count Report.resolved.where(action_taken_at: time_period).count
end end
def previous_total def perform_previous_total_query
Report.resolved.where(action_taken_at: previous_time_period).count Report.resolved.where(action_taken_at: previous_time_period).count
end end
def data def perform_data_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT axis.*, ( SELECT axis.*, (
WITH resolved_reports AS ( WITH resolved_reports AS (

View File

@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::Bas
'tag_accounts' 'tag_accounts'
end end
def total protected
def perform_total_query
tag.history.aggregate(time_period).accounts tag.history.aggregate(time_period).accounts
end end
def previous_total def perform_previous_total_query
tag.history.aggregate(previous_time_period).accounts tag.history.aggregate(previous_time_period).accounts
end end
def data def perform_data_query
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } }
end end
protected
def tag def tag
@tag ||= Tag.find(params[:id]) @tag ||= Tag.find(params[:id])
end end

View File

@ -9,15 +9,17 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base
'tag_servers' 'tag_servers'
end end
def total protected
def perform_total_query
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain')
end end
def previous_total def perform_previous_total_query
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain')
end end
def data def perform_data_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT axis.*, ( SELECT axis.*, (
SELECT count(distinct accounts.domain) AS value SELECT count(distinct accounts.domain) AS value
@ -38,8 +40,6 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base
rows.map { |row| { date: row['day'], value: row['value'].to_s } } rows.map { |row| { date: row['day'], value: row['value'].to_s } }
end end
protected
def tag def tag
@tag ||= Tag.find(params[:id]) @tag ||= Tag.find(params[:id])
end end

View File

@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMea
'tag_uses' 'tag_uses'
end end
def total protected
def perform_total_query
tag.history.aggregate(time_period).uses tag.history.aggregate(time_period).uses
end end
def previous_total def perform_previous_total_query
tag.history.aggregate(previous_time_period).uses tag.history.aggregate(previous_time_period).uses
end end
def data def perform_data_query
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } }
end end
protected
def tag def tag
@tag ||= Tag.find(params[:id]) @tag ||= Tag.find(params[:id])
end end

View File

@ -1,6 +1,8 @@
# frozen_string_literal: true # frozen_string_literal: true
class Admin::Metrics::Retention class Admin::Metrics::Retention
CACHE_TTL = 5.minutes.freeze
class Cohort < ActiveModelSerializers::Model class Cohort < ActiveModelSerializers::Model
attributes :period, :frequency, :data attributes :period, :frequency, :data
end end
@ -9,13 +11,37 @@ class Admin::Metrics::Retention
attributes :date, :rate, :value attributes :date, :rate, :value
end end
attr_reader :loaded
alias loaded? loaded
def initialize(start_at, end_at, frequency) def initialize(start_at, end_at, frequency)
@start_at = start_at&.to_date @start_at = start_at&.to_date
@end_at = end_at&.to_date @end_at = end_at&.to_date
@frequency = %w(day month).include?(frequency) ? frequency : 'day' @frequency = %w(day month).include?(frequency) ? frequency : 'day'
@loaded = false
end
def cache_key
['metrics/retention', @start_at, @end_at, @frequency].join(';')
end end
def cohorts def cohorts
load
end
protected
def load
unless loaded?
@values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query }
@loaded = true
end
@values
end
def perform_query
sql = <<-SQL.squish sql = <<-SQL.squish
SELECT axis.*, ( SELECT axis.*, (
WITH new_users AS ( WITH new_users AS (