Merge branch 'main' into glitch-soc/merge-upstream

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:20.04 as build-dep
 SHELL ["/usr/bin/bash", "-c"]
 
 # Install Node v12 (LTS)
-ENV NODE_VER="12.20.1"
+ENV NODE_VER="12.21.0"
 RUN ARCH= && \
     dpkgArch="$(dpkg --print-architecture)" && \
   case "${dpkgArch##*-}" in \

Gemfile

@@ -14,10 +14,10 @@ gem 'rack', '~> 2.2.3'
 gem 'hamlit-rails', '~> 0.2'
 gem 'pg', '~> 1.2'
 gem 'makara', '~> 0.5'
-gem 'pghero', '~> 2.7'
+gem 'pghero', '~> 2.8'
 gem 'dotenv-rails', '~> 2.7'
 
-gem 'aws-sdk-s3', '~> 1.89', require: false
+gem 'aws-sdk-s3', '~> 1.91', require: false
 gem 'fog-core', '<= 2.1.0'
 gem 'fog-openstack', '~> 0.3', require: false
 gem 'paperclip', '~> 6.0'
@@ -85,7 +85,7 @@ gem 'sanitize', '~> 5.2'
 gem 'scenic', '~> 1.5'
 gem 'sidekiq', '~> 6.1'
 gem 'sidekiq-scheduler', '~> 3.0'
-gem 'sidekiq-unique-jobs', '~> 6.0'
+gem 'sidekiq-unique-jobs', '~> 7.0'
 gem 'sidekiq-bulk', '~>0.2.0'
 gem 'simple-navigation', '~> 4.1'
 gem 'simple_form', '~> 5.1'
@@ -121,7 +121,7 @@ end
 group :test do
   gem 'capybara', '~> 3.35'
   gem 'climate_control', '~> 0.2'
-  gem 'faker', '~> 2.16'
+  gem 'faker', '~> 2.17'
   gem 'microformats', '~> 4.2'
   gem 'rails-controller-testing', '~> 1.0'
   gem 'rspec-sidekiq', '~> 3.1'
@@ -140,7 +140,7 @@ group :development do
   gem 'letter_opener', '~> 1.7'
   gem 'letter_opener_web', '~> 1.4'
   gem 'memory_profiler'
-  gem 'rubocop', '~> 1.9', require: false
+  gem 'rubocop', '~> 1.11', require: false
   gem 'rubocop-rails', '~> 2.9', require: false
   gem 'brakeman', '~> 4.10', require: false
   gem 'bundler-audit', '~> 0.7', require: false

Gemfile.lock

@@ -78,21 +78,21 @@ GEM
     av (0.9.0)
       cocaine (~> 0.5.3)
     awrence (1.1.1)
-    aws-eventstream (1.1.0)
+    aws-eventstream (1.1.1)
-    aws-partitions (1.429.0)
+    aws-partitions (1.432.0)
-    aws-sdk-core (3.112.0)
+    aws-sdk-core (3.113.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.42.0)
+    aws-sdk-kms (1.43.0)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.89.0)
+    aws-sdk-s3 (1.91.0)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
     bcrypt (3.1.16)
     better_errors (2.9.1)
@@ -108,6 +108,9 @@ GEM
       msgpack (~> 1.0)
     brakeman (4.10.1)
     browser (4.2.0)
+    brpoplpush-redis_script (0.1.1)
+      concurrent-ruby (~> 1.0, >= 1.0.5)
+      redis (>= 1.0, <= 5.0)
     builder (3.2.4)
     bullet (6.1.4)
       activesupport (>= 3.0.0)
@@ -210,7 +213,7 @@ GEM
       tzinfo
     excon (0.76.0)
     fabrication (2.21.1)
-    faker (2.16.0)
+    faker (2.17.0)
       i18n (>= 1.6, < 2)
     faraday (1.3.0)
       faraday-net_http (~> 1.0)
@@ -300,7 +303,7 @@ GEM
       multi_json (~> 1.14)
       rack (~> 2.0)
       rdf (~> 3.1)
-    json-ld-preloaded (3.1.4)
+    json-ld-preloaded (3.1.5)
       json-ld (~> 3.1)
       rdf (~> 3.1)
     jsonapi-renderer (0.2.2)
@@ -362,7 +365,7 @@ GEM
       net-ssh (>= 2.6.5, < 7.0.0)
     net-ssh (6.1.0)
     nio4r (2.5.7)
-    nokogiri (1.11.1)
+    nokogiri (1.11.2)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     nokogumbo (2.0.4)
@@ -372,7 +375,7 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.2)
       sidekiq (>= 3.5)
       statsd-ruby (~> 1.4, >= 1.4.0)
-    oj (3.11.2)
+    oj (3.11.3)
     omniauth (1.9.1)
       hashie (>= 3.4.6)
       rack (>= 1.6.2, < 3)
@@ -389,7 +392,7 @@ GEM
     openssl (2.2.0)
     openssl-signature_algorithm (0.4.0)
     orm_adapter (0.5.0)
-    ox (2.14.2)
+    ox (2.14.3)
     paperclip (6.0.0)
       activemodel (>= 4.2.0)
       activesupport (>= 4.2.0)
@@ -400,7 +403,7 @@ GEM
       av (~> 0.9.0)
       paperclip (>= 2.5.2)
     parallel (1.20.1)
-    parallel_tests (3.5.1)
+    parallel_tests (3.5.2)
       parallel
     parser (3.0.0.0)
       ast (~> 2.4.1)
@@ -408,7 +411,7 @@ GEM
     pastel (0.8.0)
       tty-color (~> 0.5)
     pg (1.2.3)
-    pghero (2.7.4)
+    pghero (2.8.0)
       activerecord (>= 5)
     pkg-config (1.4.5)
     pluck_each (0.1.3)
@@ -482,7 +485,7 @@ GEM
       thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (13.0.3)
-    rdf (3.1.12)
+    rdf (3.1.13)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
     rdf-normalize (0.4.0)
@@ -507,7 +510,7 @@ GEM
       redis-store (>= 1.2, < 2)
     redis-store (1.9.0)
       redis (>= 4, < 5)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     request_store (1.5.0)
       rack (>= 1.4)
     responders (3.0.1)
@@ -542,7 +545,7 @@ GEM
     rspec-support (3.10.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (1.9.1)
+    rubocop (1.11.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
@@ -587,9 +590,10 @@ GEM
       sidekiq (>= 3)
       thwait
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (6.0.25)
+    sidekiq-unique-jobs (7.0.4)
+      brpoplpush-redis_script (> 0.0.0, <= 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
-      sidekiq (>= 4.0, < 7.0)
+      sidekiq (>= 5.0, < 7.0)
       thor (>= 0.20, < 2.0)
     simple-navigation (4.1.0)
       activesupport (>= 2.3.2)
@@ -694,7 +698,7 @@ DEPENDENCIES
   active_record_query_trace (~> 1.8)
   addressable (~> 2.7)
   annotate (~> 3.1)
-  aws-sdk-s3 (~> 1.89)
+  aws-sdk-s3 (~> 1.91)
   better_errors (~> 2.9)
   binding_of_caller (~> 1.0)
   blurhash (~> 0.1)
@@ -723,7 +727,7 @@ DEPENDENCIES
   dotenv-rails (~> 2.7)
   ed25519 (~> 1.2)
   fabrication (~> 2.21)
-  faker (~> 2.16)
+  faker (~> 2.17)
   fast_blank (~> 1.0)
   fastimage
   fog-core (<= 2.1.0)
@@ -767,7 +771,7 @@ DEPENDENCIES
   parallel_tests (~> 3.5)
   parslet
   pg (~> 1.2)
-  pghero (~> 2.7)
+  pghero (~> 2.8)
   pkg-config (~> 1.4)
   pluck_each (~> 0.1.3)
   posix-spawn
@@ -793,7 +797,7 @@ DEPENDENCIES
   rspec-rails (~> 4.1)
   rspec-sidekiq (~> 3.1)
   rspec_junit_formatter (~> 0.4)
-  rubocop (~> 1.9)
+  rubocop (~> 1.11)
   rubocop-rails (~> 2.9)
   ruby-progressbar (~> 1.11)
   sanitize (~> 5.2)
@@ -801,7 +805,7 @@ DEPENDENCIES
   sidekiq (~> 6.1)
   sidekiq-bulk (~> 0.2.0)
   sidekiq-scheduler (~> 3.0)
-  sidekiq-unique-jobs (~> 6.0)
+  sidekiq-unique-jobs (~> 7.0)
   simple-navigation (~> 4.1)
   simple_form (~> 5.1)
   simplecov (~> 0.21)

app/models/account.rb

@@ -27,7 +27,6 @@
 #  header_file_size              :integer
 #  header_updated_at             :datetime
 #  avatar_remote_url             :string
-#  subscription_expires_at       :datetime
 #  locked                        :boolean          default(FALSE), not null
 #  header_remote_url             :string           default(""), not null
 #  last_webfingered_at           :datetime
@@ -55,6 +54,8 @@
 #
 
 class Account < ApplicationRecord
+  self.ignored_columns = %w(subscription_expires_at)
+
   USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i
 
@@ -97,7 +98,6 @@ class Account < ApplicationRecord
 
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
-  scope :expiring, ->(time) { remote.where.not(subscription_expires_at: nil).where('subscription_expires_at < ?', time) }
   scope :partitioned, -> { order(Arel.sql('row_number() over (partition by domain)')) }
   scope :silenced, -> { where.not(silenced_at: nil) }
   scope :suspended, -> { where.not(suspended_at: nil) }
@@ -194,10 +194,6 @@ class Account < ApplicationRecord
     "acct:#{local_username_and_domain}"
   end
 
-  def subscribed?
-    subscription_expires_at.present?
-  end
-
   def searchable?
     !(suspended? || moved?)
   end

app/models/concerns/expireable.rb

@@ -17,7 +17,7 @@ module Expireable
     end
 
     def expires_in=(interval)
-      self.expires_at = interval.to_i.seconds.from_now if interval.present?
+      self.expires_at = interval.present? ? interval.to_i.seconds.from_now : nil 
       @expires_in     = interval
     end
 

app/views/filters/_fields.html.haml

@@ -2,7 +2,7 @@
   .fields-row__column.fields-row__column-6.fields-group
     = f.input :phrase, as: :string, wrapper: :with_label, hint: false
   .fields-row__column.fields-row__column-6.fields-group
-    = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt')
+    = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, include_blank: I18n.t('invites.expires_in_prompt')
 
 .fields-group
   = f.input :context, wrapper: :with_block_label, collection: CustomFilter::VALID_CONTEXTS, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', label_method: lambda { |context| I18n.t("filters.contexts.#{context}") }, include_blank: false

app/views/settings/exports/show.html.haml

@@ -39,7 +39,7 @@
       %tr
         %th= t('exports.bookmarks')
         %td= number_with_delimiter @export.total_bookmarks
-        %td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv)
+        %td= table_link_to 'download', t('exports.csv'), settings_exports_bookmarks_path(format: :csv)
 
 %hr.spacer/
 

app/workers/scheduler/backup_cleanup_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::BackupCleanupScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     old_backups.reorder(nil).find_each(&:destroy!)

app/workers/scheduler/doorkeeper_cleanup_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::DoorkeeperCleanupScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all

app/workers/scheduler/email_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::EmailScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   FREQUENCY      = 7.days.freeze
   SIGN_IN_OFFSET = 1.day.freeze

app/workers/scheduler/feed_cleanup_scheduler.rb

@@ -4,7 +4,7 @@ class Scheduler::FeedCleanupScheduler
   include Sidekiq::Worker
   include Redisable
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     clean_home_feeds!

app/workers/scheduler/instance_refresh_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::InstanceRefreshScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     Instance.refresh

app/workers/scheduler/ip_cleanup_scheduler.rb

@@ -5,7 +5,7 @@ class Scheduler::IpCleanupScheduler
 
   IP_RETENTION_PERIOD = 1.year.freeze
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     clean_ip_columns!

app/workers/scheduler/media_cleanup_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::MediaCleanupScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     unattached_media.find_each(&:destroy)

app/workers/scheduler/pghero_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::PgheroScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     PgHero.capture_space_stats

app/workers/scheduler/scheduled_statuses_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::ScheduledStatusesScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     publish_scheduled_statuses!

app/workers/scheduler/trending_tags_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::TrendingTagsScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     TrendingTags.update! if Setting.trends

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -3,7 +3,7 @@
 class Scheduler::UserCleanupScheduler
   include Sidekiq::Worker
 
-  sidekiq_options lock: :until_executed, retry: 0
+  sidekiq_options retry: 0
 
   def perform
     clean_unconfirmed_accounts!

chart/Chart.lock

@@ -1,12 +1,12 @@
 dependencies:
 - name: elasticsearch
   repository: https://charts.bitnami.com/bitnami
-  version: 12.8.2
+  version: 14.2.3
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 8.10.14
 - name: redis
   repository: https://charts.bitnami.com/bitnami
   version: 10.9.0
-digest: sha256:9e423aa9a7a46f49e44f0411d61afd685eedf4475752a2b1a24a86a83b0752d0
+digest: sha256:9e3e7b987c6ffba9295a30b7fae2613fe680c2b1a1832ff5afb185414ce1898e
-generated: "2021-02-16T17:10:49.594247-08:00"
+generated: "2021-02-27T01:01:12.776919968Z"

chart/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.0
+version: 1.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -24,7 +24,7 @@ appVersion: 3.3.0
 
 dependencies:
   - name: elasticsearch
-    version: 12.8.2
+    version: 14.2.3
     repository: https://charts.bitnami.com/bitnami
     condition: elasticsearch.enabled
   - name: postgresql

config/initializers/sidekiq.rb

@@ -14,14 +14,30 @@ Sidekiq.configure_server do |config|
     chain.add SidekiqErrorHandler
   end
 
-  config.death_handlers << lambda do |job, _ex|
+  config.server_middleware do |chain|
-    digest = job['lock_digest']
+    chain.add SidekiqUniqueJobs::Middleware::Server
-    SidekiqUniqueJobs::Digests.delete_by_digest(digest) if digest
   end
+
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
+
+  SidekiqUniqueJobs::Server.configure(config)
 end
 
 Sidekiq.configure_client do |config|
   config.redis = redis_params
+
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
 end
 
 Sidekiq.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s)
+
+SidekiqUniqueJobs.configure do |config|
+  config.reaper          = :ruby
+  config.reaper_count    = 1000
+  config.reaper_interval = 600
+  config.reaper_timeout  = 150
+end

db/migrate/20200508212852_reset_unique_jobs_locks.rb

@@ -5,7 +5,7 @@ class ResetUniqueJobsLocks < ActiveRecord::Migration[5.2]
     # We do this to clean up unique job digests that were not properly
     # disposed of prior to https://github.com/tootsuite/mastodon/pull/13361
 
-    SidekiqUniqueJobs::Digests.delete_by_pattern('*', count: SidekiqUniqueJobs::Digests.count)
+    until SidekiqUniqueJobs::Digests.new.delete_by_pattern('*').nil?; end
   end
 
   def down; end

db/post_migrate/20210308133107_remove_subscription_expires_at_from_accounts.rb

@@ -0,0 +1,7 @@
+class RemoveSubscriptionExpiresAtFromAccounts < ActiveRecord::Migration[5.0]
+  def change
+    safety_assured do
+      remove_column :accounts, :subscription_expires_at, :datetime, null: true, default: nil
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_02_21_045109) do
+ActiveRecord::Schema.define(version: 2021_03_08_133107) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -166,7 +166,6 @@ ActiveRecord::Schema.define(version: 2021_02_21_045109) do
     t.integer "header_file_size"
     t.datetime "header_updated_at"
     t.string "avatar_remote_url"
-    t.datetime "subscription_expires_at"
     t.boolean "locked", default: false, null: false
     t.string "header_remote_url", default: "", null: false
     t.datetime "last_webfingered_at"

package.json

@@ -22,7 +22,7 @@
   },
   "browserslist": [
     "last 2 versions",
-    "IE >= 11",
+    "not IE 11",
     "iOS >= 9",
     "not dead"
   ],

spec/models/account_spec.rb

@@ -134,18 +134,6 @@ RSpec.describe Account, type: :model do
     end
   end
 
-  describe '#subscribed?' do
-    it 'returns false when no subscription expiration information is present' do
-      account = Fabricate(:account, subscription_expires_at: nil)
-      expect(account.subscribed?).to be false
-    end
-
-    it 'returns true when subscription expiration has been set' do
-      account = Fabricate(:account, subscription_expires_at: 30.days.from_now)
-      expect(account.subscribed?).to be true
-    end
-  end
-
   describe '#possibly_stale?' do
     let(:account) { Fabricate(:account, last_webfingered_at: last_webfingered_at) }
 
@@ -707,21 +695,6 @@ RSpec.describe Account, type: :model do
       end
     end
 
-    describe 'expiring' do
-      it 'returns remote accounts with followers whose subscription expiration date is past or not given' do
-        local = Fabricate(:account, domain: nil)
-        matches = [
-          { domain: 'remote', subscription_expires_at: '2000-01-01T00:00:00Z' },
-        ].map(&method(:Fabricate).curry(2).call(:account))
-        matches.each(&local.method(:follow!))
-        Fabricate(:account, domain: 'remote', subscription_expires_at: nil)
-        local.follow!(Fabricate(:account, domain: 'remote', subscription_expires_at: '2000-01-03T00:00:00Z'))
-        local.follow!(Fabricate(:account, domain: nil, subscription_expires_at: nil))
-
-        expect(Account.expiring('2000-01-02T00:00:00Z').recent).to eq matches.reverse
-      end
-    end
-
     describe 'remote' do
       it 'returns an array of accounts who have a domain' do
         account_1 = Fabricate(:account, domain: nil)