Add Ruby 3.0 support (#16046)

* Fix issues with POSIX::Spawn, Terrapin and Ruby 3.0

Also improve the Terrapin monkey-patch for the stderr/stdout issue.

* Fix keyword argument handling throughout the codebase

* Monkey-patch Paperclip to fix keyword arguments handling in validators

* Change validation_extensions to please CodeClimate

* Bump microformats from 4.2.1 to 4.3.1

* Allow Ruby 3.0

* Add Ruby 3.0 test target to CircleCI

* Add test for admin dashboard warnings

* Fix admin dashboard warnings on Ruby 3.0

.circleci/config.yml

@@ -129,6 +129,13 @@ jobs:
         environment: *ruby_environment
     <<: *install_ruby_dependencies
 
+  install-ruby3.0:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:3.0-buster-node
+        environment: *ruby_environment
+    <<: *install_ruby_dependencies
+
   build:
     <<: *defaults
     steps:
@@ -187,6 +194,18 @@ jobs:
       - image: circleci/redis:5-alpine
     <<: *test_steps
 
+  test-ruby3.0:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:3.0-buster-node
+        environment: *ruby_environment
+      - image: circleci/postgres:12.2
+        environment:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+      - image: circleci/redis:5-alpine
+    <<: *test_steps
+
   test-webui:
     <<: *defaults
     docker:
@@ -227,6 +246,10 @@ workflows:
           requires:
             - install
             - install-ruby2.7
+      - install-ruby3.0:
+          requires:
+            - install
+            - install-ruby2.7
       - build:
           requires:
             - install-ruby2.7
@@ -241,6 +264,10 @@ workflows:
           requires:
             - install-ruby2.6
             - build
+      - test-ruby3.0:
+          requires:
+            - install-ruby3.0
+            - build
       - test-webui:
           requires:
             - install

Gemfile

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 source 'https://rubygems.org'
-ruby '>= 2.5.0', '< 3.0.0'
+ruby '>= 2.5.0', '< 3.1.0'
 
 gem 'pkg-config', '~> 1.4'
 

Gemfile.lock

@@ -292,7 +292,7 @@ GEM
     ipaddress (0.8.3)
     iso-639 (0.3.5)
     jmespath (1.4.0)
-    json (2.3.1)
+    json (2.5.1)
     json-canonicalization (0.2.1)
     json-ld (3.1.9)
       htmlentities (~> 4.3)
@@ -344,7 +344,7 @@ GEM
       redis (>= 3.0.5)
     memory_profiler (1.0.0)
     method_source (1.0.0)
-    microformats (4.2.1)
+    microformats (4.3.1)
       json (~> 2.2)
       nokogiri (~> 1.10)
     mime-types (3.3.1)
@@ -354,7 +354,7 @@ GEM
       nokogiri (~> 1)
       rake
     mini_mime (1.0.3)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.14.4)
     msgpack (1.4.2)
     multi_json (1.15.0)

app/controllers/activitypub/outboxes_controller.rb

@@ -20,7 +20,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
   def outbox_presenter
     if page_requested?
       ActivityPub::CollectionPresenter.new(
-        id: outbox_url(page_params),
+        id: outbox_url(**page_params),
         type: :ordered,
         part_of: outbox_url,
         prev: prev_page,

app/controllers/api/v1/accounts_controller.rb

@@ -35,7 +35,7 @@ class Api::V1::AccountsController < Api::BaseController
     follow  = FollowService.new.call(current_user.account, @account, reblogs: params.key?(:reblogs) ? truthy_param?(:reblogs) : nil, notify: params.key?(:notify) ? truthy_param?(:notify) : nil, with_rate_limit: true)
     options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify? } }, requested_map: { @account.id => false } }
 
-    render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options)
+    render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(**options)
   end
 
   def block
@@ -70,7 +70,7 @@ class Api::V1::AccountsController < Api::BaseController
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id, options)
+    AccountRelationshipsPresenter.new([@account.id], current_user.account_id, **options)
   end
 
   def account_params

app/controllers/api/v1/follow_requests_controller.rb

@@ -29,7 +29,7 @@ class Api::V1::FollowRequestsController < Api::BaseController
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, options)
+    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, **options)
   end
 
   def load_accounts

app/models/session_activation.rb

@@ -44,7 +44,7 @@ class SessionActivation < ApplicationRecord
     end
 
     def activate(**options)
-      activation = create!(options)
+      activation = create!(**options)
       purge_old
       activation
     end

app/models/user.rb

@@ -370,15 +370,20 @@ class User < ApplicationRecord
 
   protected
 
-  def send_devise_notification(notification, *args)
+  def send_devise_notification(notification, *args, **kwargs)
     # This method can be called in `after_update` and `after_commit` hooks,
     # but we must make sure the mailer is actually called *after* commit,
     # otherwise it may work on stale data. To do this, figure out if we are
     # within a transaction.
+
+    # It seems like devise sends keyword arguments as a hash in the last
+    # positional argument
+    kwargs = args.pop if args.last.is_a?(Hash) && kwargs.empty?
+
     if ActiveRecord::Base.connection.current_transaction.try(:records)&.include?(self)
-      pending_devise_notifications << [notification, args]
+      pending_devise_notifications << [notification, args, kwargs]
     else
-      render_and_send_devise_message(notification, *args)
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
   end
 
@@ -389,8 +394,8 @@ class User < ApplicationRecord
   end
 
   def send_pending_devise_notifications
-    pending_devise_notifications.each do |notification, args|
+    pending_devise_notifications.each do |notification, args, kwargs|
-      render_and_send_devise_message(notification, *args)
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
 
     # Empty the pending notifications array because the
@@ -403,8 +408,8 @@ class User < ApplicationRecord
     @pending_devise_notifications ||= []
   end
 
-  def render_and_send_devise_message(notification, *args)
+  def render_and_send_devise_message(notification, *args, **kwargs)
-    devise_mailer.send(notification, self, *args).deliver_later
+    devise_mailer.send(notification, self, *args, **kwargs).deliver_later
   end
 
   def set_approved

app/views/admin/dashboard/index.html.haml

@@ -5,7 +5,7 @@
   .flash-message-stack
     - @system_checks.each do |message|
       .flash-message.warning
-        = t("admin.system_checks.#{message.key}.message_html", message.value ? { value: content_tag(:strong, message.value) } : {})
+        = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil)
         - if message.action
           = link_to t("admin.system_checks.#{message.key}.action"), message.action
 

app/workers/import/relationship_worker.rb

@@ -5,7 +5,7 @@ class Import::RelationshipWorker
 
   sidekiq_options queue: 'pull', retry: 8, dead: false
 
-  def perform(account_id, target_account_uri, relationship, options = {})
+  def perform(account_id, target_account_uri, relationship, options)
     from_account   = Account.find(account_id)
     target_domain  = domain(target_account_uri)
     target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
@@ -16,7 +16,7 @@ class Import::RelationshipWorker
     case relationship
     when 'follow'
       begin
-        FollowService.new.call(from_account, target_account, options)
+        FollowService.new.call(from_account, target_account, **options)
       rescue ActiveRecord::RecordInvalid
         raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count
       end
@@ -27,7 +27,7 @@ class Import::RelationshipWorker
     when 'unblock'
       UnblockService.new.call(from_account, target_account)
     when 'mute'
-      MuteService.new.call(from_account, target_account, options)
+      MuteService.new.call(from_account, target_account, **options)
     when 'unmute'
       UnmuteService.new.call(from_account, target_account)
     end

config/application.rb

@@ -10,6 +10,7 @@ require_relative '../lib/exceptions'
 require_relative '../lib/enumerable'
 require_relative '../lib/sanitize_ext/sanitize_config'
 require_relative '../lib/redis/namespace_extensions'
+require_relative '../lib/paperclip/validation_extensions'
 require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/media_type_spoof_detector_extensions'

config/initializers/session_store.rb

@@ -1,7 +1,6 @@
 # Be sure to restart your server when you modify this file.
 
-Rails.application.config.session_store :cookie_store, {
+Rails.application.config.session_store :cookie_store,
   key: '_mastodon_session',
   secure: (Rails.env.production? || ENV['LOCAL_HTTPS'] == 'true'),
-  same_site: :lax,
+  same_site: :lax
-}

lib/paperclip/validation_extensions.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility
+
+module Paperclip
+  module Validators
+    module AttachmentSizeValidatorExtensions
+      def validate_each(record, attr_name, _value)
+        base_attr_name = attr_name
+        attr_name = "#{attr_name}_file_size".to_sym
+        value = record.send(:read_attribute_for_validation, attr_name)
+
+        if value.present?
+          options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value|
+            option_value = option_value.call(record) if option_value.is_a?(Proc)
+            option_value = extract_option_value(option, option_value)
+
+            next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value)
+
+            error_message_key = options[:in] ? :in_between : option
+            [attr_name, base_attr_name].each do |error_attr_name|
+              record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge(
+                min: min_value_in_human_size(record),
+                max: max_value_in_human_size(record),
+                count: human_size(option_value)
+              ))
+            end
+          end
+        end
+      end
+    end
+
+    module AttachmentContentTypeValidatorExtensions
+      def mark_invalid(record, attribute, types)
+        record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') })
+      end
+    end
+
+    module AttachmentPresenceValidatorExtensions
+      def validate_each(record, attribute, _value)
+        if record.send("#{attribute}_file_name").blank?
+          record.errors.add(attribute, :blank, **options)
+        end
+      end
+    end
+
+    module AttachmentFileNameValidatorExtensions
+      def mark_invalid(record, attribute, patterns)
+        record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') })
+      end
+    end
+  end
+end
+
+Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions)
+Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions)
+Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions)
+Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions)

lib/terrapin/multi_pipe_extensions.rb

@@ -1,61 +1,64 @@
 # frozen_string_literal: false
-# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5
+
+require 'fcntl'
 
 module Terrapin
   module MultiPipeExtensions
+    def initialize
+      @stdout_in, @stdout_out = IO.pipe
+      @stderr_in, @stderr_out = IO.pipe
+
+      clear_nonblocking_flags!
+    end
+
+    def pipe_options
+      # Add some flags to explicitly close the other end of the pipes
+      { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close }
+    end
+
     def read
-      read_streams(@stdout_in, @stderr_in)
+      # While we are patching Terrapin, fix child process potentially getting stuck on writing
-    end
+      # to stderr.
 
-    def close_read
+      @stdout_output = +''
-      begin
+      @stderr_output = +''
-        @stdout_in.close
-      rescue IOError
-        # Do nothing
-      end
 
-      begin
+      fds_to_read = [@stdout_in, @stderr_in]
-        @stderr_in.close
+      until fds_to_read.empty?
-      rescue IOError
+        rs, = IO.select(fds_to_read)
-        # Do nothing
+
+        read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in)
+        read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in)
       end
     end
 
-    def read_streams(output, error)
+    private
-      @stdout_output = ''
-      @stderr_output = ''
 
-      read_fds = [output, error]
+    # @param [IO] io IO Stream to read until there is nothing to read
-
+    # @param [String] result Mutable string to which read values will be appended to
-      until read_fds.empty?
+    # @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF
-        to_read, = IO.select(read_fds)
+    def read_nonblocking!(io, result, fds_to_read)
-
+      while (partial_result = io.read_nonblock(8192))
-        if to_read.include?(output)
+        result << partial_result
-          @stdout_output << read_stream(output)
-          read_fds.delete(output) if output.closed?
-        end
-
-        if to_read.include?(error)
-          @stderr_output << read_stream(error)
-          read_fds.delete(error) if error.closed?
-        end
       end
+    rescue IO::WaitReadable
+      # Do nothing
+    rescue EOFError
+      fds_to_read.delete(io)
     end
 
-    def read_stream(io)
+    def clear_nonblocking_flags!
-      result = ''
+      # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as
+      # needed when calling fork/exec-related syscalls, but posix-spawn does
+      # not currently do that, so we need to do it manually for the time being
+      # so that the child process do not error out when the buffers are full.
+      stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL)
+      @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK
 
-      begin
+      stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL)
-        while (partial_result = io.read_nonblock(8192))
+      @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK
-          result << partial_result
+    rescue NameError, NotImplementedError, Errno::EINVAL
-        end
+      # Probably on windows, where pipes are blocking by default
-      rescue EOFError, Errno::EPIPE
-        io.close
-      rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN
-        # Do nothing
-      end
-
-      result
     end
   end
 end

spec/controllers/admin/dashboard_controller_spec.rb

@@ -3,9 +3,19 @@
 require 'rails_helper'
 
 describe Admin::DashboardController, type: :controller do
+  render_views
+
   describe 'GET #index' do
-    it 'returns 200' do
+    before do
+      allow(Admin::SystemCheck).to receive(:perform).and_return([
+        Admin::SystemCheck::Message.new(:database_schema_check),
+        Admin::SystemCheck::Message.new(:rules_check, nil, admin_rules_path),
+        Admin::SystemCheck::Message.new(:sidekiq_process_check, 'foo, bar'),
+      ])
       sign_in Fabricate(:user, admin: true)
+    end
+
+    it 'returns 200' do
       get :index
 
       expect(response).to have_http_status(200)

spec/mailers/notification_mailer_spec.rb

@@ -10,12 +10,12 @@ RSpec.describe NotificationMailer, type: :mailer do
     it 'renders subject localized for the locale of the receiver' do
       locale = %i(de en).sample
       receiver.update!(locale: locale)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale))
     end
 
     it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do
       receiver.update!(locale: nil)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale))
     end
   end
 

spec/mailers/user_mailer_spec.rb

@@ -9,12 +9,12 @@ describe UserMailer, type: :mailer do
     it 'renders subject localized for the locale of the receiver' do
       locale = I18n.available_locales.sample
       receiver.update!(locale: locale)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale))
     end
 
     it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do
       receiver.update!(locale: nil)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale))
     end
   end
 

spec/models/session_activation_spec.rb

@@ -74,13 +74,13 @@ RSpec.describe SessionActivation, type: :model do
     let(:options) { { user: Fabricate(:user), session_id: '1' } }
 
     it 'calls create! and purge_old' do
-      expect(described_class).to receive(:create!).with(options)
+      expect(described_class).to receive(:create!).with(**options)
       expect(described_class).to receive(:purge_old)
-      described_class.activate(options)
+      described_class.activate(**options)
     end
 
     it 'returns an instance of SessionActivation' do
-      expect(described_class.activate(options)).to be_kind_of SessionActivation
+      expect(described_class.activate(**options)).to be_kind_of SessionActivation
     end
   end
 

spec/presenters/account_relationships_presenter_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe AccountRelationshipsPresenter do
       allow(Account).to receive(:domain_blocking_map).with(account_ids, current_account_id).and_return(default_map)
     end
 
-    let(:presenter)          { AccountRelationshipsPresenter.new(account_ids, current_account_id, options) }
+    let(:presenter)          { AccountRelationshipsPresenter.new(account_ids, current_account_id, **options) }
     let(:current_account_id) { Fabricate(:account).id }
     let(:account_ids)        { [Fabricate(:account).id] }
     let(:default_map)        { { 1 => true } }