Merge remote-tracking branch 'up/main'

.circleci/config.yml

@@ -19,7 +19,7 @@ executors:
           DB_USER: root
           DISABLE_SIMPLECOV: true
           RAILS_ENV: test
-      - image: cimg/postgres:12.7
+      - image: cimg/postgres:14.0
         environment:
           POSTGRES_USER: root
           POSTGRES_HOST_AUTH_METHOD: trust

.env.production.sample

@@ -4,6 +4,12 @@
 # not demonstrate all available configuration options. Please look at
 # https://docs.joinmastodon.org/admin/config/ for the full documentation.
 
+# Note that this file accepts slightly different syntax depending on whether
+# you are using `docker-compose` or not. In particular, if you use
+# `docker-compose`, the value of each declared variable will be taken verbatim,
+# including surrounding quotes.
+# See: https://github.com/mastodon/mastodon/issues/16895
+
 # Federation
 # ----------
 # This identifies your server and cannot be changed safely later
@@ -28,6 +34,9 @@ DB_PORT=5432
 ES_ENABLED=true
 ES_HOST=localhost
 ES_PORT=9200
+# Authentication for ES (optional)
+ES_USER=elastic
+ES_PASS=password
 
 # Secrets
 # -------

.github/workflows/build-image.yml

@@ -0,0 +1,34 @@
+name: Build container image
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "*"
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/metadata-action@v3
+        id: meta
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/mastodon
+          flavor: |
+            latest=true
+          tags: |
+            type=edge,branch=main
+            type=semver,pattern={{ raw }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest
+          cache-to: type=inline

CHANGELOG.md

@@ -3,6 +3,54 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [3.4.3] - 2021-11-06
+### Fixed
+
+- Fix login being broken due to inaccurately applied backport fix in 3.4.2 ([Gargron](https://github.com/mastodon/mastodon/commit/5c47a18c8df3231aa25c6d1f140a71a7fac9cbf9))
+
+## [3.4.2] - 2021-11-06
+### Added
+
+- Add `configuration` attribute to `GET /api/v1/instance` ([Gargron](https://github.com/mastodon/mastodon/pull/16485))
+
+### Fixed
+
+- Fix handling of back button with modal windows in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16499))
+- Fix pop-in player when author has long username in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16468))
+- Fix crash when a status with a playing video gets deleted in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16384))
+- Fix crash with Microsoft Translate in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16525))
+- Fix PWA not being usable from alternate domains ([HolgerHuo](https://github.com/mastodon/mastodon/pull/16714))
+- Fix locale-specific number rounding errors ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16469))
+- Fix scheduling a status decreasing status count ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16791))
+- Fix user's canonical email address being blocked when user deletes own account ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16503))
+- Fix not being able to suspend users that already have their canonical e-mail blocked ([Gargron](https://github.com/mastodon/mastodon/pull/16455))
+- Fix anonymous access to outbox not being cached by the reverse proxy ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16458))
+- Fix followers synchronization mechanism not working when URI has empty path ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16744))
+- Fix serialization of counts in REST API when user hides their network ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16418))
+- Fix inefficiencies in auto-linking code ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16506))
+- Fix `tootctl self-destruct` not sending delete activities for recently-suspended accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16688))
+- Fix suspicious sign-in e-mail text being out of date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16690))
+- Fix some frameworks being unnecessarily loaded ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16725))
+- Fix canonical e-mail blocks missing foreign key constraints ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16448))
+- Fix inconsistent order on account's statuses page in admin UI ([tribela](https://github.com/mastodon/mastodon/pull/16937))
+- Fix media from blocked domains being redownloaded by `tootctl media refresh` ([tribela](https://github.com/mastodon/mastodon/pull/16914))
+- Fix `mastodon:setup` generated env-file syntax ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16896))
+- Fix link previews being incorrectly generated from earlier links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16885))
+- Fix wrong `to`/`cc` values for remote groups in ActivityPub ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16700))
+- Fix mentions with non-ascii TLDs not being processed ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16689))
+- Fix authentication failures halfway through a sign-in attempt ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16792))
+- Fix suspended accounts statuses being merged back into timelines ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16628))
+- Fix crash when encountering invalid account fields ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16598))
+- Fix invalid blurhash handling for remote activities ([noellabo](https://github.com/mastodon/mastodon/pull/16583))
+- Fix newlines being added to accout notes when an account moves ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16415), [noellabo](https://github.com/mastodon/mastodon/pull/16576))
+- Fix crash when creating an announcement with links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16941))
+- Fix logging out from one browser logging out all other sessions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943))
+
+### Security
+
+- Fix user notes not having a length limit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16942))
+- Fix revoking a specific session not working ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943))
+
 ## [3.4.1] - 2021-06-03
 ### Added
 

Gemfile

@@ -17,7 +17,7 @@ gem 'makara', '~> 0.5'
 gem 'pghero', '~> 2.8'
 gem 'dotenv-rails', '~> 2.7'
 
-gem 'aws-sdk-s3', '~> 1.103', require: false
+gem 'aws-sdk-s3', '~> 1.104', require: false
 gem 'fog-core', '<= 2.1.0'
 gem 'fog-openstack', '~> 0.3', require: false
 gem 'kt-paperclip', '~> 7.0'
@@ -112,7 +112,7 @@ group :production, :test do
 end
 
 group :test do
-  gem 'capybara', '~> 3.35'
+  gem 'capybara', '~> 3.36'
   gem 'climate_control', '~> 0.2'
   gem 'faker', '~> 2.19'
   gem 'microformats', '~> 4.2'

Gemfile.lock

@@ -79,17 +79,17 @@ GEM
       encryptor (~> 3.0.0)
     awrence (1.1.1)
     aws-eventstream (1.2.0)
-    aws-partitions (1.503.0)
+    aws-partitions (1.519.0)
-    aws-sdk-core (3.121.0)
+    aws-sdk-core (3.121.3)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.48.0)
+    aws-sdk-kms (1.50.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+      aws-sdk-core (~> 3, >= 3.121.2)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.103.0)
+    aws-sdk-s3 (1.104.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+      aws-sdk-core (~> 3, >= 3.121.2)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.4.0)
@@ -106,7 +106,7 @@ GEM
       ffi (~> 1.14)
     bootsnap (1.9.1)
       msgpack (~> 1.0)
-    brakeman (5.1.1)
+    brakeman (5.1.2)
     browser (4.2.0)
     brpoplpush-redis_script (0.1.2)
       concurrent-ruby (~> 1.0, >= 1.0.5)
@@ -134,8 +134,9 @@ GEM
       sshkit (~> 1.3)
     capistrano-yarn (2.0.2)
       capistrano (~> 3.0)
-    capybara (3.35.3)
+    capybara (3.36.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
       rack (>= 1.6.0)
@@ -349,6 +350,7 @@ GEM
     marcel (1.0.1)
     mario-redis-lock (1.2.1)
       redis (>= 3.0.5)
+    matrix (0.4.2)
     memory_profiler (1.0.0)
     method_source (1.0.0)
     microformats (4.3.1)
@@ -427,7 +429,7 @@ GEM
     pundit (2.1.1)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (2.2.3)
     rack-attack (6.5.0)
       rack (>= 1.0, < 3)
@@ -517,7 +519,7 @@ GEM
     rspec-support (3.10.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (1.22.1)
+    rubocop (1.22.3)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
@@ -620,7 +622,7 @@ GEM
       unf (~> 0.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.4)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
@@ -668,7 +670,7 @@ DEPENDENCIES
   active_record_query_trace (~> 1.8)
   addressable (~> 2.8)
   annotate (~> 3.1)
-  aws-sdk-s3 (~> 1.103)
+  aws-sdk-s3 (~> 1.104)
   better_errors (~> 2.9)
   binding_of_caller (~> 1.0)
   blurhash (~> 0.1)
@@ -681,7 +683,7 @@ DEPENDENCIES
   capistrano-rails (~> 1.6)
   capistrano-rbenv (~> 2.2)
   capistrano-yarn (~> 2.0)
-  capybara (~> 3.35)
+  capybara (~> 3.36)
   charlock_holmes (~> 0.7.7)
   chewy (~> 5.2)
   cld3 (~> 3.4.2)

app/controllers/admin/statuses_controller.rb

@@ -14,7 +14,7 @@ module Admin
       @statuses = @account.statuses.where(visibility: [:public, :unlisted])
 
       if params[:media]
-        @statuses.merge!(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id))
+        @statuses.merge!(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)).reorder('statuses.id desc')
       end
 
       @statuses = @statuses.preload(:media_attachments, :mentions).page(params[:page]).per(PER_PAGE)

app/controllers/auth/passwords_controller.rb

@@ -10,7 +10,6 @@ class Auth::PasswordsController < Devise::PasswordsController
     super do |resource|
       if resource.errors.empty?
         resource.session_activations.destroy_all
-        resource.forget_me!
       end
     end
   end

app/controllers/auth/registrations_controller.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::RegistrationsController < Devise::RegistrationsController
-  include Devise::Controllers::Rememberable
   include RegistrationSpamConcern
 
   layout :determine_layout
@@ -30,8 +29,6 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super do |resource|
       if resource.saved_change_to_encrypted_password?
         resource.clear_other_sessions(current_session.session_id)
-        resource.forget_me!
-        remember_me(resource)
       end
     end
   end

app/controllers/auth/sessions_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
-  include Devise::Controllers::Rememberable
-
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -150,7 +148,6 @@ class Auth::SessionsController < Devise::SessionsController
     clear_attempt_from_session
 
     user.update_sign_in!(request, new_sign_in: true)
-    remember_me(user)
     sign_in(user)
     flash.delete(:notice)
 

app/javascript/mastodon/features/notifications/components/column_settings.js

@@ -26,7 +26,8 @@ export default class ColumnSettings extends React.PureComponent {
   render () {
     const { settings, pushSettings, onChange, onClear, alertsEnabled, browserSupport, browserPermission, onRequestNotificationPermission } = this.props;
 
-    const filterShowStr = <FormattedMessage id='notifications.column_settings.filter_bar.show' defaultMessage='Show' />;
+    const unreadMarkersShowStr = <FormattedMessage id='notifications.column_settings.unread_notifications.highlight' defaultMessage='Highlight unread notifications' />;
+    const filterBarShowStr = <FormattedMessage id='notifications.column_settings.filter_bar.show_bar' defaultMessage='Show filter bar' />;
     const filterAdvancedStr = <FormattedMessage id='notifications.column_settings.filter_bar.advanced' defaultMessage='Display all categories' />;
     const alertStr = <FormattedMessage id='notifications.column_settings.alert' defaultMessage='Desktop notifications' />;
     const showStr = <FormattedMessage id='notifications.column_settings.show' defaultMessage='Show in column' />;
@@ -57,11 +58,11 @@ export default class ColumnSettings extends React.PureComponent {
 
         <div role='group' aria-labelledby='notifications-unread-markers'>
           <span id='notifications-unread-markers' className='column-settings__section'>
-            <FormattedMessage id='notifications.column_settings.unread_markers.category' defaultMessage='Unread notification markers' />
+            <FormattedMessage id='notifications.column_settings.unread_notifications.category' defaultMessage='Unread notifications' />
           </span>
 
           <div className='column-settings__row'>
-            <SettingToggle id='unread-notification-markers' prefix='notifications' settings={settings} settingPath={['showUnread']} onChange={onChange} label={filterShowStr} />
+            <SettingToggle id='unread-notification-markers' prefix='notifications' settings={settings} settingPath={['showUnread']} onChange={onChange} label={unreadMarkersShowStr} />
           </div>
         </div>
 
@@ -71,7 +72,7 @@ export default class ColumnSettings extends React.PureComponent {
           </span>
 
           <div className='column-settings__row'>
-            <SettingToggle id='show-filter-bar' prefix='notifications' settings={settings} settingPath={['quickFilter', 'show']} onChange={onChange} label={filterShowStr} />
+            <SettingToggle id='show-filter-bar' prefix='notifications' settings={settings} settingPath={['quickFilter', 'show']} onChange={onChange} label={filterBarShowStr} />
             <SettingToggle id='show-filter-bar' prefix='notifications' settings={settings} settingPath={['quickFilter', 'advanced']} onChange={onChange} label={filterAdvancedStr} />
           </div>
         </div>

app/javascript/mastodon/locales/defaultMessages.json

@@ -2264,8 +2264,12 @@
   {
     "descriptors": [
       {
-        "defaultMessage": "Show",
+        "defaultMessage": "Highlight unread notifications",
-        "id": "notifications.column_settings.filter_bar.show"
+        "id": "notifications.column_settings.unread_notifications.highlight"
+      },
+      {
+        "defaultMessage": "Show filter bar",
+        "id": "notifications.column_settings.filter_bar.show_bar"
       },
       {
         "defaultMessage": "Display all categories",
@@ -2296,8 +2300,8 @@
         "id": "notifications.permission_required"
       },
       {
-        "defaultMessage": "Unread notification markers",
+        "defaultMessage": "Unread notifications",
-        "id": "notifications.column_settings.unread_markers.category"
+        "id": "notifications.column_settings.unread_notifications.category"
       },
       {
         "defaultMessage": "Quick filter bar",

app/javascript/mastodon/locales/en.json

@@ -312,7 +312,7 @@
   "notifications.column_settings.favourite": "Favourites:",
   "notifications.column_settings.filter_bar.advanced": "Display all categories",
   "notifications.column_settings.filter_bar.category": "Quick filter bar",
-  "notifications.column_settings.filter_bar.show": "Show",
+  "notifications.column_settings.filter_bar.show_bar": "Show filter bar",
   "notifications.column_settings.follow": "New followers:",
   "notifications.column_settings.follow_request": "New follow requests:",
   "notifications.column_settings.mention": "Mentions:",
@@ -322,7 +322,8 @@
   "notifications.column_settings.show": "Show in column",
   "notifications.column_settings.sound": "Play sound",
   "notifications.column_settings.status": "New posts:",
-  "notifications.column_settings.unread_markers.category": "Unread notification markers",
+  "notifications.column_settings.unread_notifications.category": "Unread notifications",
+  "notifications.column_settings.unread_notifications.highlight": "Highlight unread notifications",
   "notifications.filter.all": "All",
   "notifications.filter.boosts": "Boosts",
   "notifications.filter.favourites": "Favourites",

app/lib/link_details_extractor.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+class LinkDetailsExtractor
+  include ActionView::Helpers::TagHelper
+
+  class StructuredData
+    def initialize(data)
+      @data = data
+    end
+
+    def headline
+      json['headline']
+    end
+
+    def description
+      json['description']
+    end
+
+    def image
+      obj = first_of_value(json['image'])
+
+      return obj['url'] if obj.is_a?(Hash)
+
+      obj
+    end
+
+    def date_published
+      json['datePublished']
+    end
+
+    def date_modified
+      json['dateModified']
+    end
+
+    def author_name
+      author['name']
+    end
+
+    def author_url
+      author['url']
+    end
+
+    def publisher_name
+      publisher['name']
+    end
+
+    private
+
+    def author
+      first_of_value(json['author']) || {}
+    end
+
+    def publisher
+      first_of_value(json['publisher']) || {}
+    end
+
+    def first_of_value(arr)
+      arr.is_a?(Array) ? arr.first : arr
+    end
+
+    def json
+      @json ||= Oj.load(@data)
+    end
+  end
+
+  def initialize(original_url, html, html_charset)
+    @original_url = Addressable::URI.parse(original_url)
+    @html         = html
+    @html_charset = html_charset
+  end
+
+  def to_preview_card_attributes
+    {
+      title: title || '',
+      description: description || '',
+      image_remote_url: image,
+      type: type,
+      width: width || 0,
+      height: height || 0,
+      html: html || '',
+      provider_name: provider_name || '',
+      provider_url: provider_url || '',
+      author_name: author_name || '',
+      author_url: author_url || '',
+      embed_url: embed_url || '',
+    }
+  end
+
+  def type
+    player_url.present? ? :video : :link
+  end
+
+  def html
+    player_url.present? ? content_tag(:iframe, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
+  end
+
+  def width
+    opengraph_tag('twitter:player:width')
+  end
+
+  def height
+    opengraph_tag('twitter:player:height')
+  end
+
+  def title
+    structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first
+  end
+
+  def description
+    structured_data&.description || opengraph_tag('og:description') || meta_tag('description')
+  end
+
+  def image
+    valid_url_or_nil(opengraph_tag('og:image'))
+  end
+
+  def canonical_url
+    valid_url_or_nil(opengraph_tag('og:url') || link_tag('canonical'), same_origin_only: true) || @original_url.to_s
+  end
+
+  def provider_name
+    structured_data&.publisher_name || opengraph_tag('og:site_name')
+  end
+
+  def provider_url
+    valid_url_or_nil(host_to_url(opengraph_tag('og:site')))
+  end
+
+  def author_name
+    structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username')
+  end
+
+  def author_url
+    structured_data&.author_url
+  end
+
+  def embed_url
+    valid_url_or_nil(opengraph_tag('twitter:player:stream'))
+  end
+
+  private
+
+  def player_url
+    valid_url_or_nil(opengraph_tag('twitter:player'))
+  end
+
+  def host_to_url(str)
+    return if str.blank?
+
+    str.start_with?(/https?:\/\//) ? str : "http://#{str}"
+  end
+
+  def valid_url_or_nil(str, same_origin_only: false)
+    return if str.blank?
+
+    url = @original_url + Addressable::URI.parse(str)
+
+    return if url.host.blank? || !%w(http https).include?(url.scheme) || (same_origin_only && url.host != @original_url.host)
+
+    url.to_s
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  def link_tag(name)
+    document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first
+  end
+
+  def opengraph_tag(name)
+    document.xpath("//meta[@property=\"#{name}\" or @name=\"#{name}\"]").map { |meta| meta['content'] }.first
+  end
+
+  def meta_tag(name)
+    document.xpath("//meta[@name=\"#{name}\"]").map { |meta| meta['content'] }.first
+  end
+
+  def structured_data
+    @structured_data ||= begin
+      json_ld = document.xpath('//script[@type="application/ld+json"]').map(&:content).first
+      json_ld.present? ? StructuredData.new(json_ld) : nil
+    end
+  end
+
+  def document
+    @document ||= Nokogiri::HTML(@html, nil, encoding)
+  end
+
+  def encoding
+    @encoding ||= begin
+      guess = detector.detect(@html, @html_charset)
+      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+    end
+  end
+
+  def detector
+    @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
+      detector.strip_tags = true
+    end
+  end
+end

app/lib/request.rb

@@ -94,7 +94,7 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 2)
+      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
     end
   end
 

app/models/account_note.rb

@@ -17,4 +17,5 @@ class AccountNote < ApplicationRecord
   belongs_to :target_account, class_name: 'Account'
 
   validates :account_id, uniqueness: { scope: :target_account_id }
+  validates :comment, length: { maximum: 2_000 }
 end

app/models/status.rb

@@ -338,7 +338,7 @@ class Status < ApplicationRecord
     def from_text(text)
       return [] if text.blank?
 
-      text.scan(FetchLinkCardService::URL_PATTERN).map(&:first).uniq.filter_map do |url|
+      text.scan(FetchLinkCardService::URL_PATTERN).map(&:second).uniq.filter_map do |url|
         status = begin
           if TagManager.instance.local_url?(url)
             ActivityPub::TagManager.instance.uri_to_resource(url, Status)

app/models/user.rb

@@ -64,7 +64,7 @@ class User < ApplicationRecord
   devise :two_factor_backupable,
          otp_number_of_backup_codes: 10
 
-  devise :registerable, :recoverable, :rememberable, :validatable,
+  devise :registerable, :recoverable, :validatable,
          :confirmable
 
   include Omniauthable

app/services/fetch_link_card_service.rb

@@ -14,11 +14,11 @@ class FetchLinkCardService < BaseService
 
   def call(status)
     @status       = status
-    @url    = parse_urls
+    @original_url = parse_urls
 
-    return if @url.nil? || @status.preview_cards.any?
+    return if @original_url.nil? || @status.preview_cards.any?
 
-    @url = @url.to_s
+    @url = @original_url.to_s
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
@@ -31,7 +31,7 @@ class FetchLinkCardService < BaseService
 
     attach_card if @card&.persisted?
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
-    Rails.logger.debug "Error fetching link #{@url}: #{e}"
+    Rails.logger.debug "Error fetching link #{@original_url}: #{e}"
     nil
   end
 
@@ -47,6 +47,12 @@ class FetchLinkCardService < BaseService
     return @html if defined?(@html)
 
     Request.new(:get, @url).add_headers('Accept' => 'text/html', 'User-Agent' => Mastodon::Version.user_agent + ' Bot').perform do |res|
+      # We follow redirects, and ideally we want to save the preview card for
+      # the destination URL and not any link shortener in-between, so here
+      # we set the URL to the one of the last response in the redirect chain
+      @url  = res.request.uri.to_s.to_s
+      @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url
+
       if res.code == 200 && res.mime_type == 'text/html'
         @html_charset = res.charset
         @html = res.body_with_limit
@@ -63,12 +69,15 @@ class FetchLinkCardService < BaseService
   end
 
   def parse_urls
+    urls = begin
       if @status.local?
-      urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[1]).normalize }
+        @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[1]).normalize }
       else
-      html  = Nokogiri::HTML(@status.text)
+        document = Nokogiri::HTML(@status.text)
-      links = html.css('a')
+        links    = document.css('a')
-      urls  = links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
+
+        links.filter_map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.filter_map(&:normalize)
+      end
     end
 
     urls.reject { |uri| bad_url?(uri) }.first
@@ -79,18 +88,16 @@ class FetchLinkCardService < BaseService
     uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
   end
 
-  # rubocop:disable Naming/MethodParameterName
+  def mention_link?(anchor)
-  def mention_link?(a)
     @status.mentions.any? do |mention|
-      a['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
+      anchor['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
     end
   end
 
-  def skip_link?(a)
+  def skip_link?(anchor)
     # Avoid links for hashtags and mentions (microformats)
-    a['rel']&.include?('tag') || a['class']&.match?(/u-url|h-card/) || mention_link?(a)
+    anchor['rel']&.include?('tag') || anchor['class']&.match?(/u-url|h-card/) || mention_link?(anchor)
   end
-  # rubocop:enable Naming/MethodParameterName
 
   def attempt_oembed
     service         = FetchOEmbedService.new
@@ -139,42 +146,14 @@ class FetchLinkCardService < BaseService
   def attempt_opengraph
     return if html.nil?
 
-    detector = CharlockHolmes::EncodingDetector.new
+    link_details_extractor = LinkDetailsExtractor.new(@url, @html, @html_charset)
-    detector.strip_tags = true
 
-    guess      = detector.detect(@html, @html_charset)
+    @card = PreviewCard.find_or_initialize_by(url: link_details_extractor.canonical_url) if link_details_extractor.canonical_url != @card.url
-    encoding   = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+    @card.assign_attributes(link_details_extractor.to_preview_card_attributes)
-    page       = Nokogiri::HTML(@html, nil, encoding)
+    @card.save_with_optional_image! unless @card.title.blank? && @card.html.blank?
-    player_url = meta_property(page, 'twitter:player')
-
-    if player_url && !bad_url?(Addressable::URI.parse(player_url))
-      @card.type   = :video
-      @card.width  = meta_property(page, 'twitter:player:width') || 0
-      @card.height = meta_property(page, 'twitter:player:height') || 0
-      @card.html   = content_tag(:iframe, nil, src: player_url,
-                                               width: @card.width,
-                                               height: @card.height,
-                                               allowtransparency: 'true',
-                                               scrolling: 'no',
-                                               frameborder: '0')
-    else
-      @card.type = :link
-    end
-
-    @card.title            = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
-    @card.description      = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
-    @card.image_remote_url = (Addressable::URI.parse(@url) + meta_property(page, 'og:image')).to_s if meta_property(page, 'og:image')
-
-    return if @card.title.blank? && @card.html.blank?
-
-    @card.save_with_optional_image!
-  end
-
-  def meta_property(page, property)
-    page.at_xpath("//meta[contains(concat(' ', normalize-space(@property), ' '), ' #{property} ')]")&.attribute('content')&.value || page.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
   end
 
   def lock_options
-    { redis: Redis.current, key: "fetch:#{@url}", autorelease: 15.minutes.seconds }
+    { redis: Redis.current, key: "fetch:#{@original_url}", autorelease: 15.minutes.seconds }
   end
 end

app/workers/move_worker.rb

@@ -53,10 +53,16 @@ class MoveWorker
 
       new_note = AccountNote.find_by(account: note.account, target_account: @target_account)
       if new_note.nil?
+        begin
           AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join("\n"))
+        rescue ActiveRecord::RecordInvalid
+          AccountNote.create!(account: note.account, target_account: @target_account, comment: note.comment)
+        end
       else
         new_note.update!(comment: [text, note.comment, "\n", new_note.comment].join("\n"))
       end
+    rescue ActiveRecord::RecordInvalid
+      nil
     rescue => e
       @deferred_error = e
     end

config/initializers/chewy.rb

@@ -1,6 +1,8 @@
 enabled         = ENV['ES_ENABLED'] == 'true'
 host            = ENV.fetch('ES_HOST') { 'localhost' }
 port            = ENV.fetch('ES_PORT') { 9200 }
+user            = ENV.fetch('ES_USER') { nil }
+password        = ENV.fetch('ES_PASS') { nil }
 fallback_prefix = ENV.fetch('REDIS_NAMESPACE') { nil }
 prefix          = ENV.fetch('ES_PREFIX') { fallback_prefix }
 
@@ -9,6 +11,8 @@ Chewy.settings = {
   prefix: prefix,
   enabled: enabled,
   journal: false,
+  user: user,
+  password: password,
   sidekiq: { queue: 'pull' },
 }
 

config/initializers/devise.rb

@@ -1,3 +1,5 @@
+require 'devise/strategies/authenticatable'
+
 Warden::Manager.after_set_user except: :fetch do |user, warden|
   if user.session_active?(warden.cookies.signed['_session_id'] || warden.raw_session['auth_id'])
     session_id = warden.cookies.signed['_session_id'] || warden.raw_session['auth_id']
@@ -72,17 +74,48 @@ module Devise
   mattr_accessor :ldap_uid_conversion_replace
   @@ldap_uid_conversion_replace = nil
 
-  class Strategies::PamAuthenticatable
+  module Strategies
+    class PamAuthenticatable
       def valid?
         super && ::Devise.pam_authentication
       end
     end
+
+    class SessionActivationRememberable < Authenticatable
+      def valid?
+        @session_cookie = nil
+        session_cookie.present?
+      end
+
+      def authenticate!
+        resource = SessionActivation.find_by(session_id: session_cookie)&.user
+
+        unless resource
+          cookies.delete('_session_id')
+          return pass
+        end
+
+        if validate(resource)
+          success!(resource)
+        end
+      end
+
+      private
+
+      def session_cookie
+        @session_cookie ||= cookies.signed['_session_id']
+      end
+    end
+  end
 end
 
+Warden::Strategies.add(:session_activation_rememberable, Devise::Strategies::SessionActivationRememberable)
+
 Devise.setup do |config|
   config.warden do |manager|
     manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication
     manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable  if Devise.pam_authentication
+    manager.default_strategies(scope: :user).unshift :session_activation_rememberable
     manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
     manager.default_strategies(scope: :user).unshift :two_factor_backupable
   end

dist/mastodon-sidekiq.service

@@ -13,6 +13,9 @@ Environment="LD_PRELOAD=libjemalloc.so"
 ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 25
 TimeoutSec=15
 Restart=always
+# Proc filesystem
+ProcSubset=pid
+ProtectProc=invisible
 # Capabilities
 CapabilityBoundingSet=
 # Security
@@ -35,11 +38,15 @@ RestrictNamespaces=true
 LockPersonality=true
 RestrictRealtime=true
 RestrictSUIDSGID=true
+RemoveIPC=true
 PrivateMounts=true
 ProtectClock=true
 # System Call Filtering
 SystemCallArchitectures=native
-SystemCallFilter=~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @reboot @setuid @swap
+SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileged @setuid
+SystemCallFilter=@chown
+SystemCallFilter=pipe
+SystemCallFilter=pipe2
 
 [Install]
 WantedBy=multi-user.target

dist/mastodon-streaming.service

@@ -12,6 +12,9 @@ Environment="STREAMING_CLUSTER_NUM=1"
 ExecStart=/usr/bin/node ./streaming
 TimeoutSec=15
 Restart=always
+# Proc filesystem
+ProcSubset=pid
+ProtectProc=invisible
 # Capabilities
 CapabilityBoundingSet=
 # Security
@@ -34,11 +37,14 @@ RestrictNamespaces=true
 LockPersonality=true
 RestrictRealtime=true
 RestrictSUIDSGID=true
+RemoveIPC=true
 PrivateMounts=true
 ProtectClock=true
 # System Call Filtering
 SystemCallArchitectures=native
-SystemCallFilter=~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @privileged @raw-io @reboot @resources @setuid @swap
+SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @memlock @mount @obsolete @privileged @resources @setuid
+SystemCallFilter=pipe
+SystemCallFilter=pipe2
 
 [Install]
 WantedBy=multi-user.target

dist/mastodon-web.service

@@ -13,6 +13,9 @@ ExecStart=/home/mastodon/.rbenv/shims/bundle exec puma -C config/puma.rb
 ExecReload=/bin/kill -SIGUSR1 $MAINPID
 TimeoutSec=15
 Restart=always
+# Proc filesystem
+ProcSubset=pid
+ProtectProc=invisible
 # Capabilities
 CapabilityBoundingSet=
 # Security
@@ -35,11 +38,15 @@ RestrictNamespaces=true
 LockPersonality=true
 RestrictRealtime=true
 RestrictSUIDSGID=true
+RemoveIPC=true
 PrivateMounts=true
 ProtectClock=true
 # System Call Filtering
 SystemCallArchitectures=native
-SystemCallFilter=~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @reboot @resources @setuid @swap
+SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileged @setuid
+SystemCallFilter=@chown
+SystemCallFilter=pipe
+SystemCallFilter=pipe2
 
 [Install]
 WantedBy=multi-user.target

lib/mastodon/media_cli.rb

@@ -230,6 +230,7 @@ module Mastodon
 
       processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
         next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
+        next if DomainBlock.reject_media?(media_attachment.account.domain)
 
         unless options[:dry_run]
           media_attachment.reset_file!

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      1
+      3
     end
 
     def flags

lib/tasks/mastodon.rake

@@ -333,8 +333,12 @@ namespace :mastodon do
       prompt.say 'This configuration will be written to .env.production'
 
       if prompt.yes?('Save configuration?')
+        incompatible_syntax = false
+
         env_contents = env.each_pair.map do |key, value|
           if value.is_a?(String) && value =~ /[\s\#\\"]/
+            incompatible_syntax = true
+
             if value =~ /[']/
               value = value.to_s.gsub(/[\\"\$]/) { |x| "\\#{x}" }
               "#{key}=\"#{value}\""
@@ -346,12 +350,19 @@ namespace :mastodon do
           end
         end.join("\n")
 
-        File.write(Rails.root.join('.env.production'), "# Generated with mastodon:setup on #{Time.now.utc}\n\n" + env_contents + "\n")
+        generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup
+
+        if incompatible_syntax
+          generated_header << "# Some variables in this file will be interpreted differently whether you are\n"
+          generated_header << "# using docker-compose or not.\n\n"
+        end
+
+        File.write(Rails.root.join('.env.production'), "#{generated_header}#{env_contents}\n")
 
         if using_docker
           prompt.ok 'Below is your configuration, save it to an .env.production file outside Docker:'
           prompt.say "\n"
-          prompt.say File.read(Rails.root.join('.env.production'))
+          prompt.say "#{generated_header}#{env.each_pair.map { |key, value| "#{key}=#{value}" }.join("\n")}"
           prompt.say "\n"
           prompt.ok 'It is also saved within this container so you can proceed with this wizard.'
         end

package.json

@@ -60,21 +60,21 @@
   },
   "private": true,
   "dependencies": {
-    "@babel/core": "^7.15.8",
+    "@babel/core": "^7.16.0",
-    "@babel/plugin-proposal-decorators": "^7.15.8",
+    "@babel/plugin-proposal-decorators": "^7.16.0",
-    "@babel/plugin-transform-react-inline-elements": "^7.14.5",
+    "@babel/plugin-transform-react-inline-elements": "^7.16.0",
-    "@babel/plugin-transform-runtime": "^7.15.8",
+    "@babel/plugin-transform-runtime": "^7.16.0",
-    "@babel/preset-env": "^7.15.8",
+    "@babel/preset-env": "^7.16.0",
-    "@babel/preset-react": "^7.14.5",
+    "@babel/preset-react": "^7.16.0",
-    "@babel/runtime": "^7.15.4",
+    "@babel/runtime": "^7.16.0",
     "@gamestdio/websocket": "^0.3.2",
     "@github/webauthn-json": "^0.5.7",
     "@rails/ujs": "^6.1.4",
     "array-includes": "^3.1.4",
     "arrow-key-navigation": "^1.2.0",
     "autoprefixer": "^9.8.8",
-    "axios": "^0.23.0",
+    "axios": "^0.24.0",
-    "babel-loader": "^8.2.2",
+    "babel-loader": "^8.2.3",
     "babel-plugin-lodash": "^3.3.4",
     "babel-plugin-preval": "^5.0.0",
     "babel-plugin-react-intl": "^6.2.0",
@@ -134,7 +134,7 @@
     "react-motion": "^0.5.2",
     "react-notification": "^6.8.5",
     "react-overlays": "^0.9.3",
-    "react-redux": "^7.2.5",
+    "react-redux": "^7.2.6",
     "react-redux-loading-bar": "^4.0.8",
     "react-router-dom": "^4.1.1",
     "react-router-scroll-4": "^1.0.0-beta.1",
@@ -144,15 +144,15 @@
     "react-textarea-autosize": "^8.3.3",
     "react-toggle": "^4.1.2",
     "redis": "^3.1.2",
-    "redux": "^4.1.1",
+    "redux": "^4.1.2",
     "redux-immutable": "^4.0.0",
-    "redux-thunk": "^2.2.0",
+    "redux-thunk": "^2.4.0",
     "regenerator-runtime": "^0.13.9",
     "rellax": "^1.12.1",
     "requestidlecallback": "^0.3.0",
-    "reselect": "^4.0.0",
+    "reselect": "^4.1.1",
     "rimraf": "^3.0.2",
-    "sass": "^1.43.2",
+    "sass": "^1.43.4",
     "sass-loader": "^10.2.0",
     "stacktrace-js": "^2.0.2",
     "stringz": "^2.1.0",
@@ -179,7 +179,7 @@
     "eslint": "^7.32.0",
     "eslint-plugin-import": "~2.25.2",
     "eslint-plugin-jsx-a11y": "~6.4.1",
-    "eslint-plugin-promise": "~5.1.0",
+    "eslint-plugin-promise": "~5.1.1",
     "eslint-plugin-react": "~7.26.1",
     "jest": "^27.3.1",
     "raf": "^3.4.1",

spec/controllers/api/v1/accounts/notes_controller_spec.rb

@@ -0,0 +1,48 @@
+require 'rails_helper'
+
+describe Api::V1::Accounts::NotesController do
+  render_views
+
+  let(:user)    { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:accounts') }
+  let(:account) { Fabricate(:account) }
+  let(:comment) { 'foo' }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  describe 'POST #create' do
+    subject do
+      post :create, params: { account_id: account.id, comment: comment }
+    end
+
+    context 'when account note has reasonable length' do
+      let(:comment) { 'foo' }
+
+      it 'returns http success' do
+        subject
+        expect(response).to have_http_status(200)
+      end
+
+      it 'updates account note' do
+        subject
+        expect(AccountNote.find_by(account_id: user.account.id, target_account_id: account.id).comment).to eq comment
+      end
+    end
+
+    context 'when account note exceends allowed length' do
+      let(:comment) { 'a' * 2_001 }
+
+      it 'returns 422' do
+        subject
+        expect(response).to have_http_status(422)
+      end
+
+      it 'does not create account note' do
+        subject
+        expect(AccountNote.where(account_id: user.account.id, target_account_id: account.id).exists?).to be_falsey
+      end
+    end
+  end
+end

spec/lib/link_details_extractor_spec.rb

@@ -0,0 +1,29 @@
+require 'rails_helper'
+
+RSpec.describe LinkDetailsExtractor do
+  let(:original_url) { '' }
+  let(:html) { '' }
+  let(:html_charset) { nil }
+
+  subject { described_class.new(original_url, html, html_charset) }
+
+  describe '#canonical_url' do
+    let(:original_url) { 'https://foo.com/article?bar=baz123' }
+
+    context 'when canonical URL points to another host' do
+      let(:html) { '<!doctype html><link rel="canonical" href="https://bar.com/different-article" />' }
+
+      it 'ignores the canonical URLs' do
+        expect(subject.canonical_url).to eq original_url
+      end
+    end
+
+    context 'when canonical URL points to the same host' do
+      let(:html) { '<!doctype html><link rel="canonical" href="https://foo.com/article" />' }
+
+      it 'ignores the canonical URLs' do
+        expect(subject.canonical_url).to eq 'https://foo.com/article'
+      end
+    end
+  end
+end

spec/services/fetch_link_card_service_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 
 RSpec.describe FetchLinkCardService, type: :service do
-  subject { FetchLinkCardService.new }
+  subject { described_class.new }
 
   before do
     stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt'))

spec/workers/move_worker_spec.rb

@@ -9,7 +9,8 @@ describe MoveWorker do
   let(:source_account)   { Fabricate(:account, protocol: :activitypub, domain: 'example.com') }
   let(:target_account)   { Fabricate(:account, protocol: :activitypub, domain: 'example.com') }
   let(:local_user)       { Fabricate(:user) }
-  let!(:account_note)    { Fabricate(:account_note, account: local_user.account, target_account: source_account) }
+  let(:comment)          { 'old note prior to move' }
+  let!(:account_note)    { Fabricate(:account_note, account: local_user.account, target_account: source_account, comment: comment) }
 
   let(:block_service) { double }
 
@@ -26,7 +27,8 @@ describe MoveWorker do
   end
 
   shared_examples 'user note handling' do
-    it 'copies user note' do
+    context 'when user notes are short enough' do
+      it 'copies user note with prelude' do
         subject.perform(source_account.id, target_account.id)
         expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(source_account.acct)
         expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment)
@@ -42,6 +44,23 @@ describe MoveWorker do
       end
     end
 
+    context 'when user notes are too long' do
+      let(:comment) { 'abc' * 333 }
+
+      it 'copies user note without prelude' do
+        subject.perform(source_account.id, target_account.id)
+        expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment)
+      end
+
+      it 'keeps user notes unchanged' do
+        new_account_note = AccountNote.create!(account: account_note.account, target_account: target_account, comment: 'new note prior to move')
+
+        subject.perform(source_account.id, target_account.id)
+        expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(new_account_note.comment)
+      end
+    end
+  end
+
   shared_examples 'block and mute handling' do
     it 'makes blocks carry over and add a note' do
       subject.perform(source_account.id, target_account.id)

spec/workers/publish_scheduled_announcement_worker_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe PublishScheduledAnnouncementWorker do
+  subject { described_class.new }
+
+  let!(:remote_account) { Fabricate(:account, domain: 'domain.com', username: 'foo', uri: 'https://domain.com/users/foo') }
+  let!(:remote_status)  { Fabricate(:status, uri: 'https://domain.com/users/foo/12345', account: remote_account) }
+  let!(:local_status)   { Fabricate(:status) }
+  let(:scheduled_announcement) { Fabricate(:announcement, text: "rebooting very soon, see #{ActivityPub::TagManager.instance.uri_for(remote_status)} and #{ActivityPub::TagManager.instance.uri_for(local_status)}") }
+
+  describe 'perform' do
+    before do
+      service = double
+      allow(FetchRemoteStatusService).to receive(:new).and_return(service)
+      allow(service).to receive(:call).with('https://domain.com/users/foo/12345') { remote_status.reload }
+
+      subject.perform(scheduled_announcement.id)
+    end
+
+    it 'updates the linked statuses' do
+      expect(scheduled_announcement.reload.status_ids).to eq [remote_status.id, local_status.id]
+    end
+  end
+end