Merge branch 'master' into patch-1

This commit is contained in:
Eugen 2017-04-04 14:51:42 +02:00 committed by GitHub
commit 48dfdad492
30 changed files with 165 additions and 85 deletions

View File

@ -38,7 +38,7 @@ gem 'rqrcode'
gem 'twitter-text'
gem 'oj'
gem 'hiredis'
gem 'redis', '~>3.2'
gem 'redis', '~>3.2', require: ['redis', 'redis/connection/hiredis']
gem 'fast_blank'
gem 'htmlentities'
gem 'simple_form'
@ -46,6 +46,7 @@ gem 'will_paginate'
gem 'rack-attack'
gem 'rack-cors', require: 'rack/cors'
gem 'sidekiq'
gem 'sidekiq-unique-jobs'
gem 'rails-settings-cached'
gem 'simple-navigation'
gem 'statsd-instrument'

View File

@ -387,6 +387,9 @@ GEM
connection_pool (~> 2.2, >= 2.2.0)
rack-protection (>= 1.5.0)
redis (~> 3.2, >= 3.2.1)
sidekiq-unique-jobs (4.0.18)
sidekiq (>= 2.6)
thor
simple-navigation (4.0.3)
activesupport (>= 2.3.2)
simple_form (3.2.1)
@ -510,6 +513,7 @@ DEPENDENCIES
sass-rails (~> 5.0)
sdoc (~> 0.4.0)
sidekiq
sidekiq-unique-jobs
simple-navigation
simple_form
simplecov

5
ISSUE_TEMPLATE.md Normal file
View File

@ -0,0 +1,5 @@
[Issue text goes here].
* * * *
- [ ] I searched or browsed the repos other issues to ensure this is not a duplicate.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 MiB

After

Width:  |  Height:  |  Size: 59 KiB

View File

@ -4,6 +4,12 @@ class Api::V1::AppsController < ApiController
respond_to :json
def create
@app = Doorkeeper::Application.create!(name: params[:client_name], redirect_uri: params[:redirect_uris], scopes: (params[:scopes] || Doorkeeper.configuration.default_scopes), website: params[:website])
@app = Doorkeeper::Application.create!(name: app_params[:client_name], redirect_uri: app_params[:redirect_uris], scopes: (app_params[:scopes] || Doorkeeper.configuration.default_scopes), website: app_params[:website])
end
private
def app_params
params.permit(:client_name, :redirect_uris, :scopes, :website)
end
end

View File

@ -7,7 +7,7 @@ class Api::V1::FollowsController < ApiController
respond_to :json
def create
raise ActiveRecord::RecordNotFound if params[:uri].blank?
raise ActiveRecord::RecordNotFound if follow_params[:uri].blank?
@account = FollowService.new.call(current_user.account, target_uri).try(:target_account)
render action: :show
@ -16,6 +16,10 @@ class Api::V1::FollowsController < ApiController
private
def target_uri
params[:uri].strip.gsub(/\A@/, '')
follow_params[:uri].strip.gsub(/\A@/, '')
end
def follow_params
params.permit(:uri)
end
end

View File

@ -10,10 +10,16 @@ class Api::V1::MediaController < ApiController
respond_to :json
def create
@media = MediaAttachment.create!(account: current_user.account, file: params[:file])
@media = MediaAttachment.create!(account: current_user.account, file: media_params[:file])
rescue Paperclip::Errors::NotIdentifiedByImageMagickError
render json: { error: 'File type of uploaded media could not be verified' }, status: 422
rescue Paperclip::Error
render json: { error: 'Error processing thumbnail for uploaded media' }, status: 500
end
private
def media_params
params.permit(:file)
end
end

View File

@ -12,13 +12,19 @@ class Api::V1::ReportsController < ApiController
end
def create
status_ids = params[:status_ids].is_a?(Enumerable) ? params[:status_ids] : [params[:status_ids]]
status_ids = report_params[:status_ids].is_a?(Enumerable) ? report_params[:status_ids] : [report_params[:status_ids]]
@report = Report.create!(account: current_account,
target_account: Account.find(params[:account_id]),
target_account: Account.find(report_params[:account_id]),
status_ids: Status.find(status_ids).pluck(:id),
comment: params[:comment])
comment: report_params[:comment])
render :show
end
private
def report_params
params.permit(:account_id, :comment, status_ids: [])
end
end

View File

@ -62,11 +62,11 @@ class Api::V1::StatusesController < ApiController
end
def create
@status = PostStatusService.new.call(current_user.account, params[:status], params[:in_reply_to_id].blank? ? nil : Status.find(params[:in_reply_to_id]), media_ids: params[:media_ids],
sensitive: params[:sensitive],
spoiler_text: params[:spoiler_text],
visibility: params[:visibility],
application: doorkeeper_token.application)
@status = PostStatusService.new.call(current_user.account, status_params[:status], status_params[:in_reply_to_id].blank? ? nil : Status.find(status_params[:in_reply_to_id]), media_ids: status_params[:media_ids],
sensitive: status_params[:sensitive],
spoiler_text: status_params[:spoiler_text],
visibility: status_params[:visibility],
application: doorkeeper_token.application)
render action: :show
end
@ -111,4 +111,8 @@ class Api::V1::StatusesController < ApiController
@status = Status.find(params[:id])
raise ActiveRecord::RecordNotFound unless @status.permitted?(current_account)
end
def status_params
params.permit(:status, :in_reply_to_id, :sensitive, :spoiler_text, :visibility, media_ids: [])
end
end

View File

@ -39,7 +39,14 @@ class ApplicationController < ActionController::Base
end
def set_user_activity
current_user.touch(:current_sign_in_at) if !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
return unless !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
# Mark user as signed-in today
current_user.update_tracked_fields(request)
# If the sign in is after a two week break, we need to regenerate their feed
RegenerationWorker.perform_async(current_user.account_id) if current_user.last_sign_in_at < 14.days.ago
return
end
def check_suspension

View File

@ -3,6 +3,7 @@
class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
skip_before_action :authenticate_resource_owner!
before_action :set_locale
before_action :store_current_location
before_action :authenticate_resource_owner!
@ -11,4 +12,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
def store_current_location
store_location_for(:user, request.url)
end
def set_locale
I18n.locale = current_user.try(:locale) || I18n.default_locale
rescue I18n::InvalidLocale
I18n.locale = I18n.default_locale
end
end

View File

@ -5,7 +5,7 @@ require 'singleton'
class FeedManager
include Singleton
MAX_ITEMS = 800
MAX_ITEMS = 400
def key(type, id)
"feed:#{type}:#{id}"
@ -50,10 +50,18 @@ class FeedManager
def merge_into_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id)
query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
from_account.statuses.limit(MAX_ITEMS).each do |status|
next if status.direct_visibility? || filter?(:home, status, into_account)
redis.zadd(timeline_key, status.id, status.id)
if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
query = query.where('id > ?', oldest_home_score)
end
redis.pipelined do
query.each do |status|
next if status.direct_visibility? || filter?(:home, status, into_account)
redis.zadd(timeline_key, status.id, status.id)
end
end
trim(:home, into_account.id)
@ -61,31 +69,20 @@ class FeedManager
def unmerge_from_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id)
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
from_account.statuses.select('id').find_each do |status|
redis.zrem(timeline_key, status.id)
redis.zremrangebyscore(timeline_key, status.id, status.id)
from_account.statuses.select('id').where('id > ?', oldest_home_score).find_in_batches do |statuses|
redis.pipelined do
statuses.each do |status|
redis.zrem(timeline_key, status.id)
redis.zremrangebyscore(timeline_key, status.id, status.id)
end
end
end
end
def inline_render(target_account, template, object)
rabl_scope = Class.new do
include RoutingHelper
def initialize(account)
@account = account
end
def current_user
@account.try(:user)
end
def current_account
@account
end
end
Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: rabl_scope.new(target_account)).render
Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render
end
private
@ -95,36 +92,38 @@ class FeedManager
end
def filter_from_home?(status, receiver)
return true if receiver.muting?(status.account)
return true if status.reply? && status.in_reply_to_id.nil?
should_filter = false
check_for_mutes = [status.account_id]
check_for_mutes.concat([status.reblog.account_id]) if status.reblog?
if status.reply? && status.in_reply_to_id.nil?
should_filter = true
elsif status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply
return true if receiver.muting?(check_for_mutes)
check_for_blocks = status.mentions.map(&:account_id)
check_for_blocks.concat([status.reblog.account_id]) if status.reblog?
return true if receiver.blocking?(check_for_blocks)
if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply
should_filter = !receiver.following?(status.in_reply_to_account) # and I'm not following the person it's a reply to
should_filter &&= !(receiver.id == status.in_reply_to_account_id) # and it's not a reply to me
should_filter &&= !(status.account_id == status.in_reply_to_account_id) # and it's not a self-reply
return should_filter
elsif status.reblog? # Filter out a reblog
should_filter = receiver.blocking?(status.reblog.account) # if I'm blocking the reblogged person
should_filter ||= receiver.muting?(status.reblog.account) # or muting that person
should_filter ||= status.reblog.account.blocking?(receiver) # or if the author of the reblogged status is blocking me
return status.reblog.account.blocking?(receiver) # or if the author of the reblogged status is blocking me
end
should_filter ||= receiver.blocking?(status.mentions.map(&:account_id)) # or if it mentions someone I blocked
should_filter
false
end
def filter_from_mentions?(status, receiver)
should_filter = receiver.id == status.account_id # Filter if I'm mentioning myself
should_filter ||= receiver.blocking?(status.account) # or it's from someone I blocked
should_filter ||= receiver.blocking?(status.mentions.includes(:account).map(&:account)) # or if it mentions someone I blocked
should_filter ||= (status.account.silenced? && !receiver.following?(status.account)) # of if the account is silenced and I'm not following them
check_for_blocks = [status.account_id]
check_for_blocks.concat(status.mentions.select('account_id').map(&:account_id))
check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
if status.reply? && !status.in_reply_to_account_id.nil? # or it's a reply
should_filter ||= receiver.blocking?(status.in_reply_to_account) # to a user I blocked
end
should_filter = receiver.id == status.account_id # Filter if I'm mentioning myself
should_filter ||= receiver.blocking?(check_for_blocks) # or it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked
should_filter ||= (status.account.silenced? && !receiver.following?(status.account)) # of if the account is silenced and I'm not following them
should_filter
end

View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
class InlineRablScope
include RoutingHelper
def initialize(account)
@account = account
end
def current_user
@account.try(:user)
end
def current_account
@account
end
end

View File

@ -10,17 +10,9 @@ class Feed
max_id = '+inf' if max_id.blank?
since_id = '-inf' if since_id.blank?
unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:last).map(&:to_i)
status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
# If we're after most recent items and none are there, we need to precompute the feed
if unhydrated.empty? && max_id == '+inf' && since_id == '-inf'
RegenerationWorker.perform_async(@account.id, @type)
@statuses = Status.send("as_#{@type}_timeline", @account).cache_ids.paginate_by_max_id(limit, nil, nil)
else
status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
@statuses = unhydrated.map { |id| status_map[id] }.compact
end
@statuses
unhydrated.map { |id| status_map[id] }.compact
end
private

View File

@ -188,7 +188,7 @@ class Status < ApplicationRecord
end
before_validation do
text.strip!
text&.strip!
spoiler_text&.strip!
self.reply = !(in_reply_to_id.nil? && thread.nil?) unless reply

View File

@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService
# @param [Symbol] type :home or :mentions
# @param [Account] account
def call(_, account)
Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status|
next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
redis.pipelined do
Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
end
end
end

View File

@ -3,7 +3,7 @@
class AfterRemoteFollowRequestWorker
include Sidekiq::Worker
sidekiq_options retry: 5
sidekiq_options queue: 'pull', retry: 5
def perform(follow_request_id)
follow_request = FollowRequest.find(follow_request_id)

View File

@ -3,7 +3,7 @@
class AfterRemoteFollowWorker
include Sidekiq::Worker
sidekiq_options retry: 5
sidekiq_options queue: 'pull', retry: 5
def perform(follow_id)
follow = Follow.find(follow_id)

View File

@ -5,7 +5,7 @@ require 'csv'
class ImportWorker
include Sidekiq::Worker
sidekiq_options retry: false
sidekiq_options queue: 'pull', retry: false
def perform(import_id)
import = Import.find(import_id)

View File

@ -3,7 +3,7 @@
class LinkCrawlWorker
include Sidekiq::Worker
sidekiq_options retry: false
sidekiq_options queue: 'pull', retry: false
def perform(status_id)
FetchLinkCardService.new.call(Status.find(status_id))

View File

@ -3,6 +3,8 @@
class MergeWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(from_account_id, into_account_id)
FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id))
end

View File

@ -3,7 +3,7 @@
class NotificationWorker
include Sidekiq::Worker
sidekiq_options retry: 5
sidekiq_options queue: 'push', retry: 5
def perform(xml, source_account_id, target_account_id)
SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))

View File

@ -3,7 +3,9 @@
class RegenerationWorker
include Sidekiq::Worker
def perform(account_id, timeline_type)
PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
def perform(account_id, _ = :home)
PrecomputeFeedService.new.call(:home, Account.find(account_id))
end
end

View File

@ -3,7 +3,7 @@
class ThreadResolveWorker
include Sidekiq::Worker
sidekiq_options retry: false
sidekiq_options queue: 'pull', retry: false
def perform(child_status_id, parent_url)
child_status = Status.find(child_status_id)

View File

@ -3,6 +3,8 @@
class UnmergeWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(from_account_id, into_account_id)
FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id))
end

View File

@ -33,7 +33,7 @@ services:
restart: always
build: .
env_file: .env.production
command: bundle exec sidekiq -q default -q mailers -q push
command: bundle exec sidekiq -q default -q mailers -q pull -q push
depends_on:
- db
- redis

View File

@ -8,6 +8,6 @@ Mastodon can theoretically run indefinitely on a free [Heroku](https://heroku.co
1. Click the above button.
2. Fill in the options requested.
* You can use a .herokuapp.com domain, which will be simple to set up, or you can use a custom domain. If you want a custom domain and HTTPS, you will need to upgrade to a paid plan (to use Heroku's SSL features), or set up [CloudFlare](https://cloudflare.com) who offer free "Flexible SSL" (note: CloudFlare have some undefined limits on WebSockets. So far, no one has reported hitting concurrent connection limits).
* You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saaved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details.
* You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details.
* If you want your Mastodon to be able to send emails, configure SMTP settings here (or later). Consider using [Mailgun](https://mailgun.com) or similar, who offer free plans that should suit your interests.
3. Deploy! The app should be set up, with a working web interface and database. You can change settings and manage versions from the Heroku dashboard.

View File

@ -180,7 +180,7 @@ User=mastodon
WorkingDirectory=/home/mastodon/live
Environment="RAILS_ENV=production"
Environment="DB_POOL=5"
ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push
ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push
TimeoutSec=15
Restart=always

View File

@ -11,17 +11,31 @@ There is also a list at [instances.mastodon.xyz](https://instances.mastodon.xyz)
| [animalliberation.social](https://animalliberation.social) |Animal Rights|Yes|No|
| [socially.constructed.space](https://socially.constructed.space) |Single user|No|No|
| [epiktistes.com](https://epiktistes.com) |N/A|Yes|No|
| [fern.surgeplay.com](https://fern.surgeplay.com) |Federates everywhere, Minecraft-focused|Yes|No
| [gay.crime.team](https://gay.crime.team) |the place for doin' gay crime online (please don't actually do crime here)|Yes|No|
| [icosahedron.website](https://icosahedron.website/) |Icosahedron-themed (well, visually), open registration.|Yes|No|
| [memetastic.space](https://memetastic.space) |Memes|Yes|No|
| [social.diskseven.com](https://social.diskseven.com) |Single user|No|No (DNS entry but no response)|
| [social.gestaltzerfall.net](https://social.gestaltzerfall.net) |Single user|No|No|
| [mastodon.xyz](https://mastodon.xyz) |N/A|Yes|Yes|
| [social.targaryen.house](https://social.targaryen.house) |N/A|Yes|No|
| [social.targaryen.house](https://social.targaryen.house) |Federates everywhere, quick updates.|Yes|Yes|
| [social.mashek.net](https://social.mashek.net) |Themed and customised for Mashekstein Labs community. Selectively federates.|Yes|No|
| [masto.themimitoof.fr](https://masto.themimitoof.fr) |N/A|Yes|Yes|
| [social.imirhil.fr](https://social.imirhil.fr) |N/A|No|Yes|
| [social.wxcafe.net](https://social.wxcafe.net) |Open registrations, federates everywhere, no moderation yet|Yes|Yes|
| [octodon.social](https://octodon.social) |Open registrations, federates everywhere, cutest instance yet|Yes|Yes|
| [hostux.social](https://hostux.social) |N/A|Yes|Yes|
| [social.alex73630.xyz](https://social.alex73630.xyz) |Francophones|Yes|Yes|
| [maly.io](https://maly.io) |N/A|Yes|No|
| [social.lou.lt](https://social.lou.lt) |N/A|Yes|No|
| [mastodon.ninetailed.uk](https://mastodon.ninetailed.uk) |N/A|Yes|No|
| [soc.louiz.org](https://soc.louiz.org) |"Coucou"|Yes|No|
| [7nw.eu](https://7nw.eu) |N/A|Yes|No|
| [mastodon.gougere.fr](https://mastodon.gougere.fr)|N/A|Yes|No|
| [aleph.land](https://aleph.land)|N/A|Yes|No|
| [share.elouworld.org](https://share.elouworld.org)|N/A|No|No|
| [social.lkw.tf](https://social.lkw.tf)|N/A|No|No|
| [manowar.social](https://manowar.social)|N/A|No|No|
| [social.ballpointcarrot.net](https://social.ballpointcarrot.net)|Down at time of entry|No|No|
Let me know if you start running one so I can add it to the list! (Alternatively, add it yourself as a pull request).

View File

@ -26,17 +26,17 @@ Mastodon User's Guide
## Intro
Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "instance"), and users of any instance can interact freely with those of other instances (called "federation"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities.
Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "*instance*"), and users of any instance can interact freely with those of other instances (called "*federation*"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities.
#### Decentralization and Federation
Mastodon is a system decentralized through a concept called "federation" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail.
Mastodon is a system decentralized through a concept called "*federation*" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail.
As such, anyone can download Mastodon and e.g. run it for a small community of people, but any user registered on that instance can follow and send and read posts from other Mastodon instances (as well as servers running other GNU Social-compatible services). This means that not only is users' data not inherently owned by a company with an interest in selling it to advertisers, but also that if any given server shuts down its users can set up a new one or migrate to another instance, rather than the entire service being lost.
Within each Mastodon instance, usernames just appear as `@username`, similar to other services such as Twitter. Users from other instances appear, and can be searched for and followed, as `@user@servername.ext` - so e.g. `@gargron` on the `mastodon.social` instance can be followed from other instances as `@gargron@mastodon.social`).
Posts from users on external instances are "federated" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section.
Posts from users on external instances are "*federated*" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section.
## Getting Started