pitchum/debianize-mobilizon
pitchum/debianize-mobilizon
1
0
Fork 0
Code Issues Pull Requests Projects Releases 2 Wiki Activity

Save remote profiles avatars & banners locally

Browse Source 
Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2020-12-15 17:17:42 +01:00
parent ae03f84950
commit 9b27e70eb0
No known key found for this signature in database
GPG Key ID: A061B9DDE0CA0773
41 changed files with 1424 additions and 716 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -20,6 +20,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Docker
`docker-compose exec mobilizon mobilizon_ctl maintenance.fix_unattached_media_in_body`
* **Refresh remote profiles to save avatars locally**
Profile avatars and banners were previously only proxified and cached. Now we save them locally. Refreshing all remote actors will save profile media locally instead.
* Source install
`MIX_ENV=prod mix mobilizon.actors.refresh --all`
* Docker
`docker-compose exec mobilizon mobilizon_ctl actors.refresh --all`
### Added
- **Add a command to clean orphan media files**. There's a `--dry-run` option to see what files would have been deleted.

11
config/config.exs
View File

@ -81,17 +81,6 @@ config :mobilizon, Mobilizon.Web.Upload,
config :mobilizon, Mobilizon.Web.Upload.Uploader.Local, uploads: "uploads"
config :mobilizon, :media_proxy,
enabled: true,
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
http: [
follow_redirect: true,
pool: :media
]
]
config :mobilizon, Mobilizon.Web.Email.Mailer,
adapter: Bamboo.SMTPAdapter,
server: "localhost",

5
lib/federation/activity_pub/activity_pub.ex
View File

@ -766,7 +766,10 @@ defmodule Mobilizon.Federation.ActivityPub do
res =
with {:ok, %{status: 200, body: body}} <-
Tesla.get(url, headers: [{"Accept", "application/activity+json"}]),
Tesla.get(url,
headers: [{"Accept", "application/activity+json"}],
follow_redirect: true
),
:ok <- Logger.debug("response okay, now decoding json"),
{:ok, data} <- Jason.decode(body) do
Logger.debug("Got activity+json response at actor's endpoint, now converting data")

2
lib/federation/activity_pub/transmogrifier.ex
View File

@ -382,7 +382,7 @@ defmodule Mobilizon.Federation.ActivityPub.Transmogrifier do
{:ok, activity, new_actor}
else
e ->
Logger.debug(inspect(e))
Logger.error(inspect(e))
:error
end
end

28
lib/federation/activity_stream/converter/actor.ex
View File

@ -11,7 +11,9 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
alias Mobilizon.Federation.ActivityPub.Utils
alias Mobilizon.Federation.ActivityStream.{Converter, Convertible}
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Service.HTTP.RemoteMediaDownloaderClient
alias Mobilizon.Service.RichMedia.Parser
alias Mobilizon.Web.Upload
@behaviour Converter
@ -30,18 +32,10 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
@spec as_to_model_data(map()) :: {:ok, map()}
def as_to_model_data(%{"type" => type} = data) when type in @allowed_types do
avatar =
data["icon"]["url"] &&
%{
"name" => data["icon"]["name"] || "avatar",
"url" => MediaProxy.url(data["icon"]["url"])
}
download_picture(get_in(data, ["icon", "url"]), get_in(data, ["icon", "name"]), "avatar")
banner =
data["image"]["url"] &&
%{
"name" => data["image"]["name"] || "banner",
"url" => MediaProxy.url(data["image"]["url"])
}
download_picture(get_in(data, ["image", "url"]), get_in(data, ["image", "name"]), "banner")
%{
url: data["id"],
@ -140,4 +134,16 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
})
end
end
@spec download_picture(String.t() | nil, String.t(), String.t()) :: map()
defp download_picture(nil, _name, _default_name), do: nil
defp download_picture(url, name, default_name) do
with {:ok, %{body: body, status: code, headers: response_headers}}
when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
{:ok, file} <- Upload.store(%{body: body, name: name}) do
file
end
end
end

5
lib/graphql/resolvers/event.ex
View File

@ -10,7 +10,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
alias Mobilizon.Users.User
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Federation.ActivityPub.Activity
import Mobilizon.Web.Gettext
@ -35,7 +34,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
) do
case {:has_event, Events.get_own_event_by_uuid_with_preload(uuid, user_id)} do
{:has_event, %Event{} = event} ->
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
{:has_event, _} ->
{:error, :event_not_found}
@ -51,7 +50,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Event do
{:has_event, Events.get_public_event_by_uuid_with_preload(uuid)},
{:access_valid, true} <-
{:access_valid, Map.has_key?(context, :current_user) || check_event_access(event)} do
{:ok, Map.put(event, :organizer_actor, Person.proxify_pictures(event.organizer_actor))}
{:ok, event}
else
{:has_event, _} ->
find_private_event(parent, args, resolution)

5
lib/graphql/resolvers/group.ex
View File

@ -8,7 +8,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
alias Mobilizon.Actors.{Actor, Member}
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.GraphQL.API
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Upload
import Mobilizon.Web.Gettext
@ -30,8 +29,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
with {:ok, %Actor{id: group_id} = group} <-
ActivityPub.find_or_make_group_from_nickname(name),
{:actor, %Actor{id: actor_id} = _actor} <- {:actor, Users.get_actor_for_user(user)},
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)},
group <- Person.proxify_pictures(group) do
{:member, true} <- {:member, Actors.is_member?(actor_id, group_id)} do
{:ok, group}
else
{:member, false} ->
@ -44,7 +42,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Group do
def find_group(_parent, %{preferred_username: name}, _resolution) do
with {:ok, actor} <- ActivityPub.find_or_make_group_from_nickname(name),
%Actor{} = actor <- Person.proxify_pictures(actor),
%Actor{} = actor <- restrict_fields_for_non_member_request(actor) do
{:ok, actor}
else

3
lib/graphql/resolvers/participant.ex
View File

@ -6,7 +6,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
alias Mobilizon.Actors.Actor
alias Mobilizon.Events.{Event, Participant}
alias Mobilizon.GraphQL.API.Participations
alias Mobilizon.GraphQL.Resolvers.Person
alias Mobilizon.Users.User
alias Mobilizon.Web.Email
alias Mobilizon.Web.Email.Checker
@ -114,7 +113,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Participant do
%Participant{} = participant <-
participant
|> Map.put(:event, event)
|> Map.put(:actor, Person.proxify_pictures(actor)) do
|> Map.put(:actor, actor) do
{:ok, participant}
else
{:maximum_attendee_capacity, _} ->

70
lib/graphql/resolvers/person.ex
View File

@ -15,15 +15,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
alias Mobilizon.Federation.ActivityPub
require Logger
alias Mobilizon.Web.{MediaProxy, Upload}
alias Mobilizon.Web.Upload
@doc """
Get a person
"""
def get_person(_parent, %{id: id}, %{context: %{current_user: %User{role: role}}}) do
with %Actor{suspended: suspended} = actor <- Actors.get_actor_with_preload(id, true),
true <- suspended == false or is_moderator(role),
actor <- proxify_pictures(actor) do
true <- suspended == false or is_moderator(role) do
{:ok, actor}
else
_ ->
@ -31,6 +30,8 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def get_person(_parent, _args, _resolution), do: {:error, :unauthorized}
@doc """
Find a person
"""
@ -39,8 +40,7 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
}) do
with {:ok, %Actor{id: actor_id} = actor} <-
ActivityPub.find_or_make_actor_from_nickname(preferred_username),
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)},
actor <- proxify_pictures(actor) do
{:own, {:is_owned, _}} <- {:own, User.owns_actor(user, actor_id)} do
{:ok, actor}
else
{:own, nil} ->
@ -120,9 +120,12 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
args = Map.put(args, :user_id, user.id)
with args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
end
end
@ -144,10 +147,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
with {:find_actor, %Actor{} = actor} <-
{:find_actor, Actors.get_actor(id)},
{:is_owned, %Actor{}} <- User.owns_actor(user, actor.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, _activity, %Actor{} = actor} <- ActivityPub.update(actor, args, true) do
{:ok, actor}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:find_actor, nil} ->
{:error, dgettext("errors", "Profile not found")}
@ -199,18 +205,27 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
defp save_attached_pictures(args) do
Enum.reduce([:avatar, :banner], args, fn key, args ->
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
media = args[key][:media]
with args when is_map(args) <- save_attached_picture(args, :avatar),
args when is_map(args) <- save_attached_picture(args, :banner) do
args
end
end
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
Map.put(args, key, %{"name" => name, "url" => url, "mediaType" => content_type})
defp save_attached_picture(args, key) do
if Map.has_key?(args, key) && !is_nil(args[key][:media]) do
with media when is_map(media) <- save_picture(args[key][:media], key) do
Map.put(args, key, media)
end
else
args
end
end)
end
defp save_picture(media, key) do
with {:ok, %{name: name, url: url, content_type: content_type, size: _size}} <-
Upload.store(media.file, type: key, description: media.alt) do
%{"name" => name, "url" => url, "mediaType" => content_type}
end
end
@doc """
@ -223,10 +238,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
{:no_actor, true} <- {:no_actor, no_actor},
args <- Map.update(args, :preferred_username, "", &String.downcase/1),
args <- Map.put(args, :user_id, user.id),
args <- save_attached_pictures(args),
{:picture, args} when is_map(args) <- {:picture, save_attached_pictures(args)},
{:ok, %Actor{} = new_person} <- Actors.new_person(args, true) do
{:ok, new_person}
else
{:picture, {:error, :file_too_large}} ->
{:error, dgettext("errors", "The provided picture is too heavy")}
{:error, :user_not_found} ->
{:error, dgettext("errors", "No user with this email was found")}
@ -298,12 +316,6 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
end
end
def proxify_pictures(%Actor{} = actor) do
actor
|> proxify_avatar
|> proxify_banner
end
def user_for_person(%Actor{type: :Person, user_id: user_id}, _args, %{
context: %{current_user: %User{role: role}}
})
@ -343,20 +355,4 @@ defmodule Mobilizon.GraphQL.Resolvers.Person do
defp last_admin_of_a_group?(actor_id) do
length(Actors.list_group_ids_where_last_administrator(actor_id)) > 0
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{avatar: %{url: avatar_url} = avatar} = actor) do
actor |> Map.put(:avatar, avatar |> Map.put(:url, MediaProxy.url(avatar_url)))
end
@spec proxify_avatar(Actor.t()) :: Actor.t()
defp proxify_avatar(%Actor{} = actor), do: actor
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{banner: %{url: banner_url} = banner} = actor) do
actor |> Map.put(:banner, banner |> Map.put(:url, MediaProxy.url(banner_url)))
end
@spec proxify_banner(Actor.t()) :: Actor.t()
defp proxify_banner(%Actor{} = actor), do: actor
end

6
lib/mix/tasks/mobilizon/actors/refresh.ex
View File

@ -73,6 +73,12 @@ defmodule Mix.Tasks.Mobilizon.Actors.Refresh do
{:actor, nil} ->
shell_error("Error: No such actor")
{:error, err} when is_binary(err) ->
shell_error(err)
_err ->
shell_error("Error while refreshing actor #{preferred_username}")
end
end

31
lib/mobilizon/actors/actors.ex
View File

@ -13,11 +13,13 @@ defmodule Mobilizon.Actors do
alias Mobilizon.Actors.{Actor, Bot, Follower, Member}
alias Mobilizon.Addresses.Address
alias Mobilizon.{Crypto, Events}
alias Mobilizon.Events.FeedToken
alias Mobilizon.Federation.ActivityPub
alias Mobilizon.Medias.File
alias Mobilizon.Service.Workers
alias Mobilizon.Storage.{Page, Repo}
alias Mobilizon.Users
alias Mobilizon.Users.User
alias Mobilizon.Web.Email.Group
alias Mobilizon.Web.Upload
@ -215,18 +217,35 @@ defmodule Mobilizon.Actors do
def new_person(args, default_actor \\ false) do
args = Map.put(args, :keys, Crypto.generate_rsa_2048_private_key())
with {:ok, %Actor{id: person_id} = person} <-
%Actor{}
|> Actor.registration_changeset(args)
|> Repo.insert() do
Events.create_feed_token(%{user_id: args.user_id, actor_id: person.id})
multi =
Multi.new()
|> Multi.insert(:person, Actor.registration_changeset(%Actor{}, args))
|> Multi.insert(:token, fn %{person: person} ->
FeedToken.changeset(%FeedToken{}, %{
user_id: args.user_id,
actor_id: person.id,
token: Ecto.UUID.generate()
})
end)
multi =
if default_actor do
user = Users.get_user!(args.user_id)
Users.update_user(user, %{default_actor_id: person_id})
Multi.update(multi, :user, fn %{person: person} ->
User.changeset(user, %{default_actor_id: person.id})
end)
else
multi
end
case Repo.transaction(multi) do
{:ok, %{person: %Actor{} = person}} ->
{:ok, person}
{:error, _step, err, _} ->
Logger.error("Error while creating a new person")
{:error, err}
end
end

6
lib/service/export/feed.ex
View File

@ -13,7 +13,7 @@ defmodule Mobilizon.Service.Export.Feed do
alias Mobilizon.Storage.Page
alias Mobilizon.Users.User
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.Router.Helpers, as: Routes
require Logger
@ -85,14 +85,14 @@ defmodule Mobilizon.Service.Export.Feed do
feed =
if actor.avatar do
feed |> Feed.icon(actor.avatar.url |> MediaProxy.url())
feed |> Feed.icon(actor.avatar.url)
else
feed
end
feed =
if actor.banner do
feed |> Feed.logo(actor.banner.url |> MediaProxy.url())
feed |> Feed.logo(actor.banner.url)
else
feed
end

22
lib/service/http/remote_media_downloader_client.ex Normal file
View File

@ -0,0 +1,22 @@
defmodule Mobilizon.Service.HTTP.RemoteMediaDownloaderClient do
@moduledoc """
Tesla HTTP Basic Client that fetches HTML to extract metadata preview
"""
use Tesla
alias Mobilizon.Config
@default_opts [
recv_timeout: 20_000
]
adapter(Tesla.Adapter.Hackney, @default_opts)
@user_agent Config.instance_user_agent()
plug(Tesla.Middleware.FollowRedirects)
plug(Tesla.Middleware.Timeout, timeout: 10_000)
plug(Tesla.Middleware.Headers, [{"User-Agent", @user_agent}])
end

3
lib/service/metadata/actor.ex
View File

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
alias Phoenix.HTML.Tag
alias Mobilizon.Actors.Actor
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, default_description: 1]
def build_tags(_actor, _locale \\ "en")
@ -36,7 +35,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Actors.Actor do
tags
else
tags ++
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url |> MediaProxy.url())]
[Tag.tag(:meta, property: "og:image", content: actor.avatar.url)]
end
end

3
lib/service/metadata/event.ex
View File

@ -3,7 +3,6 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
alias Phoenix.HTML.Tag
alias Mobilizon.Events.Event
alias Mobilizon.Web.JsonLD.ObjectView
alias Mobilizon.Web.MediaProxy
import Mobilizon.Service.Metadata.Utils, only: [process_description: 2, strip_tags: 1]
def build_tags(%Event{} = event, locale \\ "en") do
@ -28,7 +27,7 @@ defimpl Mobilizon.Service.Metadata, for: Mobilizon.Events.Event do
[
Tag.tag(:meta,
property: "og:image",
content: event.picture.file.url |> MediaProxy.url()
content: event.picture.file.url
)
]
end

8
lib/service/rich_media/parser.ex
View File

@ -47,6 +47,14 @@ defmodule Mobilizon.Service.RichMedia.Parser do
{:error, "Cachex error: #{inspect(e)}"}
end
@doc """
Get a filename for the fetched data, using the response header or the last part of the URL
"""
@spec get_filename_from_response(Enum.t(), String.t()) :: String.t() | nil
def get_filename_from_response(response_headers, url) do
get_filename_from_headers(response_headers) || get_filename_from_url(url)
end
@spec parse_url(String.t(), Enum.t()) :: {:ok, map()} | {:error, any()}
defp parse_url(url, options \\ []) do
user_agent = Keyword.get(options, :user_agent, Config.instance_user_agent())

50
lib/web/controllers/media_proxy_controller.ex
View File

@ -1,50 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/controller.ex
defmodule Mobilizon.Web.MediaProxyController do
use Mobilizon.Web, :controller
alias Plug.Conn
alias Mobilizon.Config
alias Mobilizon.Web.MediaProxy
alias Mobilizon.Web.ReverseProxy
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
with config <- Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
:ok <- filename_matches(Map.has_key?(params, "filename"), conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
send_resp(conn, 404, Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
send_resp(conn, 403, Conn.Status.reason_phrase(403))
{:wrong_filename, filename} ->
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
end
end
def filename_matches(has_filename, path, url) do
filename =
url
|> MediaProxy.filename()
|> URI.decode()
path = URI.decode(path)
if has_filename && filename && Path.basename(path) != filename do
{:wrong_filename, filename}
else
:ok
end
end
end

91
lib/web/proxy/media_proxy.ex
View File

@ -1,91 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/lib/pleroma/web/media_proxy/media_proxy.ex
defmodule Mobilizon.Web.MediaProxy do
@moduledoc """
Handles proxifying media files
"""
alias Mobilizon.Config
alias Mobilizon.Web.Endpoint
@base64_opts [padding: false]
def url(nil), do: nil
def url(""), do: nil
def url("/" <> _ = url), do: url
def url(url) do
config = Application.get_env(:mobilizon, :media_proxy, [])
if !Keyword.get(config, :enabled, false) or
String.starts_with?(url, Endpoint.url()) do
url
else
encode_url(url)
end
end
def encode_url(url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
# Must preserve `%2F` for compatibility with S3
# https://git.pleroma.social/pleroma/pleroma/issues/580
replacement = get_replacement(url, ":2F:")
# The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
base64 =
url
|> String.replace("%2F", replacement)
|> URI.decode()
|> URI.encode()
|> String.replace(replacement, "%2F")
|> Base.url_encode64(@base64_opts)
sig = :crypto.hmac(:sha, secret, base64)
sig64 = sig |> Base.url_encode64(@base64_opts)
build_url(sig64, base64, filename(url))
end
def decode_url(sig, url) do
secret = Application.get_env(:mobilizon, Endpoint)[:secret_key_base]
sig = Base.url_decode64!(sig, @base64_opts)
local_sig = :crypto.hmac(:sha, secret, url)
if local_sig == sig do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
{:error, :invalid_signature}
end
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
[
Config.get([:media_proxy, :base_url], Endpoint.url()),
"proxy",
sig_base64,
url_base64,
filename
]
|> Enum.filter(fn value -> value end)
|> Path.join()
end
defp get_replacement(url, replacement) do
if String.contains?(url, replacement) do
get_replacement(url, replacement <> replacement)
else
replacement
end
end
end

8
lib/web/proxy/reverse_proxy.ex
View File

@ -69,8 +69,6 @@ defmodule Mobilizon.Web.ReverseProxy do
alias Plug.Conn
alias Mobilizon.Web.MediaProxy
require Logger
@type option ::
@ -111,7 +109,7 @@ defmodule Mobilizon.Web.ReverseProxy do
req_headers = build_req_headers(conn.req_headers, opts)
opts =
if filename = MediaProxy.filename(url) do
if filename = filename(url) do
Keyword.put_new(opts, :attachment_name, filename)
else
opts
@ -388,4 +386,8 @@ defmodule Mobilizon.Web.ReverseProxy do
defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit}
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
end

7
lib/web/router.ex
View File

@ -162,13 +162,6 @@ defmodule Mobilizon.Web.Router do
post("/auth/:provider/callback", AuthController, :callback)
end
scope "/proxy/", Mobilizon.Web do
pipe_through(:remote_media)
get("/:sig/:url", MediaProxyController, :remote)
get("/:sig/:url/:filename", MediaProxyController, :remote)
end
if Application.fetch_env!(:mobilizon, :env) in [:dev, :e2e] do
# If using Phoenix
forward("/sent_emails", Bamboo.SentEmailViewerPlug)

4
lib/web/views/json_ld/object_view.ex
View File

@ -5,7 +5,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
alias Mobilizon.Addresses.Address
alias Mobilizon.Events.Event
alias Mobilizon.Posts.Post
alias Mobilizon.Web.{Endpoint, MediaProxy}
alias Mobilizon.Web.Endpoint
alias Mobilizon.Web.JsonLD.ObjectView
def render("group.json", %{group: %Actor{} = group}) do
@ -41,7 +41,7 @@ defmodule Mobilizon.Web.JsonLD.ObjectView do
"image" =>
if(event.picture,
do: [
event.picture.file.url |> MediaProxy.url()
event.picture.file.url
],
else: ["#{Endpoint.url()}/img/mobilizon_default_card.png"]
)

2
mix.exs
View File

@ -264,7 +264,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Plugs.UploadedMedia,
Mobilizon.Web.FallbackController,
Mobilizon.Web.FeedController,
Mobilizon.Web.MediaProxyController,
Mobilizon.Web.PageController,
Mobilizon.Web.ChangesetView,
Mobilizon.Web.JsonLD.ObjectView,
@ -295,7 +294,6 @@ defmodule Mobilizon.Mixfile do
Mobilizon.Web.Upload.MIME,
Mobilizon.Web.Upload.Uploader,
Mobilizon.Web.Upload.Uploader.Local,
Mobilizon.Web.MediaProxy,
Mobilizon.Web.ReverseProxy
],
Geospatial: [

4
test/federation/activity_pub/activity_pub_test.exs
View File

@ -45,13 +45,13 @@ defmodule Mobilizon.Federation.ActivityPubTest do
use_cassette "activity_pub/fetch_tcit@framapiaf.org" do
assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :public} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
_actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end
use_cassette "activity_pub/fetch_tcit@framapiaf.org_not_discoverable" do
assert {:ok,
%Actor{preferred_username: "tcit", domain: "framapiaf.org", visibility: :unlisted} =
actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
_actor} = ActivityPub.make_actor_from_nickname("tcit@framapiaf.org")
end
end

14
test/federation/activity_pub/transmogrifier_test.exs
View File

@ -39,7 +39,7 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
Transmogrifier.handle_incoming(data)
assert data["id"] ==
"https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity"
"https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity"
assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@ -49,12 +49,12 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
# "http://localtesting.pleroma.lol/users/lain"
# ]
assert data["actor"] == "https://test.mobilizon.org/@Alicia"
assert data["actor"] == "https://mobilizon.fr/@metacartes"
object = data["object"]
assert object["id"] ==
"https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0"
"https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93"
assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@ -63,9 +63,9 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
# "http://localtesting.pleroma.lol/users/lain"
# ]
assert object["actor"] == "https://test.mobilizon.org/@Alicia"
assert object["actor"] == "https://mobilizon.fr/@metacartes"
assert object["location"]["name"] == "Locaux de Framasoft"
# assert object["attributedTo"] == "https://test.mobilizon.org/@Alicia"
# assert object["attributedTo"] == "https://mobilizon.fr/@metacartes"
assert event.physical_address.street == "10 Rue Jangot"
@ -84,8 +84,8 @@ defmodule Mobilizon.Federation.ActivityPub.TransmogrifierTest do
%Actor{url: actor_url, id: actor_id} =
actor =
insert(:actor,
domain: "test.mobilizon.org",
url: "https://test.mobilizon.org/@member",
domain: "mobilizon.fr",
url: "https://mobilizon.fr/@member",
preferred_username: "member"
)

16
test/fixtures/mastodon-update.json vendored
View File

@ -20,15 +20,15 @@
"endpoints": {
"sharedInbox": "https://framapiaf.org/inbox"
},
"icon":{
"type":"Image",
"mediaType":"image/png",
"url":"https://files.mastodon.social/accounts/avatars/000/000/001/original/a285c086605e4182.png"
"icon": {
"type": "Image",
"mediaType": "image/png",
"url": "https://files.mastodon.social/accounts/avatars/000/000/001/original/d96d39a0abb45b92.jpg"
},
"image":{
"type":"Image",
"mediaType":"image/png",
"url":"https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png"
"image": {
"type": "Image",
"mediaType": "image/png",
"url": "https://files.mastodon.social/accounts/headers/000/000/001/original/c91b871f294ea63e.png"
}
},
"id": "https://framapiaf.org/users/gargron#updates/1519563538",

18
test/fixtures/mobilizon-post-activity.json vendored
View File

@ -28,12 +28,12 @@
"uuid": "sc:identifier"
}
],
"actor": "https://test.mobilizon.org/@Alicia",
"actor": "https://mobilizon.fr/@metacartes",
"cc": [
"https://framapiaf.org/users/admin/followers",
"https://framapiaf.org/users/tcit"
],
"id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0/activity",
"id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93/activity",
"object": {
"attachment": [
{
@ -49,7 +49,7 @@
"type": "Link"
}
],
"attributedTo": "https://test.mobilizon.org/@Alicia",
"attributedTo": "https://mobilizon.fr/@metacartes",
"startTime": "2018-02-12T14:08:20Z",
"cc": [
"https://framapiaf.org/users/admin/followers",
@ -57,7 +57,7 @@
],
"content": "<p><span class=\"h-card\"><a href=\"https://framapiaf.org/users/tcit\" class=\"u-url mention\">@<span>tcit</span></a></span></p>",
"category": "TODO remove me",
"id": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0",
"id": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
"inReplyTo": null,
"location": {
"type": "Place",
@ -81,16 +81,12 @@
"type": "Mention"
}
],
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"to": ["https://www.w3.org/ns/activitystreams#Public"],
"type": "Event",
"url": "https://test.mobilizon.org/events/39026210-0c69-4238-b3cc-986f33f98ed0",
"url": "https://mobilizon.fr/events/39a0c4a6-f2b6-41dc-bbe2-fc5bff76cc93",
"uuid": "109ccdfd-ee3e-46e1-a877-6c228763df0c"
},
"published": "2018-02-12T14:08:20Z",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"to": ["https://www.w3.org/ns/activitystreams#Public"],
"type": "Create"
}

105
test/fixtures/vcr_cassettes/activity_pub/event_update_activities.json vendored
View File

File diff suppressed because one or more lines are too long

105
test/fixtures/vcr_cassettes/activity_pub/fetch_mobilizon_post_activity.json vendored
View File

File diff suppressed because one or more lines are too long

119
test/fixtures/vcr_cassettes/activity_pub/fetch_tcit@framapiaf.org.json vendored
View File

File diff suppressed because one or more lines are too long

123
test/fixtures/vcr_cassettes/activity_pub/fetch_tcit@framapiaf.org_not_discoverable.json vendored
View File

File diff suppressed because one or more lines are too long

82
test/fixtures/vcr_cassettes/activity_pub/mastodon_follow_activity.json vendored
View File

File diff suppressed because one or more lines are too long

13
test/fixtures/vcr_cassettes/activity_pub/mobilizon_parent_resource.json vendored
View File

File diff suppressed because one or more lines are too long

110
test/fixtures/vcr_cassettes/activity_pub/signature/invalid_not_found.json vendored
View File

File diff suppressed because one or more lines are too long

96
test/fixtures/vcr_cassettes/activity_pub/signature/invalid_payload.json vendored
View File

File diff suppressed because one or more lines are too long

87
test/fixtures/vcr_cassettes/activity_pub/signature/valid.json vendored
View File

File diff suppressed because one or more lines are too long

87
test/fixtures/vcr_cassettes/activity_pub/signature/valid_payload.json vendored
View File

File diff suppressed because one or more lines are too long

250
test/fixtures/vcr_cassettes/activity_pub/update_actor_activity.json vendored
View File

File diff suppressed because one or more lines are too long

174
test/fixtures/vcr_cassettes/activity_pub_controller/mastodon-post-activity_actor_call.json vendored
View File

File diff suppressed because one or more lines are too long

4
test/mobilizon/actors/actors_test.exs
View File

@ -108,7 +108,7 @@ defmodule Mobilizon.ActorsTest do
avatar: %FileModel{name: picture_name} = _picture
} = _actor} = ActivityPub.get_or_fetch_actor_by_url(@remote_account_url)
assert picture_name == "avatar"
assert picture_name == "a28c50ce5f2b13fd.jpg"
%Actor{
id: actor_found_id,
@ -116,7 +116,7 @@ defmodule Mobilizon.ActorsTest do
} = Actors.get_actor_by_name("#{preferred_username}@#{domain}")
assert actor_found_id == actor_id
assert picture_name == "avatar"
assert picture_name == "a28c50ce5f2b13fd.jpg"
end
end

2
test/web/plugs/mapped_identity_to_signature_test.exs
View File

@ -52,7 +52,7 @@ defmodule Mobilizon.Web.Plugs.MappedSignatureToIdentityTest do
use_cassette "activity_pub/signature/invalid_not_found" do
conn =
build_conn(:post, "/doesntmattter", %{"actor" => "https://framapiaf.org/users/admin"})
|> set_signature("http://niu.moe/users/rye")
|> set_signature("https://mastodon.social/users/gargron")
|> MappedSignatureToIdentity.call(%{})
assert %{valid_signature: false} == conn.assigns

185
test/web/proxy/media_proxy_test.exs
View File

@ -1,185 +0,0 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/blob/develop/test/media_proxy_test.ex
defmodule Mobilizon.Web.MediaProxyTest do
use ExUnit.Case
import Mobilizon.Web.MediaProxy
alias Mobilizon.Config
alias Mobilizon.Web.{Endpoint, MediaProxyController}
setup do
enabled = Config.get([:media_proxy, :enabled])
on_exit(fn -> Config.put([:media_proxy, :enabled], enabled) end)
:ok
end
describe "when enabled" do
setup do
Config.put([:media_proxy, :enabled], true)
:ok
end
test "ignores invalid url" do
assert url(nil) == nil
assert url("") == nil
end
test "ignores relative url" do
assert url("/local") == "/local"
assert url("/") == "/"
end
test "ignores local url" do
local_url = Endpoint.url() <> "/hello"
local_root = Endpoint.url()
assert url(local_url) == local_url
assert url(local_root) == local_root
end
test "encodes and decodes URL" do
url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url)
assert String.starts_with?(
encoded,
Config.get([:media_proxy, :base_url], Endpoint.url())
)
assert String.ends_with?(encoded, "/logo.png")
assert decode_result(encoded) == url
end
test "encodes and decodes URL without a path" do
url = "https://pleroma.soykaf.com"
encoded = url(url)
assert decode_result(encoded) == url
end
test "encodes and decodes URL without an extension" do
url = "https://pleroma.soykaf.com/path/"
encoded = url(url)
assert String.ends_with?(encoded, "/path")
assert decode_result(encoded) == url
end
test "encodes and decodes URL and ignores query params for the path" do
url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true"
encoded = url(url)
assert String.ends_with?(encoded, "/logo.png")
assert decode_result(encoded) == url
end
test "ensures urls are url-encoded" do
assert decode_result(url("https://pleroma.social/Hello world.jpg")) ==
"https://pleroma.social/Hello%20world.jpg"
assert decode_result(url("https://pleroma.social/Hello%20world.jpg")) ==
"https://pleroma.social/Hello%20world.jpg"
end
test "validates signature" do
secret_key_base = Config.get([Endpoint, :secret_key_base])
on_exit(fn ->
Config.put([Endpoint, :secret_key_base], secret_key_base)
end)
encoded = url("https://pleroma.social")
Config.put(
[Endpoint, :secret_key_base],
"00000000000000000000000000000000000000000000000"
)
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
assert decode_url(sig, base64) == {:error, :invalid_signature}
end
test "filename_matches matches url encoded paths" do
assert MediaProxyController.filename_matches(
true,
"/Hello%20world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
true,
"/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
end
test "filename_matches matches non-url encoded paths" do
assert MediaProxyController.filename_matches(
true,
"/Hello world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
true,
"/Hello world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
end
test "uses the configured base_url" do
base_url = Config.get([:media_proxy, :base_url])
if base_url do
on_exit(fn ->
Config.put([:media_proxy, :base_url], base_url)
end)
end
Config.put([:media_proxy, :base_url], "https://cache.pleroma.social")
url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url)
assert String.starts_with?(encoded, Config.get([:media_proxy, :base_url]))
end
# https://git.pleroma.social/pleroma/pleroma/issues/580
test "encoding S3 links (must preserve `%2F`)" do
url =
"https://s3.amazonaws.com/example/test.png?X-Amz-Credential=your-access-key-id%2F20130721%2Fus-east-1%2Fs3%2Faws4_request"
encoded = url(url)
assert decode_result(encoded) == url
end
end
describe "when disabled" do
setup do
enabled = Config.get([:media_proxy, :enabled])
if enabled do
Config.put([:media_proxy, :enabled], false)
on_exit(fn ->
Config.put([:media_proxy, :enabled], enabled)
:ok
end)
end
:ok
end
test "does not encode remote urls" do
assert url("https://google.fr") == "https://google.fr"
end
end
defp decode_result(encoded) do
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
{:ok, decoded} = decode_url(sig, base64)
decoded
end
end