From a24e08a6de55f91538978431fc254dcb72ca5381 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Thu, 10 Jun 2021 09:36:25 +0200 Subject: [PATCH] Add blurhash support to backend Signed-off-by: Thomas Citharel --- config/config.exs | 1 + lib/federation/activity_pub/utils.ex | 12 ++--- .../activity_stream/converter/actor.ex | 2 +- .../activity_stream/converter/media.ex | 12 ++--- lib/graphql/api/events.ex | 17 ++++--- lib/graphql/resolvers/media.ex | 19 ++++++-- lib/graphql/resolvers/post.ex | 17 ++++--- lib/graphql/schema/media.ex | 10 ++++ lib/mobilizon/discussions/discussions.ex | 3 +- lib/mobilizon/events/event.ex | 2 +- lib/mobilizon/medias/media.ex | 21 ++++++++- lib/web/upload/filter/analyze_metadata.ex | 47 +++++++++++++++++++ lib/web/upload/upload.ex | 9 ++-- mix.exs | 3 ++ mix.lock | 1 + .../20210608133040_add_metadata_to_media.exs | 9 ++++ .../upload/filter/analyze_metadata_test.exs | 20 ++++++++ 17 files changed, 161 insertions(+), 44 deletions(-) create mode 100644 lib/web/upload/filter/analyze_metadata.ex create mode 100644 priv/repo/migrations/20210608133040_add_metadata_to_media.exs create mode 100644 test/web/upload/filter/analyze_metadata_test.exs diff --git a/config/config.exs b/config/config.exs index a73bba6a..52b94973 100644 --- a/config/config.exs +++ b/config/config.exs @@ -66,6 +66,7 @@ config :mobilizon, Mobilizon.Web.Upload, uploader: Mobilizon.Web.Upload.Uploader.Local, filters: [ Mobilizon.Web.Upload.Filter.Dedupe, + Mobilizon.Web.Upload.Filter.AnalyzeMetadata, Mobilizon.Web.Upload.Filter.Optimize ], allow_list_mime_types: ["image/gif", "image/jpeg", "image/png", "image/webp"], diff --git a/lib/federation/activity_pub/utils.ex b/lib/federation/activity_pub/utils.ex index fe5fd4f0..fcb1d340 100644 --- a/lib/federation/activity_pub/utils.ex +++ b/lib/federation/activity_pub/utils.ex @@ -352,18 +352,14 @@ defmodule Mobilizon.Federation.ActivityPub.Utils do end def make_media_data(media) when is_map(media) do - with {:ok, %{"url" => [%{"href" => url, "mediaType" => content_type}], "size" => size}} <- + with {:ok, %{url: url} = uploaded} <- Mobilizon.Web.Upload.store(media.file), {:media_exists, nil} <- {:media_exists, Mobilizon.Medias.get_media_by_url(url)}, {:ok, %Media{file: _file} = media} <- Mobilizon.Medias.create_media(%{ - "file" => %{ - "url" => url, - "name" => media.name, - "content_type" => content_type, - "size" => size - }, - "actor_id" => media.actor_id + file: Map.take(uploaded, [:url, :name, :content_type, :size]), + metadata: Map.take(uploaded, [:width, :height, :blurhash]), + actor_id: media.actor_id }) do Converter.Media.model_to_as(media) else diff --git a/lib/federation/activity_stream/converter/actor.ex b/lib/federation/activity_stream/converter/actor.ex index 2f39e6e8..54c087a7 100644 --- a/lib/federation/activity_stream/converter/actor.ex +++ b/lib/federation/activity_stream/converter/actor.ex @@ -143,7 +143,7 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do when code in 200..299 <- RemoteMediaDownloaderClient.get(url), name <- name || Parser.get_filename_from_response(response_headers, url) || default_name, {:ok, file} <- Upload.store(%{body: body, name: name}) do - file + Map.take(file, [:content_type, :name, :url, :size]) end end end diff --git a/lib/federation/activity_stream/converter/media.ex b/lib/federation/activity_stream/converter/media.ex index 0f1b350e..d89aa2de 100644 --- a/lib/federation/activity_stream/converter/media.ex +++ b/lib/federation/activity_stream/converter/media.ex @@ -40,17 +40,13 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Media do ) when is_binary(media_url) do with {:ok, %{body: body}} <- Tesla.get(media_url, opts: @http_options), - {:ok, %{name: name, url: url, content_type: content_type, size: size}} <- + {:ok, %{url: url} = uploaded} <- Upload.store(%{body: body, name: name}), {:media_exists, nil} <- {:media_exists, Medias.get_media_by_url(url)} do Medias.create_media(%{ - "file" => %{ - "url" => url, - "name" => name, - "content_type" => content_type, - "size" => size - }, - "actor_id" => actor_id + file: Map.take(uploaded, [:url, :name, :content_type, :size]), + metadata: Map.take(uploaded, [:width, :height, :blurhash]), + actor_id: actor_id }) else {:media_exists, %MediaModel{file: _file} = media} -> diff --git a/lib/graphql/api/events.ex b/lib/graphql/api/events.ex index 983a18b4..a07d1b0d 100644 --- a/lib/graphql/api/events.ex +++ b/lib/graphql/api/events.ex @@ -52,13 +52,16 @@ defmodule Mobilizon.GraphQL.API.Events do defp process_picture(%{media_id: _picture_id} = args, _), do: args defp process_picture(%{media: media}, %Actor{id: actor_id}) do - %{ - file: - media - |> Map.get(:file) - |> Utils.make_media_data(description: Map.get(media, :name)), - actor_id: actor_id - } + with uploaded when is_map(uploaded) <- + media + |> Map.get(:file) + |> Utils.make_media_data(description: Map.get(media, :name)) do + %{ + file: Map.take(uploaded, [:url, :name, :content_type, :size]), + metadata: Map.take(uploaded, [:width, :height, :blurhash]), + actor_id: actor_id + } + end end @spec extract_pictures_from_event_body(map(), Actor.t()) :: map() diff --git a/lib/graphql/resolvers/media.ex b/lib/graphql/resolvers/media.ex index 2eeddcbf..20ab8e90 100644 --- a/lib/graphql/resolvers/media.ex +++ b/lib/graphql/resolvers/media.ex @@ -47,7 +47,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do %{context: %{current_user: %User{} = user}} ) do with %Actor{id: actor_id} <- Users.get_actor_for_user(user), - {:ok, %{name: _name, url: url, content_type: content_type, size: size}} <- + {:ok, + %{ + name: _name, + url: url, + content_type: content_type, + size: size + } = uploaded} <- Mobilizon.Web.Upload.store(file), args <- args @@ -55,7 +61,11 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do |> Map.put(:size, size) |> Map.put(:content_type, content_type), {:ok, media = %Media{}} <- - Medias.create_media(%{"file" => args, "actor_id" => actor_id}) do + Medias.create_media(%{ + file: args, + actor_id: actor_id, + metadata: Map.take(uploaded, [:width, :height, :blurhash]) + }) do {:ok, transform_media(media)} else {:error, :mime_type_not_allowed} -> @@ -124,13 +134,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do def user_size(_parent, _args, _resolution), do: {:error, :unauthenticated} @spec transform_media(Media.t()) :: map() - defp transform_media(%Media{id: id, file: file}) do + defp transform_media(%Media{id: id, file: file, metadata: metadata}) do %{ name: file.name, url: file.url, id: id, content_type: file.content_type, - size: file.size + size: file.size, + metadata: metadata } end diff --git a/lib/graphql/resolvers/post.ex b/lib/graphql/resolvers/post.ex index 34eae24f..24c952fb 100644 --- a/lib/graphql/resolvers/post.ex +++ b/lib/graphql/resolvers/post.ex @@ -215,13 +215,16 @@ defmodule Mobilizon.GraphQL.Resolvers.Post do defp process_picture(%{media_id: _picture_id} = args, _), do: args defp process_picture(%{media: media}, %Actor{id: actor_id}) do - %{ - file: - media - |> Map.get(:file) - |> Utils.make_media_data(description: Map.get(media, :name)), - actor_id: actor_id - } + with uploaded when is_map(uploaded) <- + media + |> Map.get(:file) + |> Utils.make_media_data(description: Map.get(media, :name)) do + %{ + file: Map.take(uploaded, [:url, :name, :content_type, :size]), + metadata: Map.take(uploaded, [:width, :height, :blurhash]), + actor_id: actor_id + } + end end @spec extract_pictures_from_post_body(map(), String.t()) :: map() diff --git a/lib/graphql/schema/media.ex b/lib/graphql/schema/media.ex index 643eb7d1..4674f89f 100644 --- a/lib/graphql/schema/media.ex +++ b/lib/graphql/schema/media.ex @@ -14,6 +14,7 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do field(:url, :string, description: "The media's full URL") field(:content_type, :string, description: "The media's detected content type") field(:size, :integer, description: "The media's size") + field(:metadata, :media_metadata, description: "The media's metadata") end @desc """ @@ -24,6 +25,15 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do field(:total, :integer, description: "The total number of medias in the list") end + @desc """ + Some metadata associated with a media + """ + object :media_metadata do + field(:width, :integer, description: "The media width (if a picture)") + field(:height, :integer, description: "The media width (if a height)") + field(:blurhash, :string, description: "The media blurhash (if a picture") + end + @desc "An attached media or a link to a media" input_object :media_input do # Either a full media object diff --git a/lib/mobilizon/discussions/discussions.ex b/lib/mobilizon/discussions/discussions.ex index 7710ea3d..5c10d0e9 100644 --- a/lib/mobilizon/discussions/discussions.ex +++ b/lib/mobilizon/discussions/discussions.ex @@ -80,11 +80,12 @@ defmodule Mobilizon.Discussions do # However, it also excludes all top-level comments with deleted replies from being selected # |> where([_, r], is_nil(r.deleted_at)) |> group_by([c], c.id) + |> order_by([c], desc: :is_announcement, asc: :published_at) |> select([c, r], %{c | total_replies: count(r.id)}) end def query(Comment, _) do - order_by(Comment, [c], asc: :published_at) + order_by(Comment, [c], asc: :is_announcement, asc: :published_at) end def query(queryable, _) do diff --git a/lib/mobilizon/events/event.ex b/lib/mobilizon/events/event.ex index 1121d770..e86d0bc5 100644 --- a/lib/mobilizon/events/event.ex +++ b/lib/mobilizon/events/event.ex @@ -256,7 +256,7 @@ defmodule Mobilizon.Events.Event do # In case it's a new picture defp put_picture(%Changeset{} = changeset, _attrs) do - cast_assoc(changeset, :picture) + cast_assoc(changeset, :picture, with: &Media.changeset/2) end # Created or updated with draft parameter: don't publish diff --git a/lib/mobilizon/medias/media.ex b/lib/mobilizon/medias/media.ex index 8742dad1..0133b5bf 100644 --- a/lib/mobilizon/medias/media.ex +++ b/lib/mobilizon/medias/media.ex @@ -5,21 +5,32 @@ defmodule Mobilizon.Medias.Media do use Ecto.Schema - import Ecto.Changeset, only: [cast: 3, cast_embed: 2] + import Ecto.Changeset, only: [cast: 3, cast_embed: 2, cast_embed: 3] alias Mobilizon.Actors.Actor alias Mobilizon.Discussions.Comment alias Mobilizon.Events.Event alias Mobilizon.Medias.File + alias Mobilizon.Medias.Media.Metadata alias Mobilizon.Posts.Post @type t :: %__MODULE__{ file: File.t(), + metadata: Metadata.t(), actor: Actor.t() } + @metadata_attrs [:height, :width, :blurhash] + schema "medias" do embeds_one(:file, File, on_replace: :update) + + embeds_one :metadata, Metadata, on_replace: :update do + field(:height, :integer) + field(:width, :integer) + field(:blurhash, :string) + end + belongs_to(:actor, Actor) has_many(:event_picture, Event, foreign_key: :picture_id) many_to_many(:events, Event, join_through: "events_medias") @@ -36,5 +47,13 @@ defmodule Mobilizon.Medias.Media do media |> cast(attrs, [:actor_id]) |> cast_embed(:file) + |> cast_embed(:metadata, with: &metadata_changeset/2) + end + + @doc false + @spec changeset(struct(), map) :: Ecto.Changeset.t() + def metadata_changeset(metadata, attrs) do + metadata + |> cast(attrs, @metadata_attrs) end end diff --git a/lib/web/upload/filter/analyze_metadata.ex b/lib/web/upload/filter/analyze_metadata.ex new file mode 100644 index 00000000..cb7acbba --- /dev/null +++ b/lib/web/upload/filter/analyze_metadata.ex @@ -0,0 +1,47 @@ +# Portions of this file are derived from Pleroma: +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/lib/pleroma/upload/filter/analyze_metadata.ex + +defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadata do + @moduledoc """ + Extracts metadata about the upload, such as width/height + """ + require Logger + alias Mobilizon.Web.Upload + + @behaviour Mobilizon.Web.Upload.Filter + + @spec filter(Upload.t()) :: + {:ok, :filtered, Upload.t()} | {:ok, :noop} | {:error, String.t()} + def filter(%Upload{tempfile: file, content_type: "image" <> _} = upload) do + image = + file + |> Mogrify.open() + |> Mogrify.verbose() + + upload = + upload + |> Map.put(:width, image.width) + |> Map.put(:height, image.height) + |> Map.put(:blurhash, get_blurhash(file)) + + {:ok, :filtered, upload} + rescue + e in ErlangError -> + Logger.warn("#{__MODULE__}: #{inspect(e)}") + {:ok, :noop} + end + + def filter(_), do: {:ok, :noop} + + defp get_blurhash(file) do + case :eblurhash.magick(to_charlist(file)) do + {:ok, blurhash} -> + to_string(blurhash) + + _ -> + nil + end + end +end diff --git a/lib/web/upload/upload.ex b/lib/web/upload/upload.ex index ee29d44d..9b7f1db9 100644 --- a/lib/web/upload/upload.ex +++ b/lib/web/upload/upload.ex @@ -73,12 +73,9 @@ defmodule Mobilizon.Web.Upload do {:ok, upload} <- Filter.filter(opts.filters, upload), {:ok, url_spec} <- Uploader.put_file(opts.uploader, upload) do {:ok, - %{ - name: Map.get(opts, :description) || upload.name, - url: url_from_spec(upload, opts.base_url, url_spec), - content_type: upload.content_type, - size: upload.size - }} + upload + |> Map.put(:name, Map.get(opts, :description) || upload.name) + |> Map.put(:url, url_from_spec(upload, opts.base_url, url_spec))} else {:error, error} -> Logger.error( diff --git a/mix.exs b/mix.exs index f5af930c..0ef2d7ca 100644 --- a/mix.exs +++ b/mix.exs @@ -162,6 +162,9 @@ defmodule Mobilizon.Mixfile do {:sweet_xml, "~> 0.6.6"}, {:web_push_encryption, git: "https://github.com/tcitworld/elixir-web-push-encryption", branch: "otp-24"}, + {:eblurhash, + git: "https://github.com/zotonic/eblurhash", + ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"}, # Dev and test dependencies {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]}, {:ex_machina, "~> 2.3", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index d9520581..0979faa1 100644 --- a/mix.lock +++ b/mix.lock @@ -25,6 +25,7 @@ "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, + "eblurhash": {:git, "https://github.com/zotonic/eblurhash", "04a0b76eadf4de1be17726f39b6313b88708fd12", [ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"]}, "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, "ecto_autoslug_field": {:hex, :ecto_autoslug_field, "2.0.1", "2177c1c253f6dd3efd4b56d1cb76104d0a6ef044c6b9a7a0ad6d32665c4111e5", [:mix], [{:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:slugger, ">= 0.2.0", [hex: :slugger, repo: "hexpm", optional: false]}], "hexpm", "a3cc73211f2e75b89a03332183812ebe1ac08be2e25a1df5aa3d1422f92c45c3"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, diff --git a/priv/repo/migrations/20210608133040_add_metadata_to_media.exs b/priv/repo/migrations/20210608133040_add_metadata_to_media.exs new file mode 100644 index 00000000..453db0c3 --- /dev/null +++ b/priv/repo/migrations/20210608133040_add_metadata_to_media.exs @@ -0,0 +1,9 @@ +defmodule Mobilizon.Storage.Repo.Migrations.AddMetadataToMedia do + use Ecto.Migration + + def change do + alter table(:medias) do + add(:metadata, :map) + end + end +end diff --git a/test/web/upload/filter/analyze_metadata_test.exs b/test/web/upload/filter/analyze_metadata_test.exs new file mode 100644 index 00000000..f5b7f7da --- /dev/null +++ b/test/web/upload/filter/analyze_metadata_test.exs @@ -0,0 +1,20 @@ +# Portions of this file are derived from Pleroma: +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/test/pleroma/upload/filter/analyze_metadata_test.exs + +defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadataTest do + use Mobilizon.DataCase, async: true + alias Mobilizon.Web.Upload.Filter.AnalyzeMetadata + + test "adds the image dimensions" do + upload = %Mobilizon.Web.Upload{ + name: "an… image.jpg", + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + tempfile: Path.absname("test/fixtures/image.jpg") + } + + assert {:ok, :filtered, %{width: 266, height: 67}} = AnalyzeMetadata.filter(upload) + end +end