Add more tests to upload filters

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

.gitignore

@@ -29,6 +29,7 @@ priv/errors/*
 cover/
 site/
 test/fixtures/image_tmp.jpg
+test/fixtures/DSCN0010_tmp.jpg
 test/uploads/
 uploads/*
 release/

lib/mobilizon/config.ex

@@ -276,17 +276,19 @@ defmodule Mobilizon.Config do
     end
   end
 
-  @spec put([module | atom], any) :: any
   def put([key], value), do: put(key, value)
 
   def put([parent_key | keys], value) do
-    parent = put_in(Application.get_env(:mobilizon, parent_key), keys, value)
+    parent =
+      Application.get_env(:mobilizon, parent_key, [])
+      |> put_in(keys, value)
 
     Application.put_env(:mobilizon, parent_key, parent)
   end
 
-  @spec put(module | atom, any) :: any
+  def put(key, value) do
-  def put(key, value), do: Application.put_env(:mobilizon, key, value)
+    Application.put_env(:mobilizon, key, value)
+  end
 
   @spec to_boolean(boolean | String.t()) :: boolean
   defp to_boolean(boolean), do: "true" == String.downcase("#{boolean}")

lib/mobilizon/utils.ex

@@ -0,0 +1,25 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Utils do
+  @moduledoc """
+  Module that provide generic utils functions for Mobilizon
+  """
+
+  @doc """
+  POSIX-compliant check if command is available in the system
+
+  ## Examples
+      iex> command_available?("git")
+      true
+      iex> command_available?("wrongcmd")
+      false
+
+  """
+  @spec command_available?(String.t()) :: boolean()
+  def command_available?(command) do
+    match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
+  end
+end

lib/web/upload/filter/anonymize_filename.ex

@@ -13,11 +13,20 @@ defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilename do
   @behaviour Mobilizon.Web.Upload.Filter
 
   alias Mobilizon.Config
+  alias Mobilizon.Web.Upload
 
-  def filter(upload) do
+  def filter(%Upload{name: name} = upload) do
-    extension = List.last(String.split(upload.name, "."))
+    extension = List.last(String.split(name, "."))
-    name = Config.get([__MODULE__, :text], random(extension))
+    name = predefined_name(extension) || random(extension)
-    {:ok, %Mobilizon.Web.Upload{upload | name: name}}
+    {:ok, :filtered, %Upload{upload | name: name}}
+  end
+
+  def filter(_), do: {:ok, :noop}
+
+  @spec predefined_name(String.t()) :: String.t() | nil
+  defp predefined_name(extension) do
+    with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
+         do: String.replace(name, "{extension}", extension)
   end
 
   defp random(extension) do

lib/web/upload/filter/dedupe.ex

@@ -10,11 +10,13 @@ defmodule Mobilizon.Web.Upload.Filter.Dedupe do
   @behaviour Mobilizon.Web.Upload.Filter
   alias Mobilizon.Web.Upload
 
-  def filter(%Upload{name: name} = upload) do
+  def filter(%Upload{name: name, tempfile: tempfile} = upload) do
     extension = name |> String.split(".") |> List.last()
-    shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
+    shasum = :crypto.hash(:sha256, File.read!(tempfile)) |> Base.encode16(case: :lower)
     filename = shasum <> "." <> extension
 
-    {:ok, %Upload{upload | id: shasum, path: filename}}
+    {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
   end
+
+  def filter(_), do: {:ok, :noop}
 end

lib/web/upload/filter/exiftool.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.Exiftool do
+  @moduledoc """
+  Strips GPS related EXIF tags and overwrites the file in place.
+  Also strips or replaces filesystem metadata e.g., timestamps.
+  """
+  alias Mobilizon.Web.Upload
+
+  @behaviour Mobilizon.Web.Upload.Filter
+
+  @spec filter(Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  # webp is not compatible with exiftool at this time
+  def filter(%Upload{content_type: "image/webp"}), do: {:ok, :noop}
+
+  def filter(%Upload{tempfile: file, content_type: "image" <> _}) do
+    case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
+      {_response, 0} -> {:ok, :filtered}
+      {error, 1} -> {:error, error}
+    end
+  rescue
+    _e in ErlangError ->
+      {:error, "exiftool command not found"}
+  end
+
+  def filter(_), do: {:ok, :noop}
+end

lib/web/upload/filter/filter.ex

@@ -17,7 +17,10 @@ defmodule Mobilizon.Web.Upload.Filter do
   require Logger
 
   @callback filter(Mobilizon.Web.Upload.t()) ::
-              :ok | {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
+              {:ok, :filtered}
+              | {:ok, :noop}
+              | {:ok, :filtered, Mobilizon.Web.Upload.t()}
+              | {:error, any()}
 
   @spec filter([module()], Mobilizon.Web.Upload.t()) ::
           {:ok, Mobilizon.Web.Upload.t()} | {:error, any()}
@@ -28,10 +31,13 @@ defmodule Mobilizon.Web.Upload.Filter do
 
   def filter([filter | rest], upload) do
     case filter.filter(upload) do
-      :ok ->
+      {:ok, :filtered} ->
         filter(rest, upload)
 
-      {:ok, upload} ->
+      {:ok, :filtered, upload} ->
+        filter(rest, upload)
+
+      {:ok, :noop} ->
         filter(rest, upload)
 
       error ->

lib/web/upload/filter/mogrify.ex

@@ -15,19 +15,24 @@ defmodule Mobilizon.Web.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
+  @spec filter(Mobilizon.Web.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
   def filter(%Mobilizon.Web.Upload{tempfile: file, content_type: "image" <> _}) do
-    filters = Config.get!([__MODULE__, :args])
+    do_filter(file, Config.get!([__MODULE__, :args]))
+    {:ok, :filtered}
+  rescue
+    _e in ErlangError ->
+      {:error, "mogrify command not found"}
+  end
 
+  def filter(_), do: {:ok, :noop}
+
+  def do_filter(file, filters) do
     file
     |> Mogrify.open()
     |> mogrify_filter(filters)
     |> Mogrify.save(in_place: true)
-
-    :ok
   end
 
-  def filter(_), do: :ok
-
   defp mogrify_filter(mogrify, nil), do: mogrify
 
   defp mogrify_filter(mogrify, [filter | rest]) do

lib/web/upload/filter/optimize.ex

@@ -21,7 +21,7 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
 
     case ExOptimizer.optimize(file, deps: optimizers) do
       {:ok, _res} ->
-        :ok
+        {:ok, :filtered}
 
       {:error, err} ->
         require Logger
@@ -30,12 +30,12 @@ defmodule Mobilizon.Web.Upload.Filter.Optimize do
           "Unable to optimize file #{file}. The return from the process was #{inspect(err)}"
         )
 
-        :ok
+        {:ok, :noop}
 
       err ->
-        err
+        {:error, err}
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end

lib/web/upload/uploader/local.ex

@@ -12,10 +12,12 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
 
   alias Mobilizon.Config
 
+  @impl true
   def get_file(_) do
     {:ok, {:static_dir, upload_path()}}
   end
 
+  @impl true
   def put_file(upload) do
     {path, file} = local_path(upload.path)
     result_file = Path.join(path, file)
@@ -27,6 +29,7 @@ defmodule Mobilizon.Web.Upload.Uploader.Local do
     :ok
   end
 
+  @impl true
   def remove_file(path) do
     with {path, file} <- local_path(path),
          full_path <- Path.join(path, file),

test/web/upload/filter/anonymize_filename_test.exs

@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.AnonymizeFilenameTest do
+  use Mobilizon.DataCase
+  use Mobilizon.Tests.Helpers
+
+  alias Mobilizon.Config
+  alias Mobilizon.Web.Upload
+
+  setup do
+    File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
+    upload_file = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    %{upload_file: upload_file}
+  end
+
+  setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
+
+  test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
+    Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
+    {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+    assert name == "custom-file.png"
+  end
+
+  test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
+    Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
+    {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+    assert name == "custom-file.jpg"
+  end
+
+  test "it replaces filename on random text", %{upload_file: upload_file} do
+    {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
+    assert <<_::bytes-size(14)>> <> ".jpg" = name
+    refute name == "an… image.jpg"
+  end
+end

test/web/upload/filter/dedupe_test.exs

@@ -0,0 +1,33 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.DedupeTest do
+  use Mobilizon.DataCase
+
+  alias Mobilizon.Web.Upload
+  alias Mobilizon.Web.Upload.Filter.Dedupe
+
+  @shasum "590523d60d3831ec92d05cdd871078409d5780903910efec5cd35ab1b0f19d11"
+
+  test "adds shasum" do
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    assert {
+             :ok,
+             :filtered,
+             %Upload{id: @shasum, path: @shasum <> ".jpg"}
+           } = Dedupe.filter(upload)
+  end
+end

test/web/upload/filter/exiftool_test.exs

@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.ExiftoolTest do
+  use Mobilizon.DataCase
+  alias Mobilizon.Web.Upload
+  alias Mobilizon.Web.Upload.Filter
+
+  test "apply exiftool filter" do
+    assert Mobilizon.Utils.command_available?("exiftool")
+
+    File.cp!(
+      "test/fixtures/DSCN0010.jpg",
+      "test/fixtures/DSCN0010_tmp.jpg"
+    )
+
+    upload = %Mobilizon.Web.Upload{
+      name: "image_with_GPS_data.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/DSCN0010.jpg"),
+      tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
+    }
+
+    assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
+
+    {exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
+    {exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
+
+    refute exif_original == exif_filtered
+    assert String.match?(exif_original, ~r/GPS/)
+    refute String.match?(exif_filtered, ~r/GPS/)
+  end
+
+  test "verify webp files are skipped" do
+    upload = %Upload{
+      name: "sample.webp",
+      content_type: "image/webp"
+    }
+
+    assert Filter.Exiftool.filter(upload) == {:ok, :noop}
+  end
+end

test/web/upload/filter/filter_test.exs

@@ -0,0 +1,35 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.FilterTest do
+  use Mobilizon.DataCase
+  use Mobilizon.Tests.Helpers
+
+  alias Mobilizon.Config
+  alias Mobilizon.Web.Upload.Filter
+
+  setup do: clear_config([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text])
+
+  test "applies filters" do
+    Config.put([Mobilizon.Web.Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
+
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Mobilizon.Web.Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    assert Filter.filter([], upload) == {:ok, upload}
+
+    assert {:ok, upload} = Filter.filter([Mobilizon.Web.Upload.Filter.AnonymizeFilename], upload)
+    assert upload.name == "custom-file.png"
+  end
+end

test/web/upload/filter/mogrify_test.exs

@@ -0,0 +1,44 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Filter.MogrifyTest do
+  use Mobilizon.DataCase
+  use Mobilizon.Tests.Helpers
+  import Mock
+
+  alias Mobilizon.Web.Upload
+  alias Mobilizon.Web.Upload.Filter
+
+  test "apply mogrify filter" do
+    clear_config(Filter.Mogrify, args: [{"tint", "40"}])
+
+    File.cp!(
+      "test/fixtures/image.jpg",
+      "test/fixtures/image_tmp.jpg"
+    )
+
+    upload = %Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_tmp.jpg"),
+      tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+    }
+
+    task =
+      Task.async(fn ->
+        assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000
+      end)
+
+    with_mock Mogrify,
+      open: fn _f -> %Mogrify.Image{} end,
+      custom: fn _m, _a -> :ok end,
+      custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end,
+      save: fn _f, _o -> :ok end do
+      assert Filter.Mogrify.filter(upload) == {:ok, :filtered}
+    end
+
+    Task.await(task)
+  end
+end

test/web/upload/upload_test.exs

@@ -79,7 +79,6 @@ defmodule Mobilizon.UploadTest do
       assert data.name == "an [image.jpg"
     end
 
-    @tag :skip
     test "fixes incorrect content type" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 

test/web/upload/uploader/local_test.exs

@@ -0,0 +1,58 @@
+# Portions of this file are derived from Pleroma:
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mobilizon.Web.Upload.Uploader.LocalTest do
+  use Mobilizon.DataCase
+  alias Mobilizon.Web.Upload
+  alias Mobilizon.Web.Upload.Uploader.Local
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+    end
+  end
+
+  describe "put_file/1" do
+    test "put file to local folder" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+      file_path = "local_upload/files/image.jpg"
+
+      file = %Upload{
+        name: "image.jpg",
+        content_type: "image/jpeg",
+        path: file_path,
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      assert Local.put_file(file) == :ok
+
+      assert [Local.upload_path(), file_path]
+             |> Path.join()
+             |> File.exists?()
+    end
+  end
+
+  describe "remove_file/1" do
+    test "removes local file" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+      file_path = "local_upload/files/image.jpg"
+
+      file = %Upload{
+        name: "image.jpg",
+        content_type: "image/jpeg",
+        path: file_path,
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      :ok = Local.put_file(file)
+      local_path = Path.join([Local.upload_path(), file_path])
+      assert File.exists?(local_path)
+
+      Local.remove_file(file_path)
+
+      refute File.exists?(local_path)
+    end
+  end
+end