Merge pull request #509 from eyra/fix-upload-local-fs

Fixed issues with local fs uploads

SELFHOSTING.md

@@ -134,7 +134,7 @@ Required environment variables:
 | DB_HOST | Hostname | "domain.where.database.lives" |
 | DB_NAME | Name of the database in the PostgreSQL| "self_prod" |
 | SECRET_KEY_BASE | 64-bit sequence of random characters | \<long-sequence-of-characters\> |
-| STATIC_PATH | Path to folder where uploaded files can be stored | "/home/self/uploads" |
+| STATIC_PATH | Path to folder where uploaded files can be stored | "/tmp" |
 | UNSPLASH_ACCESS_KEY | Application access key registered on [Unsplash](https://unsplash.com/) (Image Catalog) | "hcejpnHRuFWL-fKXLYqhGBt1Dz0_tTjeNifgD01VkGE" |
 | UNSPLASH_APP_NAME | Application name registered on [Unsplash](https://unsplash.com/) (Image Catalog) | "Self" |
 

core/config/dev.exs

@@ -1,5 +1,12 @@
 import Config
 
+upload_path =
+  File.cwd!()
+  |> Path.join("priv")
+  |> Path.join("static")
+  |> Path.join("uploads")
+  |> tap(&File.mkdir_p!/1)
+
 # Only in tests, remove the complexity from the password hashing algorithm
 config :bcrypt_elixir, :log_rounds, 1
 
@@ -65,13 +72,7 @@ config :core, Systems.Email.Mailer,
 
 config :core, :apns_backend, Core.APNS.LoggingBackend
 
-config :core,
-       :static_path,
-       File.cwd!()
-       |> Path.join("priv")
-       |> Path.join("static")
-       |> Path.join("uploads")
-       |> tap(&File.mkdir_p!/1)
+config :core, :upload_path, upload_path
 
 config :core,
        :admins,
@@ -85,23 +86,9 @@ config :core,
 #   access_key_id: "my_access_key",
 #   secret_access_key: "a_super_secret"
 
-config :core, :content,
-  backend: Systems.Content.LocalFS,
-  local_fs_root_path:
-    File.cwd!()
-    |> Path.join("priv")
-    |> Path.join("static")
-    |> Path.join("content")
-    |> tap(&File.mkdir_p!/1)
-
-config :core, :feldspar,
-  backend: Systems.Feldspar.LocalFS,
-  local_fs_root_path:
-    File.cwd!()
-    |> Path.join("priv")
-    |> Path.join("static")
-    |> Path.join("feldspar_apps")
-    |> tap(&File.mkdir_p!/1)
+config :core, :content, backend: Systems.Content.LocalFS
+
+config :core, :feldspar, backend: Systems.Feldspar.LocalFS
 
 try do
   import_config "dev.secret.exs"

core/config/runtime.exs

@@ -5,7 +5,7 @@ if config_env() == :prod do
   app_domain = System.fetch_env!("APP_DOMAIN")
   app_mail_domain = System.fetch_env!("APP_MAIL_DOMAIN")
   app_mail_noreply = "no-reply@#{app_mail_domain}"
-  static_path = System.fetch_env!("STATIC_PATH")
+  upload_path = System.fetch_env!("STATIC_PATH")
 
   # Allow enabling of features from an environment variable
   config :core,
@@ -19,7 +19,7 @@ if config_env() == :prod do
          :admins,
          System.get_env("APP_ADMINS", "") |> String.split() |> Systems.Admin.Public.compile()
 
-  config :core, :static_path, static_path
+  config :core, :upload_path, upload_path
 
   config :core, CoreWeb.Endpoint,
     cache_static_manifest: "priv/static/cache_manifest.json",
@@ -133,9 +133,7 @@ if config_env() == :prod do
       public_url: System.get_env("PUBLIC_S3_URL"),
       prefix: content_s3_prefix
   else
-    config :core, :content,
-      backend: Systems.Content.LocalFS,
-      local_fs_root_path: static_path
+    config :core, :content, backend: Systems.Content.LocalFS
   end
 
   if feldspar_s3_prefix = System.get_env("FELDSPAR_S3_PREFIX") do
@@ -147,9 +145,7 @@ if config_env() == :prod do
       public_url: System.get_env("PUBLIC_S3_URL"),
       prefix: feldspar_s3_prefix
   else
-    config :core, :feldspar,
-      backend: Systems.Feldspar.LocalFS,
-      local_fs_root_path: static_path
+    config :core, :feldspar, backend: Systems.Feldspar.LocalFS
   end
 
   config :core,

core/config/test.exs

@@ -55,10 +55,8 @@ config :core, :bundle, :next
 
 config :core, :banking_backend, Systems.Banking.Dummy
 
-config :core, :content,
-  backend: Systems.Content.LocalFS,
-  local_fs_root_path: "/tmp"
+config :core, :upload_path, "/tmp"
 
-config :core, :feldspar,
-  backend: Systems.Feldspar.LocalFS,
-  local_fs_root_path: "/tmp"
+config :core, :content, backend: Systems.Content.LocalFS
+
+config :core, :feldspar, backend: Systems.Feldspar.LocalFS

core/lib/core_web/controllers/uploaded_file.ex

@@ -1,9 +1,9 @@
 defmodule CoreWeb.UploadedFileController do
   use CoreWeb, :controller
-  import CoreWeb.FileUploader, only: [get_static_path: 1]
+  import CoreWeb.FileUploader, only: [get_upload_path: 1]
 
   def get(conn, %{"filename" => name}) do
-    path = get_static_path(name)
+    path = get_upload_path(name)
     send_file(conn, 200, path)
   end
 end

core/lib/core_web/file_uploader.ex

@@ -2,20 +2,23 @@ defmodule CoreWeb.FileUploader do
   @moduledoc """
   """
 
-  @allowed_filename_pattern ~r"^[a-z0-9][a-z0-9\-]+[a-z0-9]\.[a-z]{3,4}$"
+  @allowed_filename_pattern ~r"^[a-z0-9][a-z0-9\-]+[a-z0-9](\.[a-z]{3,4})?$"
 
   @callback process_file(socket :: Socket.t(), uploaded_file :: any()) :: Socket.t()
 
-  def get_static_path(filename) do
+  def get_upload_path(filename) do
     unless Regex.match?(@allowed_filename_pattern, filename) do
       throw(:invalid_filename)
     end
 
-    root = Application.get_env(:core, :static_path, "priv/static/uploads")
+    root = Application.get_env(:core, :upload_path, "priv/static/uploads")
     Path.join(root, filename)
   end
 
-  defmacro __using__(accept) do
+  defmacro __using__(opts) do
+    store = Keyword.get(opts, :store, Systems.Content.Public)
+    accept = Keyword.get(opts, :accept, ~w"*.*")
+
     quote do
       @behaviour CoreWeb.FileUploader
 
@@ -43,11 +46,9 @@ defmodule CoreWeb.FileUploader do
 
       def consume_file(socket, entry) do
         consume_uploaded_entry(socket, entry, fn %{path: tmp_path} ->
-          file = "#{entry.uuid}.#{ext(entry)}"
-          local_full_path = CoreWeb.FileUploader.get_static_path(file)
-          File.cp!(tmp_path, local_full_path)
-          local_relative_path = CoreWeb.Endpoint.static_path("/uploads/#{file}")
-          {:ok, {local_relative_path, local_full_path, entry.client_name}}
+          path = apply(unquote(store), :store, [tmp_path, entry.client_name])
+          public_url = apply(unquote(store), :get_public_url, [path])
+          {:ok, {path, public_url, entry.client_name}}
         end)
       end
 

core/lib/core_web/live/user/forms/profile.ex

@@ -1,6 +1,6 @@
 defmodule CoreWeb.User.Forms.Profile do
   use CoreWeb.LiveForm
-  use CoreWeb.FileUploader, ~w(.png .jpg .jpeg)
+  use CoreWeb.FileUploader, accept: ~w(.png .jpg .jpeg)
 
   alias Core.Accounts
   alias Core.Accounts.UserProfileEdit
@@ -11,9 +11,9 @@ defmodule CoreWeb.User.Forms.Profile do
   @impl true
   def process_file(
         %{assigns: %{entity: entity}} = socket,
-        {local_relative_path, _local_full_path, _remote_file}
+        {_path, photo_url, _original_filename}
       ) do
-    save(socket, entity, :auto_save, %{photo_url: local_relative_path})
+    save(socket, entity, :auto_save, %{photo_url: photo_url})
   end
 
   # Handle Selector Update

core/systems/admin/import_rewards_page.ex

@@ -1,7 +1,7 @@
 defmodule Systems.Admin.ImportRewardsPage do
   use CoreWeb, :live_view
   use CoreWeb.Layouts.Workspace.Component, :import_rewards
-  use CoreWeb.FileUploader, ~w(.csv)
+  use CoreWeb.FileUploader, accept: ~w(.csv)
 
   import CoreWeb.Layouts.Workspace.Component
 
@@ -19,19 +19,20 @@ defmodule Systems.Admin.ImportRewardsPage do
   @impl true
   def process_file(
         %{assigns: %{entity: entity}} = socket,
-        {_local_relative_path, local_full_path, remote_file}
+        {path, _url, original_filename}
       ) do
     changeset =
-      entity |> Admin.ImportRewardsModel.changeset(%{session_key: Path.rootname(remote_file)})
+      entity
+      |> Admin.ImportRewardsModel.changeset(%{session_key: Path.rootname(original_filename)})
 
     socket
     |> assign(
       changeset: changeset,
       lines_error: [],
       lines_unknown: [],
       lines_valid: [],
-      local_file: local_full_path,
-      uploaded_file: remote_file
+      local_file: path,
+      uploaded_file: original_filename
     )
   end
 

core/systems/assignment/crew_page.ex

@@ -117,11 +117,13 @@ defmodule Systems.Assignment.CrewPage do
       key: key
     }
 
-    assignment
-    |> Storage.Private.storage_info()
-    |> Storage.Public.store(panel_info, data, meta_data)
-
-    socket
+    if storage_info = Storage.Private.storage_info(assignment) do
+      Storage.Public.store(storage_info, panel_info, data, meta_data)
+    else
+      message = "Please setup connection to a data storage"
+      Logger.error(message)
+      socket |> put_flash(:error, message)
+    end
   end
 
   @impl true

core/systems/assignment/info_form.ex

@@ -1,7 +1,7 @@
 defmodule Systems.Assignment.InfoForm do
   use CoreWeb.LiveForm, :fabric
   use Fabric.LiveComponent
-  use CoreWeb.FileUploader, ~w(.png .jpg .jpeg .svg)
+  use CoreWeb.FileUploader, accept: ~w(.png .jpg .jpeg .svg)
 
   import Core.ImageCatalog, only: [image_catalog: 0]
   import Frameworks.Pixel.Form
@@ -11,17 +11,12 @@ defmodule Systems.Assignment.InfoForm do
   alias CoreWeb.UI.ImageCatalogPicker
 
   alias Systems.Assignment
-  alias Systems.Content
 
   @impl true
   def process_file(
         %{assigns: %{entity: entity}} = socket,
-        {_local_relative_path, local_full_path, _remote_file}
+        {_path, logo_url, _original_filename}
       ) do
-    logo_url =
-      Content.Public.store(local_full_path)
-      |> Content.Public.get_public_url()
-
     socket
     |> save(entity, :auto_save, %{logo_url: logo_url})
   end

core/systems/benchmark/import_form.ex

@@ -1,12 +1,12 @@
 defmodule Systems.Benchmark.ImportForm do
   use CoreWeb, :live_component
-  use CoreWeb.FileUploader, ~w(.csv)
+  use CoreWeb.FileUploader, accept: ~w(.csv)
 
   @impl true
-  def process_file(socket, {_local_relative_path, local_full_path, remote_file}) do
+  def process_file(socket, {path, _url, original_filename}) do
     socket
-    |> assign(csv_local_path: local_full_path)
-    |> assign(csv_remote_file: remote_file)
+    |> assign(csv_local_path: path)
+    |> assign(csv_remote_file: original_filename)
   end
 
   @impl true

core/systems/content/_public.ex

@@ -11,8 +11,8 @@ defmodule Systems.Content.Public do
   alias Systems.Content.TextItemModel, as: TextItem
   alias Systems.Content.TextBundleModel, as: TextBundle
 
-  def store(file) do
-    Content.Private.get_backend().store(file)
+  def store(path, original_filename) do
+    Content.Private.get_backend().store(path, original_filename)
   end
 
   def get_public_url(id) do

core/systems/content/local_fs.ex

@@ -1,38 +1,28 @@
 defmodule Systems.Content.LocalFS do
-  alias CoreWeb.Endpoint
+  use CoreWeb, :verified_routes
 
-  def store(tmp_path) do
-    uuid = Ecto.UUID.generate()
-    extname = Path.extname(tmp_path)
-    id = "#{uuid}#{extname}"
-    path = get_path(id)
-    File.cp!(tmp_path, path)
-    id
+  def public_path, do: "/uploads"
+
+  def get_public_url(path) do
+    filename = Path.basename(path)
+    ~p"/uploads/#{filename}"
   end
 
-  def storage_path(id) do
-    get_path(id)
+  def store(path, original_filename) do
+    uuid = Ecto.UUID.generate()
+    root_path = get_root_path()
+    new_path = "#{root_path}/#{uuid}_#{original_filename}"
+    File.cp!(path, new_path)
+    new_path
   end
 
-  def get_public_url(id) do
-    "#{Endpoint.url()}/#{public_path()}/#{id}"
+  def get_root_path do
+    Application.get_env(:core, :upload_path)
   end
 
-  def remove(id) do
-    with {:ok, _} <- File.rm_rf(get_path(id)) do
+  def remove(path) do
+    with {:ok, _} <- File.rm_rf(path) do
       :ok
     end
   end
-
-  defp get_path(id) do
-    Path.join(get_root_path(), id)
-  end
-
-  def get_root_path do
-    :core
-    |> Application.get_env(:content, [])
-    |> Access.fetch!(:local_fs_root_path)
-  end
-
-  def public_path, do: "/content"
 end

core/systems/content/s3.ex

@@ -1,7 +1,7 @@
 defmodule Systems.Content.S3 do
   alias ExAws.S3
 
-  def store(file) do
+  def store(file, _original_filename) do
     bucket = Access.fetch!(s3_settings(), :bucket)
     uuid = Ecto.UUID.generate()
     extname = Path.extname(file)

core/systems/document/tool_form.ex

@@ -1,8 +1,6 @@
 defmodule Systems.Document.ToolForm do
   use CoreWeb.LiveForm
-  use CoreWeb.FileUploader, ~w(.pdf)
-
-  alias CoreWeb.Endpoint
+  use CoreWeb.FileUploader, accept: ~w(.pdf)
 
   alias Systems.{
     Document
@@ -11,12 +9,10 @@ defmodule Systems.Document.ToolForm do
   @impl true
   def process_file(
         %{assigns: %{entity: entity}} = socket,
-        {local_relative_path, _local_full_path, remote_file}
+        {_path, url, original_filename}
       ) do
-    ref = "#{Endpoint.url()}#{local_relative_path}"
-
     socket
-    |> save(entity, %{ref: ref, name: remote_file})
+    |> save(entity, %{ref: url, name: original_filename})
   end
 
   @impl true