Merge pull request #627 from eyra/611-data-storage-on-next-Copy

eyra · Feb 20, 2024 · 753eead · 753eead
2 parents f5d4d99 + 653e3db
commit 753eead
Show file tree

Hide file tree

Showing 42 changed files with 558 additions and 279 deletions.
diff --git a/SELFHOSTING.md b/SELFHOSTING.md
@@ -132,6 +132,7 @@ Required environment variables:
 | STATIC_PATH | Path to folder where uploaded files can be stored | "/tmp" |
 | UNSPLASH_ACCESS_KEY | Application access key registered on [Unsplash](https://unsplash.com/) (Image Catalog) | "hcejpnHRuFWL-fKXLYqhGBt1Dz0_tTjeNifgD01VkGE" |
 | UNSPLASH_APP_NAME | Application name registered on [Unsplash](https://unsplash.com/) (Image Catalog) | "Self" |
+| STORAGE_SERVICES | Comma seperated list of storage services | "builtin yoda aws azure" |
 
 Optional environment variables:
 
@@ -144,8 +145,9 @@ Optional environment variables:
 | SURFCONEXT_SITE | SURFconext site | "https://connect.test.surfconext.nl" |
 | SURFCONEXT_CLIENT_ID | SURFconext client ID | "self.com" |
 | SURFCONEXT_CLIENT_SECRET | SURFconext client secret | "12343HieOjb1234hcBpL" |
-| CONTENT_S3_PREFIX | Prefix for S3 content storage | "content" |
-| FELDSPAR_S3_PREFIX | Prefix for S3 feldspar storage | "feldspar" |
+| STORAGE_S3_PREFIX | Prefix for S3 builtin storage objects. Without this variable "builtin" storage service will default to local filesystem | "storage" |
+| CONTENT_S3_PREFIX | Prefix for S3 content objects | "content" |
+| FELDSPAR_S3_PREFIX | Prefix for S3 feldspar objects | "feldspar" |
 | PUBLIC_S3_URL | Public accessable url of an S3 service | "https://self-public.s3.eu-central-1.amazonaws.com" |
 | PUBLIC_S3_BUCKET | Name of the bucket on the S3 service | "self-prod" |
 | DIST_HOSTS | Comma seperated list of hosts in the cluster, see: [OTP Distribution](https://elixirschool.com/en/lessons/advanced/otp_distribution) | "one, two" |
diff --git a/core/config/config.exs b/core/config/config.exs
@@ -131,7 +131,7 @@ config :core, :version, System.get_env("VERSION", "dev")
 
 config :core, :assignment, external_panels: ~w(liss ioresearch generic)
 
-config :core, :storage, services: ~w(azure aws yoda)
+config :core, :storage, services: ~w(builtin yoda)
 
 config :core, BankingClient,
   host: 'localhost',

diff --git a/core/config/dev.exs b/core/config/dev.exs
@@ -51,6 +51,8 @@ config :core,
     "*@eyra.co"
   ]
 
+config :core, Systems.Storage.BuiltIn, special: Systems.Storage.BuiltIn.LocalFS
+
 config :core, :rate,
   prune_interval: 5 * 60 * 1000,
   quotas: [

diff --git a/core/config/runtime.exs b/core/config/runtime.exs
@@ -42,28 +42,17 @@ if config_env() == :prod do
       hackney_opts: [recv_timeout: :timer.minutes(1)]
   end
 
-  # AWS
-
-  if bucket = System.get_env("AWS_S3_BUCKET") do
-    config :core, :s3, bucket: bucket
-  end
-
-  if aws_access_key_id = System.get_env("AWS_ACCESS_KEY_ID") do
-    config :ex_aws, access_key_id: aws_access_key_id
-
-    config :core, Systems.Email.Mailer,
-      adapter: Bamboo.SesAdapter,
-      domain: app_domain,
-      default_from_email: {app_name, app_mail_noreply}
-  end
-
-  if secret_access_key = System.get_env("AWS_SECRET_ACCESS_KEY") do
-    config :ex_aws, secret_access_key: secret_access_key
-  end
-
-  if aws_region = System.get_env("AWS_REGION") do
-    config :ex_aws, region: aws_region
-  end
+  # EX AWS
+  config :ex_aws,
+    access_key_id: System.get_env("AWS_ACCESS_KEY_ID"),
+    secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"),
+    region: System.get_env("AWS_REGION")
+
+  # AWS SES
+  config :core, Systems.Email.Mailer,
+    adapter: Bamboo.SesAdapter,
+    domain: app_domain,
+    default_from_email: {app_name, app_mail_noreply}
 
   # AZURE BLOB
 
@@ -126,6 +115,23 @@ if config_env() == :prod do
       environment_name: System.get_env("RELEASE_ENV") || "prod"
   end
 
+  config :core, :storage,
+    services:
+      System.get_env("STORAGE_SERVICES", "builtin, yoda")
+      |> String.split(",", trim: true)
+      |> Enum.map(&String.trim/1)
+      |> Enum.map(&String.to_atom/1)
+
+  if storage_s3_prefix = System.get_env("STORAGE_S3_PREFIX") do
+    config :core, Systems.Storage.BuiltIn, special: Systems.Storage.BuiltIn.S3
+
+    config :core, Systems.Storage.BuiltIn.S3,
+      bucket: System.get_env("AWS_S3_BUCKET"),
+      prefix: storage_s3_prefix
+  else
+    config :core, Systems.Storage.BuiltIn, special: Systems.Storage.BuiltIn.LocalFS
+  end
+
   if content_s3_prefix = System.get_env("CONTENT_S3_PREFIX") do
     config :core, :content,
       backend: Systems.Content.S3,

diff --git a/core/frameworks/fabric/live_component.ex b/core/frameworks/fabric/live_component.ex
@@ -38,7 +38,7 @@ defmodule Fabric.LiveComponent do
 
       @impl true
       def handle_event(_name, _payload, socket) do
-        Logger.error("handle_event/3 not implemented")
+        Logger.error("[#{__MODULE__}] handle_event/3 not implemented")
         {:noreply, socket}
       end
 

diff --git a/core/frameworks/utililty/ecto_helper.ex b/core/frameworks/utililty/ecto_helper.ex
@@ -1,5 +1,6 @@
 defmodule Frameworks.Utility.EctoHelper do
   import Ecto.Query, only: [from: 2]
+  require Logger
   alias Ecto.{Multi, Changeset}
   alias Core.Repo
   alias Frameworks.Signal
@@ -20,11 +21,16 @@ defmodule Frameworks.Utility.EctoHelper do
     Repo.insert(changeset)
   end
 
-  def update_and_dispatch(changeset, key) do
+  def update_and_dispatch(%Changeset{} = changeset, key) do
     Multi.new()
+    |> update_and_dispatch(changeset, key)
+    |> Repo.transaction()
+  end
+
+  def update_and_dispatch(%Multi{} = multi, %Changeset{} = changeset, key) do
+    multi
     |> Repo.multi_update(key, changeset)
     |> Signal.Public.multi_dispatch({key, :update_and_dispatch}, %{changeset: changeset})
-    |> Repo.transaction()
   end
 
   def delete(multi, name, %table{id: id}) do

diff --git a/core/priv/gettext/en/LC_MESSAGES/eyra-account.po b/core/priv/gettext/en/LC_MESSAGES/eyra-account.po
@@ -58,30 +58,6 @@ msgstr "Change"
 msgid "professionaltitle.label"
 msgstr "Professional title"
 
-#, elixir-autogen, elixir-format
-msgid "push.registration.button"
-msgstr "Activate"
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.title"
-msgstr "Push notifications"
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.activated"
-msgstr "Push notifications are activated on this browser."
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.denied"
-msgstr "Push notifications are deactivated on this browser"
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.label"
-msgstr "Activate to receive notifications from the platform in your browser. After activation, your browser might ask for permission to show notifications."
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.pending"
-msgstr "Determining state.."
-
 #, elixir-autogen, elixir-format
 msgid "feature.study.description"
 msgstr "To earn credits for research participation (OP / RPR) for bedrijfskunde (BK) or international business administration (IBA), select one or more courses below. If you did not finish a course last year, make sure to select that course as well."
@@ -110,10 +86,6 @@ msgstr "Inclusion"
 msgid "features.content.description"
 msgstr "Select the inclusion criteria for your study. If nothing is selected, everyone in the pool is eligible to participate."
 
-#, elixir-autogen, elixir-format
-msgid "push.unavailable.label"
-msgstr "Registering for push notifications is currently available on Chrome and Firefox browsers. Please open the website on one of these browsers to make use of push notifications."
-
 #, elixir-autogen, elixir-format
 msgid "login.google.button"
 msgstr "Sign in with Google"
@@ -157,7 +129,3 @@ msgstr "Sign in"
 #, elixir-autogen, elixir-format, fuzzy
 msgid "await.confirmation.description"
 msgstr "Your account has been created. We send you an activation email. Please click the link in the email to activate your account."
-
-#, elixir-autogen, elixir-format
-msgid "account.created.info.flash"
-msgstr "Account created successfully"
diff --git a/core/priv/gettext/en/LC_MESSAGES/eyra-enums.po b/core/priv/gettext/en/LC_MESSAGES/eyra-enums.po
@@ -377,3 +377,7 @@ msgstr "Benchmark"
 #, elixir-autogen, elixir-format, fuzzy
 msgid "platforms.netflix"
 msgstr "Netflix"
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "storage_service_ids.builtin"
+msgstr "Internal"
diff --git a/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po b/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po
@@ -57,4 +57,24 @@ msgstr "Folder url"
 
 #, elixir-autogen, elixir-format
 msgid "yoda.user.label"
-msgstr "Email"
+msgstr "E-mail"
+
+#, elixir-autogen, elixir-format
+msgid "aws.annotation"
+msgstr "<div>Use Amazon S3 storage. More information: <a href=\"https://aws.amazon.com/s3\">https://aws.amazon.com/s3</a></div>"
+
+#, elixir-autogen, elixir-format
+msgid "azure.annotation"
+msgstr "<div>Use Microsoft Azure blob storage. More information: <a href=\"https://azure.microsoft.com/en-us/products/storage/blobs\">https://azure.microsoft.com/en-us/products/storage/blobs</a></div>"
+
+#, elixir-autogen, elixir-format
+msgid "builtin.annotation"
+msgstr "<div>Use the internal data storage if you do not wish to connect to a third-party data storage service. <br> Once your data donation flow is published, a data tab will appear with instructions on how to download the donated data.</div>"
+
+#, elixir-autogen, elixir-format
+msgid "centerdata.annotation"
+msgstr "<div>More information: <a href=\"https://www.centerdata.nl\">https://www.centerdata.nl</a></div>"
+
+#, elixir-autogen, elixir-format
+msgid "yoda.annotation"
+msgstr "<div>Use Yoda, a research data management service developed at Utrecht University. <br> More information: <a href=\"https://www.uu.nl/en/research/yoda\">https://www.uu.nl/en/research/yoda</a></div>"
diff --git a/core/priv/gettext/en/LC_MESSAGES/eyra-ui.po b/core/priv/gettext/en/LC_MESSAGES/eyra-ui.po
@@ -26,10 +26,6 @@ msgstr "My console"
 msgid "menu.item.profile"
 msgstr "My profile"
 
-#, elixir-autogen, elixir-format
-msgid "menu.item.settings"
-msgstr "Settings"
-
 #, elixir-autogen, elixir-format
 msgid "menu.item.signin"
 msgstr "Sign in"
@@ -58,26 +54,10 @@ msgstr "To-do"
 msgid "tabbar.item.features"
 msgstr "Characteristics"
 
-#, elixir-autogen, elixir-format
-msgid "tabbar.item.profile"
-msgstr "My profile"
-
 #, elixir-autogen, elixir-format
 msgid "tabbar.item.student"
 msgstr "Study program"
 
-#, elixir-autogen, elixir-format
-msgid "tabbar.item.features.forward"
-msgstr "Update characteristics"
-
-#, elixir-autogen, elixir-format
-msgid "tabbar.item.profile.forward"
-msgstr "Update profile"
-
-#, elixir-autogen, elixir-format
-msgid "tabbar.item.student.forward"
-msgstr "Select study program"
-
 #, elixir-autogen, elixir-format
 msgid "menu.item.helpdesk"
 msgstr "Helpdesk"
@@ -194,14 +174,6 @@ msgstr "Collapse"
 msgid "expand.button"
 msgstr "Expand"
 
-#, elixir-autogen, elixir-format, fuzzy
-msgid "tabbar.item.settings"
-msgstr "Settings"
-
-#, elixir-autogen, elixir-format, fuzzy
-msgid "tabbar.item.settings.forward"
-msgstr "Settings"
-
 #, elixir-autogen, elixir-format
 msgid "privacy.link"
 msgstr "Privacy"

diff --git a/core/priv/gettext/eyra-account.pot b/core/priv/gettext/eyra-account.pot
@@ -58,30 +58,6 @@ msgstr ""
 msgid "professionaltitle.label"
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "push.registration.button"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.title"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.activated"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.denied"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.label"
-msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "push.registration.pending"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "feature.study.description"
 msgstr ""
@@ -110,10 +86,6 @@ msgstr ""
 msgid "features.content.description"
 msgstr ""
 
-#, elixir-autogen, elixir-format
-msgid "push.unavailable.label"
-msgstr ""
-
 #, elixir-autogen, elixir-format
 msgid "login.google.button"
 msgstr ""
@@ -157,7 +129,3 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "await.confirmation.description"
 msgstr ""
-
-#, elixir-autogen, elixir-format
-msgid "account.created.info.flash"
-msgstr ""
diff --git a/core/priv/gettext/eyra-enums.pot b/core/priv/gettext/eyra-enums.pot
@@ -377,3 +377,7 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "platforms.netflix"
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "storage_service_ids.builtin"
+msgstr ""
diff --git a/core/priv/gettext/eyra-storage.pot b/core/priv/gettext/eyra-storage.pot
@@ -58,3 +58,23 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "yoda.user.label"
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "aws.annotation"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "azure.annotation"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "builtin.annotation"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "centerdata.annotation"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "yoda.annotation"
+msgstr ""