Mise à jour Elixir 1.10 -> 1.12 et OTP 23 -> 24 (#1683)

* Bump up Elixir & OTP to latest

* Revert to version available as hex.pm image

* Bump up base image + OTP & Elixir

Hex.pm has published a new image, which has allowed me to release (https://github.com/etalab/transport-ops/releases/tag/elixir-1.12.1-erlang-24.0.3-alpine-3.13.3) a compatible image.

* Fix base image reference

* Trigger CI build

* Update credo to latest and fix credo blockers

* Fix postgrex-related Dialyzer warnings

See elixir-ecto/postgrex#549

* Apply mix format

* Remove warning during tests

* Upgrade cowboy & ranch for OTP 24 compatibility (#1691)

See https://ninenines.eu/docs/en/cowboy/2.9/guide/migrating_from_2.8/

This upgrades cowboy, cowlib, and also ranch.

* Upgrade mochiweb to latest (OTP 24 compat)

I had for now to work-around the lack of official hex.pm release (mochi/mochiweb#233).

* Update Vex to master for Elixir 1.11 compat

I also removed vex dependency for apps which did not depend on vex.

* Bump up Elixir to 1.12.2

* Backport coveralls fixes from #1446

* Bump up coveralls

* Fix credo warning & use fetch_env! instead of get for config

* Skip credo issue for now

* Fix regression after refactoring

.circleci/config.yml

@@ -12,7 +12,7 @@ default_docker: &default_docker
 defaults: &defaults
   parameters:
     base_image:
-      default: betagouv/transport:elixir-1.10.4-erlang-23.2.7.2-alpine-3.13.3
+      default: betagouv/transport:elixir-1.12.2-erlang-24.0.3-alpine-3.13.3
       type: string
   working_directory: ~/transport
   docker:

.credo.exs

@@ -107,7 +107,6 @@
         {Credo.Check.Warning.ExpensiveEmptyEnumCheck},
         {Credo.Check.Warning.IExPry},
         {Credo.Check.Warning.IoInspect},
-        {Credo.Check.Warning.LazyLogging},
         {Credo.Check.Warning.OperationOnSameValues},
         {Credo.Check.Warning.OperationWithConstantResult},
         {Credo.Check.Warning.UnusedEnumOperation},

.dialyzer_ignore.exs

@@ -0,0 +1,5 @@
+[
+  # temporary fix for https://github.com/elixir-ecto/postgrex/issues/549
+  ~r/deps\/postgrex\/lib\/postgrex\/type_module.ex/,
+  ~r/lib\/postgrex\/type_module.ex/
+]

.tool-versions

@@ -7,12 +7,12 @@
 # - https://hexdocs.pm/elixir/compatibility-and-deprecations.html
 # - https://github.com/elixir-lang/elixir/releases
 # - `asdf list all elixir`
-elixir 1.10.4-otp-23
+elixir 1.12.2-otp-24
 
 # See:
 # - https://github.com/erlang/otp/releases
 # - https://github.com/erlang/otp/blob/master/otp_versions.table
 # - `asdf list all erlang`
-erlang 23.2.7.2
+erlang 24.0.3
 
 nodejs 14.16.1

Dockerfile

@@ -1,4 +1,4 @@
-FROM betagouv/transport:elixir-1.10.4-erlang-23.2.7.2-alpine-3.13.3
+FROM betagouv/transport:elixir-1.12.2-erlang-24.0.3-alpine-3.13.3
 
 RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ gnu-libiconv-dev
 

README.md

@@ -107,12 +107,12 @@ The following commands will launch the test and generate coverage:
 
 ```
 # Display overall (whole app) coverage for all tests in the console
-RUN_ALL=1 mix coveralls --umbrella
+MIX_ENV=test RUN_ALL=1 mix coveralls --umbrella
 # Same with a HTML report
-RUN_ALL=1 mix coveralls.html --umbrella
+MIX_ENV=test RUN_ALL=1 mix coveralls.html --umbrella
 
 # Display coverage for each umbrella component, rather
-mix coveralls
+MIX_ENV=test mix coveralls
 ```
 
 The coverage is written on screen by default, or in the `cover` subfolders for HTML output.

apps/datagouvfr/mix.exs

@@ -15,7 +15,13 @@ defmodule Datagouvfr.MixProject do
       deps: deps(),
       gettext: [{:write_reference_comments, false}],
       compilers: [:gettext] ++ Mix.compilers(),
-      test_coverage: [tool: ExCoveralls]
+      test_coverage: [tool: ExCoveralls],
+      preferred_cli_env: [
+        coveralls: :test,
+        "coveralls.detail": :test,
+        "coveralls.post": :test,
+        "coveralls.html": :test
+      ]
     ]
   end
 
@@ -35,7 +41,8 @@ defmodule Datagouvfr.MixProject do
       {:httpoison, ">= 0.0.0"},
       {:plug, ">= 0.0.0"},
       {:shared, in_umbrella: true},
-      {:vex, "~> 0.8"},
+      # Using master until https://github.com/CargoSense/vex/issues/68 is fixed
+      {:vex, github: "CargoSense/vex", ref: "328a39f7"},
       {:exvcr, "~> 0.10", only: :test},
       {:mox, "~> 1.0.0", only: :test}
     ]

apps/db/lib/db/resource.ex

@@ -383,6 +383,7 @@ defmodule DB.Resource do
         %{severity: max_severity, count_errors: count_errors}
 
       {:ok, _} ->
+        # credo:disable-for-next-line
         with %Validation{details: details} when details == %{} <- Repo.get_by(Validation, resource_id: id) do
           %{severity: "Irrevelant", count_errors: 0}
         else

apps/db/mix.exs

@@ -15,7 +15,13 @@ defmodule Db.MixProject do
       deps: deps(),
       gettext: [{:write_reference_comments, false}],
       compilers: [:gettext] ++ Mix.compilers(),
-      test_coverage: [tool: ExCoveralls]
+      test_coverage: [tool: ExCoveralls],
+      preferred_cli_env: [
+        coveralls: :test,
+        "coveralls.detail": :test,
+        "coveralls.post": :test,
+        "coveralls.html": :test
+      ]
     ]
   end
 
@@ -44,8 +50,7 @@ defmodule Db.MixProject do
       {:datagouvfr, in_umbrella: true},
       {:shared, in_umbrella: true},
       {:sentry, ">= 0.0.0"},
-      {:typed_ecto_schema, ">= 0.1.1"},
-      {:vex, "~> 0.8"}
+      {:typed_ecto_schema, ">= 0.1.1"}
     ]
   end
 end

apps/gbfs/mix.exs

@@ -14,7 +14,13 @@ defmodule GBFS.MixProject do
       compilers: [:phoenix] ++ Mix.compilers(),
       start_permanent: Mix.env() == :prod,
       deps: deps(),
-      test_coverage: [tool: ExCoveralls]
+      test_coverage: [tool: ExCoveralls],
+      preferred_cli_env: [
+        coveralls: :test,
+        "coveralls.detail": :test,
+        "coveralls.post": :test,
+        "coveralls.html": :test
+      ]
     ]
   end
 

apps/shared/mix.exs

@@ -13,7 +13,13 @@ defmodule Shared.MixProject do
       start_permanent: Mix.env() == :prod,
       deps: deps(),
       elixirc_paths: elixirc_paths(Mix.env()),
-      test_coverage: [tool: ExCoveralls]
+      test_coverage: [tool: ExCoveralls],
+      preferred_cli_env: [
+        coveralls: :test,
+        "coveralls.detail": :test,
+        "coveralls.post": :test,
+        "coveralls.html": :test
+      ]
     ]
   end
 

apps/transport/lib/mailjet/client.ex

@@ -5,9 +5,14 @@ defmodule Mailjet.Client do
   use HTTPoison.Base
   require Logger
 
-  @user Application.get_env(:transport, __MODULE__)[:mailjet_user]
-  @key Application.get_env(:transport, __MODULE__)[:mailjet_key]
-  @url Application.get_env(:transport, __MODULE__)[:mailjet_url]
+  def get_config!(key) do
+    config = Application.fetch_env!(:transport, __MODULE__)
+    Keyword.fetch!(config, key)
+  end
+
+  def mailjet_user, do: get_config!(:mailjet_user)
+  def mailjet_key, do: get_config!(:mailjet_key)
+  def mailjet_url, do: get_config!(:mailjet_url)
 
   @spec payload!(binary(), binary(), binary(), binary(), binary()) :: any()
   def payload!(from_name, from_email, reply_to, topic, text_body, html_body \\ nil) do
@@ -32,7 +37,7 @@ defmodule Mailjet.Client do
   end
 
   def send_mail(from_name, from_email, reply_to, topic, text_body, html_body, false) do
-    @url
+    mailjet_url()
     |> post(payload!(from_name, from_email, reply_to, topic, text_body, html_body))
     |> case do
       {:ok, %{status_code: 200, body: body}} -> {:ok, body}
@@ -42,7 +47,7 @@ defmodule Mailjet.Client do
   end
 
   def request(method, url, body \\ "", headers \\ [], options \\ []) do
-    options = options ++ [hackney: [basic_auth: {@user, @key}]]
+    options = options ++ [hackney: [basic_auth: {mailjet_user(), mailjet_key()}]]
     super(method, url, body, headers, options)
   end
 end

apps/transport/lib/transport/community_resource_cleaner.ex

@@ -10,10 +10,9 @@ defmodule Transport.CommunityResourcesCleaner do
   import Ecto.Query
   require Logger
 
-  @transport_publisher_label Application.get_env(
-                               :transport,
-                               :datagouvfr_transport_publisher_label
-                             )
+  def transport_publisher_label do
+    Application.fetch_env!(:transport, :datagouvfr_transport_publisher_label)
+  end
 
   def clean_community_resources do
     result =
@@ -60,7 +59,7 @@ defmodule Transport.CommunityResourcesCleaner do
     dataset.resources
     |> Enum.filter(fn r ->
       r.is_community_resource == true and
-        r.community_resource_publisher == @transport_publisher_label
+        r.community_resource_publisher == transport_publisher_label()
     end)
     |> Enum.reject(fn r -> resources_url |> Enum.member?(r.original_resource_url) end)
     |> Enum.map(fn r ->

apps/transport/lib/transport/data_checker.ex

@@ -70,11 +70,9 @@ defmodule Transport.DataChecker do
       "Jeu de données arrivant à expiration",
       """
       Bonjour,
-      Ce jeu de données arrive à expiration dans #{delay} jour#{
-        if delay != 1 do
-          "s"
-        end
-      }.
+      Ce jeu de données arrive à expiration dans #{delay} jour#{if delay != 1 do
+        "s"
+      end}.
       Afin qu’il puisse continuer à être utilisé par les différents acteurs, il faut qu’il soit mis à jour prochainement.
       L’équipe transport.data.gouv.fr
       """,

apps/transport/lib/transport/import_data.ex

@@ -11,7 +11,9 @@ defmodule Transport.ImportData do
   require Logger
   import Ecto.Query
 
-  @max_import_concurrent_jobs Application.get_env(:transport, :max_import_concurrent_jobs)
+  def max_import_concurrent_jobs do
+    Application.fetch_env!(:transport, :max_import_concurrent_jobs)
+  end
 
   @spec import_all_datasets :: :ok
   def import_all_datasets do
@@ -25,7 +27,7 @@ defmodule Transport.ImportData do
     results =
       ImportTaskSupervisor
       |> Task.Supervisor.async_stream_nolink(datasets, &import_dataset_logged/1,
-        max_concurrency: @max_import_concurrent_jobs,
+        max_concurrency: max_import_concurrent_jobs(),
         timeout: 180_000,
         on_timeout: :kill_task
       )
@@ -54,7 +56,7 @@ defmodule Transport.ImportData do
       |> Task.Supervisor.async_stream_nolink(
         resources_id,
         fn r_id -> Resource.validate_and_save(r_id, force) end,
-        max_concurrency: @max_import_concurrent_jobs,
+        max_concurrency: max_import_concurrent_jobs(),
         timeout: 180_000
       )
       |> Enum.to_list()

apps/transport/lib/transport_web/api/controllers/places_controller.ex

@@ -3,7 +3,7 @@ defmodule TransportWeb.API.PlacesController do
   alias DB.{Place, Repo}
   alias Helpers
   alias OpenApiSpex.Operation
-  alias TransportWeb.API.Schemas.{AutocompleteResponse}
+  alias TransportWeb.API.Schemas.AutocompleteResponse
   import Ecto.{Query}
 
   @spec open_api_operation(any) :: Operation.t()

apps/transport/lib/transport_web/controllers/resource_controller.ex

@@ -90,9 +90,7 @@ defmodule TransportWeb.ResourceController do
     else
       {:error, error} ->
         Logger.error(
-          "Unable to update resource #{params["resource_id"]} of dataset #{params["dataset_id"]}, error: #{
-            inspect(error)
-          }"
+          "Unable to update resource #{params["resource_id"]} of dataset #{params["dataset_id"]}, error: #{inspect(error)}"
         )
 
         conn

apps/transport/mix.exs

@@ -17,7 +17,11 @@ defmodule Transport.Mixfile do
         vcr: :test,
         "vcr.delete": :test,
         "vcr.check": :test,
-        "vcr.show": :test
+        "vcr.show": :test,
+        coveralls: :test,
+        "coveralls.detail": :test,
+        "coveralls.post": :test,
+        "coveralls.html": :test
       ],
       build_embedded: Mix.env() == :prod,
       start_permanent: Mix.env() == :prod,
@@ -56,7 +60,6 @@ defmodule Transport.Mixfile do
       {:floki, ">= 0.0.0", only: :test},
       {:plug_cowboy, "~> 2.3"},
       {:recon, "~> 2.4"},
-      {:vex, "~> 0.8"},
       {:quantum, "~> 2.3"},
       {:timex, "~> 3.5"},
       {:sentry, "~> 8.0"},
@@ -75,7 +78,7 @@ defmodule Transport.Mixfile do
       {:phoenix_live_reload, "~> 1.2", only: :dev},
       {:cors_plug, "~> 2.0"},
       {:exvcr, "~> 0.10", only: :test},
-      {:credo, "~> 1.0.0", only: [:dev, :test], runtime: false},
+      {:credo, "~> 1.5.0", only: [:dev, :test], runtime: false},
       # NOTE: we cannot upgrade to the very latest (2.2.x) because of
       # https://github.com/parroty/exvcr/issues/153#issuecomment-874864317
       {:ex_aws, "~> 2.1"},

apps/transport/test/transport/community_resource_cleaner_test.exs

@@ -4,11 +4,6 @@ defmodule Transport.CommunityResourcesCleanerTest do
   import Transport.CommunityResourcesCleaner
   alias DB.Repo
 
-  @transport_publisher_label Application.get_env(
-                               :transport,
-                               :datagouvfr_transport_publisher_label
-                             )
-
   setup do
     Mox.stub_with(
       Datagouvfr.Client.CommunityResources.Mock,
@@ -52,7 +47,7 @@ defmodule Transport.CommunityResourcesCleanerTest do
     community_resource =
       insert(:resource, %{
         is_community_resource: true,
-        community_resource_publisher: @transport_publisher_label,
+        community_resource_publisher: transport_publisher_label(),
         original_resource_url: "original_url",
         datagouv_id: resource_datagouv_id
       })
@@ -84,14 +79,14 @@ defmodule Transport.CommunityResourcesCleanerTest do
     resource1 =
       insert(:resource, %{
         is_community_resource: true,
-        community_resource_publisher: @transport_publisher_label,
+        community_resource_publisher: transport_publisher_label(),
         original_resource_url: "original_url1"
       })
 
     resource2 =
       insert(:resource, %{
         is_community_resource: true,
-        community_resource_publisher: @transport_publisher_label,
+        community_resource_publisher: transport_publisher_label(),
         original_resource_url: "original_url2"
       })
 
@@ -117,7 +112,7 @@ defmodule Transport.CommunityResourcesCleanerTest do
     resource11 =
       insert(:resource, %{
         is_community_resource: true,
-        community_resource_publisher: @transport_publisher_label,
+        community_resource_publisher: transport_publisher_label(),
         original_resource_url: "original_url11"
       })
 
@@ -127,7 +122,7 @@ defmodule Transport.CommunityResourcesCleanerTest do
     resource20 =
       insert(:resource, %{
         is_community_resource: true,
-        community_resource_publisher: @transport_publisher_label,
+        community_resource_publisher: transport_publisher_label(),
         original_resource_url: "original_url20"
       })