migrate to sidekiq

Gemfile

@@ -24,10 +24,10 @@ gem "bootsnap", "~> 1.4"
 gem "puma", ">= 4.3.5"
 gem "uglifier", "~> 4.1"
 
-gem "delayed_job_web"
 gem "faker", "~> 2.14"
 gem "sentry-raven"
-gem "whenever", require: false
+gem "sidekiq", "~> 6.0"
+gem "sidekiq-cron"
 
 group :development, :test do
   gem "byebug", "~> 11.0", platform: :mri
@@ -50,8 +50,5 @@ group :development do
 end
 
 group :production do
-  gem "daemons", "~> 1.3"
-  gem "delayed_job_active_record", "~> 4.1"
   gem "figaro", "~> 1.2"
-  gem "passenger", "~> 6.0"
 end

Gemfile.lock

@@ -382,7 +382,6 @@ GEM
     chef-utils (17.6.18)
       concurrent-ruby
     childprocess (3.0.0)
-    chronic (0.10.2)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     coffee-rails (5.0.0)
@@ -393,13 +392,13 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
     cookiejar (0.3.3)
     crack (0.4.5)
       rexml
     crass (1.0.6)
     css_parser (1.10.0)
       addressable
-    daemons (1.4.1)
     date_validator (0.9.0)
       activemodel
       activesupport
@@ -413,16 +412,6 @@ GEM
     declarative-builder (0.1.0)
       declarative-option (< 0.2.0)
     declarative-option (0.1.0)
-    delayed_job (4.1.9)
-      activesupport (>= 3.0, < 6.2)
-    delayed_job_active_record (4.1.6)
-      activerecord (>= 3.0, < 6.2)
-      delayed_job (>= 3.0, < 5)
-    delayed_job_web (1.4.4)
-      activerecord (> 3.0.0)
-      delayed_job (> 2.0.3)
-      rack-protection (>= 1.5.5)
-      sinatra (>= 1.4.4)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     devise (4.8.0)
@@ -469,6 +458,8 @@ GEM
     erbse (0.1.4)
       temple
     erubi (1.10.0)
+    et-orbi (1.2.6)
+      tzinfo
     etherpad-lite (0.3.0)
       rest-client (>= 1.6)
     eventmachine (1.2.7)
@@ -525,6 +516,9 @@ GEM
       activesupport (>= 4.1, < 6.0)
       railties (>= 4.1, < 6.0)
       tzinfo (~> 1.2, >= 1.2.2)
+    fugit (1.5.2)
+      et-orbi (~> 1.1, >= 1.1.8)
+      raabro (~> 1.4)
     geocoder (1.7.0)
     globalid (0.5.2)
       activesupport (>= 5.0)
@@ -626,8 +620,6 @@ GEM
     multi_xml (0.6.0)
     multipart-post (2.1.1)
     mustache (1.1.1)
-    mustermann (1.1.1)
-      ruby2_keywords (~> 0.0.1)
     net-scp (3.0.0)
       net-ssh (>= 2.6.5, < 7.0.0)
     net-ssh (6.1.0)
@@ -673,9 +665,6 @@ GEM
     parallel (1.21.0)
     parser (3.0.2.0)
       ast (~> 2.4.1)
-    passenger (6.0.11)
-      rack
-      rake (>= 0.8.1)
     pg (1.1.4)
     pg_search (2.3.5)
       activerecord (>= 5.2)
@@ -698,14 +687,13 @@ GEM
       multi_json
     puma (5.5.0)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.5.2)
     rack (2.2.3)
     rack-attack (6.5.0)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
       rack (>= 2.0.0)
-    rack-protection (2.1.0)
-      rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.6)
@@ -844,17 +832,19 @@ GEM
     sentry-raven (3.1.2)
       faraday (>= 1.0)
     seven_zip_ruby (1.3.0)
+    sidekiq (6.3.1)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    sidekiq-cron (1.2.0)
+      fugit (~> 1.1)
+      sidekiq (>= 4.2.1)
     simplecov (0.19.1)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
     simplecov-cobertura (1.3.1)
       simplecov (~> 0.8)
     simplecov-html (0.12.3)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.1.0)
-      tilt (~> 2.0)
     smart_properties (1.16.3)
     social-share-button (1.2.4)
       coffee-rails
@@ -923,8 +913,6 @@ GEM
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    whenever (1.0.0)
-      chronic (>= 0.6.3)
     wicked_pdf (1.4.0)
       activesupport
     wisper (2.0.1)
@@ -944,7 +932,6 @@ DEPENDENCIES
   capistrano-passenger (~> 0.2.0)
   capistrano-rails (~> 1.5)
   capistrano-rbenv (~> 2.1)
-  daemons (~> 1.3)
   decidim!
   decidim-conferences!
   decidim-consultations!
@@ -954,20 +941,18 @@ DEPENDENCIES
   decidim-notify!
   decidim-term_customizer!
   decidim-time_tracker!
-  delayed_job_active_record (~> 4.1)
-  delayed_job_web
   faker (~> 2.14)
   figaro (~> 1.2)
   letter_opener_web (~> 1.3)
   listen (~> 3.1)
-  passenger (~> 6.0)
   puma (>= 4.3.5)
   sentry-raven
+  sidekiq (~> 6.0)
+  sidekiq-cron
   spring (~> 2.0)
   spring-watcher-listen (~> 2.0)
   uglifier (~> 4.1)
   web-console (~> 3.5)
-  whenever
 
 RUBY VERSION
    ruby 2.7.4p191

app/workers/data_portability_worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "rake"
+
+Rails.application.load_tasks
+
+class DataPortabilityWorker
+  include Sidekiq::Worker
+
+  def perform(*_args)
+    Rake::Task["decidim:delete_data_portability_files"].invoke
+    Rake::Task["decidim:open_data:export"].invoke
+  end
+end

app/workers/metrics_generation_worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "rake"
+
+Rails.application.load_tasks
+
+class MetricsGenerationWorker
+  include Sidekiq::Worker
+
+  def perform(*_args)
+    Rake::Task["decidim:metrics:all"].invoke
+  end
+end

config/environments/production.rb

@@ -103,4 +103,6 @@
 
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
+
+  config.active_job.queue_adapter = :sidekiq
 end

config/routes.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
+require "sidekiq/web"
+require "sidekiq/cron/web"
+
 Rails.application.routes.draw do
-  authenticated :user, ->(user) { user.admin? } do
-    mount DelayedJobWeb, at: "/delayed_job"
+  authenticate :user, ->(u) { u.admin? } do
+    mount Sidekiq::Web => "/sidekiq"
   end
 
   mount LetterOpenerWeb::Engine, at: "/letter_opener" if Rails.env.development?

config/schedule.yml

@@ -0,0 +1,11 @@
+#config/schedule.yml
+
+metrics_job:
+  cron: "30 1 * * *"
+  class: "MetricsGenerationWorker"
+  queue: metrics
+
+data_job:
+  cron: "3 3 * * *"
+  class: "DataPortabilityWorker"
+  queue: default

config/sidekiq.yml

@@ -0,0 +1,8 @@
+:queues:
+  - ["mailers", 2]
+  - ["newsletter", 1]
+  - ["newsletters_opt_in", 1]
+  - ["decidim_events", 1]
+  - ["events", 1]
+  - ["metrics", 1]
+  - ["default", 1]

db/migrate/20211212135717_remove_delayed_job.rb

@@ -0,0 +1,5 @@
+class RemoveDelayedJob < ActiveRecord::Migration[5.2]
+  def change
+    drop_table :delayed_jobs
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_10_05_102748) do
+ActiveRecord::Schema.define(version: 2021_12_12_135717) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "ltree"
@@ -1863,21 +1863,6 @@
     t.index ["decidim_organization_id"], name: "index_verifications_csv_census_to_organization"
   end
 
-  create_table "delayed_jobs", force: :cascade do |t|
-    t.integer "priority", default: 0, null: false
-    t.integer "attempts", default: 0, null: false
-    t.text "handler", null: false
-    t.text "last_error"
-    t.datetime "run_at"
-    t.datetime "locked_at"
-    t.datetime "failed_at"
-    t.string "locked_by"
-    t.string "queue"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.index ["priority", "run_at"], name: "delayed_jobs_priority"
-  end
-
   create_table "oauth_access_grants", force: :cascade do |t|
     t.integer "resource_owner_id", null: false
     t.bigint "application_id", null: false