Merge branch 'master' into develop

mysociety · Jan 4, 2023 · 181968a · 181968a
2 parents 8c9c116 + 1608c0e
commit 181968a
Show file tree

Hide file tree

Showing 148 changed files with 5,117 additions and 3,622 deletions.
diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb
@@ -49,41 +49,9 @@ class FoiAttachment < ApplicationRecord
   BODY_MAX_TRIES = 3
   BODY_MAX_DELAY = 5
 
-  def directory
-    if file.attached?
-      warn <<~DEPRECATION.squish
-        [DEPRECATION] FoiAttachment#directory shouldn't be used when using
-        `ActiveStorage` backed file stores. This method will be removed
-        in 0.42.
-      DEPRECATION
-      return
-    end
-
-    base_dir = File.expand_path(File.join(File.dirname(__FILE__), "../../cache", "attachments_#{Rails.env}"))
-    return File.join(base_dir, self.hexdigest[0..2])
-  end
-
-  def filepath
-    if file.attached?
-      warn <<~DEPRECATION.squish
-        [DEPRECATION] FoiAttachment#filepath shouldn't be used when using
-        `ActiveStorage` backed file stores. This method will be removed
-        in 0.42.
-      DEPRECATION
-      return
-    end
-
-    File.join(self.directory, self.hexdigest)
-  end
-
   def delete_cached_file!
     @cached_body = nil
-
-    if file.attached?
-      file.purge
-    elsif File.exist?(filepath)
-      File.delete(filepath)
-    end
+    file.purge if file.attached?
   end
 
   def body=(d)
@@ -101,33 +69,21 @@ def body=(d)
   end
 
   # raw body, encoded as binary
-  def body
-    if @cached_body.nil?
-      tries = 0
-      delay = 1
-      begin
-        if file.attached?
-          @cached_body = file.download
-        else
-          @cached_body = File.open(filepath, "rb" ) { |file| file.read }
-        end
-      rescue Errno::ENOENT, ActiveStorage::FileNotFoundError
-        # we've lost our cached attachments for some reason.  Reparse them.
-        if tries > BODY_MAX_TRIES
-          raise
-        else
-          sleep delay
-        end
-        tries += 1
-        delay *= 2
-        delay = BODY_MAX_DELAY if delay > BODY_MAX_DELAY
-        force = true
-        self.incoming_message.parse_raw_email!(force)
-        reload
-        retry
-      end
+  def body(tries: 0, delay: 1)
+    return @cached_body if @cached_body
+
+    if file.attached?
+      @cached_body = file.download
+    else
+      # we've lost our cached attachments for some reason.  Reparse them.
+      raise if tries > BODY_MAX_TRIES
+      sleep [delay, BODY_MAX_DELAY].min
+
+      self.incoming_message.parse_raw_email!(true)
+      reload
+
+      body(tries: tries + 1, delay: delay * 2)
     end
-    return @cached_body
   end
 
   # body as UTF-8 text, with scrubbing of invalid chars if needed

diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb
@@ -69,45 +69,6 @@ def empty_from_field?
     mail.from_addrs.nil? || mail.from_addrs.size == 0
   end
 
-  def directory
-    if file.attached?
-      warn <<~DEPRECATION.squish
-        [DEPRECATION] RawEmail#directory shouldn't be used when using
-        `ActiveStorage` backed file stores. This method will be removed
-        in 0.42.
-      DEPRECATION
-      return
-    end
-
-    if request_id.empty?
-      raise "Failed to find the id number of the associated request: has it been saved?"
-    end
-
-    if Rails.env.test?
-      File.join(Rails.root, 'files/raw_email_test')
-    else
-      File.join(AlaveteliConfiguration::raw_emails_location,
-                request_id[0..2], request_id)
-    end
-  end
-
-  def filepath
-    if file.attached?
-      warn <<~DEPRECATION.squish
-        [DEPRECATION] RawEmail#filepath shouldn't be used when using
-        `ActiveStorage` backed file stores. This method will be removed
-        in 0.42.
-      DEPRECATION
-      return
-    end
-
-    if incoming_message_id.empty?
-      raise "Failed to find the id number of the associated incoming message: has it been saved?"
-    end
-
-    File.join(directory, incoming_message_id)
-  end
-
   def mail
     @mail ||= mail!
   end
@@ -126,9 +87,7 @@ def data=(d)
   end
 
   def data
-    return @data ||= file.download if file.attached?
-
-    File.open(filepath, "rb").read
+    @data ||= file.download if file.attached?
   end
 
   def data_as_text
@@ -172,10 +131,6 @@ def incoming_message_id
   end
 
   def destroy_file_representation!
-    if file.attached?
-      file.purge
-    elsif File.exist?(filepath)
-      File.delete(filepath)
-    end
+    file.purge if file.attached?
   end
 end
diff --git a/config/initializers/alaveteli.rb b/config/initializers/alaveteli.rb
@@ -10,7 +10,7 @@
 load "util.rb"
 
 # Application version
-ALAVETELI_VERSION = '0.41.1.1'
+ALAVETELI_VERSION = '0.42.0.0'
 
 # Add new inflection rules using the following format
 # (all these examples are active by default):

diff --git a/doc/CHANGES.md b/doc/CHANGES.md
@@ -1,12 +1,20 @@
-# develop
+# 0.42.0.0
 
 ## Highlighted Features
 
 * Use CSV output and ignore case for `users:count_per_domain` stats (Gareth
   Rees)
+* Improve storage of event parameter data (Graeme Porteous)
+* Allow admins to destroy post redirects (Gareth Rees)
+* Add initial administration UI for batch requests (Gareth Rees)
+* Add ability to hide individual attachments using prominence (Graeme Porteous)
+* Refactor requests, outgoing messages, incoming messages and attachments
+  prominence to be consistent and reusable (Graeme Porteous)
 * Fix categorisation game total requests count (Gareth Rees)
 * Add count of requests in each prominence state to body and user admin pages
   (Gareth Rees)
+* Update `all-authorities.csv` endpoint to download and cache file nightly
+  (Graeme Porteous)
 * Improved notes feature, allowing multiple notes to be associated with bodies
   and requests. Association can either be direct or via a tag. Tagged notes are
   useful for displaying one note to a subset of bodies or requests which are all
@@ -20,14 +28,20 @@
 * Improve linking from outgoing & incoming message admin pages (Gareth Rees)
 * Allow admins to destroy user post redirects (Gareth Rees)
 * Use correct mime type for cached CSV attachments (Gareth Rees)
+* Improvements to the admin object columns listings (Gareth Rees, Graeme
+  Porteous)
 * Protect mass-tag update buttons in admin bodies lists (Gareth Rees)
-* Update `all-authorities.csv` download to cache file nightly (Graeme Porteous)
-* Improve storage of event parameter data (Graeme Porteous)
+* Dependencies upgrades (Alexander Griffen, Graeme Porteous)
 
 ## Highlighted Pro Features
 
+* Allow former pro to classify embargo requests (Graeme Porteous)
+
 ## Upgrade Notes
 
+* This will the last release to support Vagrant development boxes. Please switch
+  to using Docker. See: https://alaveteli.org/docs/installing/docker/
+
 * _Required:_ There are some database structure updates so remember to run:
 
       bin/rails db:migrate
@@ -53,6 +67,15 @@
 
       bin/rails temp:sanitise_and_populate_events_params_json
 
+  After running these tasks all of your `InfoRequestEvent` objects should have a
+  populated `param` column. This can be checked by running:
+
+      bin/rails runner "p InfoRequestEvent.where(params: nil).count"
+
+  If this does not return `0` or the tasks error then
+  [please message us](https://groups.google.com/forum/#!forum/alaveteli-dev) as
+  the old `params_yaml` will be removed in the next release.
+
 * _Required:_ The crontab needs to be regenerated to include the new
   modifications:
   https://alaveteli.org/docs/installing/cron_and_daemons/#generate-crontab
@@ -65,6 +88,63 @@
 
 ### Changed Templates
 
+    app/views/admin/outgoing_messages/snippets/index.html.erb
+    app/views/admin/users/sign_ins/index.html.erb
+    app/views/admin_announcements/index.html.erb
+    app/views/admin_comment/edit.html.erb
+    app/views/admin_comment/index.html.erb
+    app/views/admin_general/_admin_navbar.html.erb
+    app/views/admin_general/_change_request_summary.html.erb
+    app/views/admin_general/timeline.html.erb
+    app/views/admin_incoming_message/_actions.html.erb
+    app/views/admin_incoming_message/_foi_attachments.html.erb
+    app/views/admin_incoming_message/_intro.html.erb
+    app/views/admin_incoming_message/edit.html.erb
+    app/views/admin_outgoing_message/edit.html.erb
+    app/views/admin_public_body/_form.html.erb
+    app/views/admin_public_body/_locale_fields.html.erb
+    app/views/admin_public_body/_one_list.html.erb
+    app/views/admin_public_body/edit.html.erb
+    app/views/admin_public_body/index.html.erb
+    app/views/admin_public_body/show.html.erb
+    app/views/admin_public_body_categories/_form.html.erb
+    app/views/admin_public_body_categories/edit.html.erb
+    app/views/admin_raw_email/show.html.erb
+    app/views/admin_request/_some_annotations.html.erb
+    app/views/admin_request/_some_requests.html.erb
+    app/views/admin_request/edit.html.erb
+    app/views/admin_request/hidden_user_explanation.text.erb
+    app/views/admin_request/index.html.erb
+    app/views/admin_request/show.html.erb
+    app/views/admin_track/index.html.erb
+    app/views/admin_user/_user_table.html.erb
+    app/views/admin_user/show.html.erb
+    app/views/alaveteli_pro/batch_request_authority_searches/_search_result.html.erb
+    app/views/alaveteli_pro/draft_info_request_batches/_summary.html.erb
+    app/views/alaveteli_pro/info_requests/_embargo_note.html.erb
+    app/views/alaveteli_pro/info_requests/_select_authority_form.html.erb
+    app/views/notification_mailer/expire_embargo_notification.text.erb
+    app/views/outgoing_messages/delivery_statuses/show.html.erb
+    app/views/public_body/_body_listing_single.html.erb
+    app/views/public_body/show.html.erb
+    app/views/request/_attention_requested.html.erb
+    app/views/request/_bubble.html.erb
+    app/views/request/_hidden_correspondence.html.erb
+    app/views/request/_hidden_correspondence.text.erb
+    app/views/request/_hidden_request_messages.html.erb
+    app/views/request/_incoming_correspondence.html.erb
+    app/views/request/_incoming_correspondence.text.erb
+    app/views/request/_outgoing_correspondence.html.erb
+    app/views/request/_outgoing_correspondence.text.erb
+    app/views/request/_restricted_correspondence.html.erb
+    app/views/request/_share_by_link.html.erb
+    app/views/request/hidden.html.erb
+    app/views/request/hidden_correspondence.html.erb
+    app/views/request/new.html.erb
+    app/views/request/new_defunct.html.erb
+    app/views/request/show.html.erb
+    app/views/request/show.text.erb
+
 # 0.41.1.1
 
 ## Highlighted Features

diff --git a/lib/configuration.rb b/lib/configuration.rb
@@ -134,16 +134,6 @@ module AlaveteliConfiguration
     # rubocop:enable Layout/LineLength
   end
 
-  def self.raw_emails_location
-    warn <<~DEPRECATION.squish
-      [DEPRECATION] AlaveteliConfiguration.raw_emails_location will be removed
-      in 0.42. You should have `ActiveStorage` configured but you still haven't
-      migrated all your old files. Please run `bin/rails storage:migrate` to
-      migrate
-    DEPRECATION
-    super
-  end
-
   def self.method_missing(name)
     key = name.to_s.upcase
     if DEFAULTS.has_key?(key.to_sym)