Move commonlib function into lib

There seems to be an issue with `MySociety::Format.simplify_url_part`
but we're not getting a full stack trace due to the source being outside
of app/lib.

This change should hopefully give us more information to allow us to
debug this issue.

See:
  #5870
  #5871
  #5873
  #5874
  #5876
  #5877
  #5878
  #5879

app/controllers/public_body_controller.rb

@@ -21,8 +21,8 @@ def show
       raise ActiveRecord::RecordNotFound.new("Sorry. No pages after #{MAX_RESULTS / requests_per_page}.")
     end
 
-    if MySociety::Format.simplify_url_part(params[:url_name], 'body') != params[:url_name]
-      redirect_to :url_name => MySociety::Format.simplify_url_part(params[:url_name], 'body'), :status => :moved_permanently
+    if Alaveteli::Format.simplify_url_part(params[:url_name], 'body') != params[:url_name]
+      redirect_to :url_name => Alaveteli::Format.simplify_url_part(params[:url_name], 'body'), :status => :moved_permanently
       return
     end
 

app/controllers/user_controller.rb

@@ -380,8 +380,8 @@ def set_in_pro_area
   end
 
   def normalize_url_name
-    unless MySociety::Format.simplify_url_part(params[:url_name], 'user') == params[:url_name]
-      redirect_to :url_name => MySociety::Format.simplify_url_part(params[:url_name], 'user'), :status => :moved_permanently
+    unless Alaveteli::Format.simplify_url_part(params[:url_name], 'user') == params[:url_name]
+      redirect_to :url_name => Alaveteli::Format.simplify_url_part(params[:url_name], 'user'), :status => :moved_permanently
     end
   end
 

app/helpers/download_helper.rb

@@ -3,7 +3,7 @@
 #
 module DownloadHelper
   def generate_download_filename(resource:, id:, title:, type: nil, ext:)
-    url_title = MySociety::Format.simplify_url_part(
+    url_title = Alaveteli::Format.simplify_url_part(
       title, resource, 32
     )
     url_title += "-#{type}" if type

app/models/concerns/public_body_derived_fields.rb

@@ -41,7 +41,7 @@ def set_first_letter
 
   def update_url_name
     if changed.include?('name') || changed.include?('short_name')
-      self.url_name = MySociety::Format.
+      self.url_name = Alaveteli::Format.
                         simplify_url_part(short_or_long_name, 'body')
     end
   end

app/models/info_request.rb

@@ -766,7 +766,7 @@ def user_name_slug
       if external_user_name.nil?
         fake_slug = "anonymous"
       else
-        fake_slug = MySociety::Format.simplify_url_part(external_user_name, 'external_user', 32)
+        fake_slug = Alaveteli::Format.simplify_url_part(external_user_name, 'external_user', 32)
       end
       (public_body.url_name || "") + "_" + fake_slug
     else

app/models/info_request/sluggable.rb

@@ -35,7 +35,7 @@ def url_title(opts = {})
   def update_url_title
     return unless title
 
-    url_title = MySociety::Format.simplify_url_part(title, 'request', 32)
+    url_title = Alaveteli::Format.simplify_url_part(title, 'request', 32)
     suffix = suffix_number(url_title)
     unique_url_title = suffix ? "#{url_title}_#{suffix}" : url_title
 

app/models/user.rb

@@ -380,7 +380,7 @@ def name=(name)
   end
 
   def update_url_name
-    url_name = MySociety::Format.simplify_url_part(read_attribute(:name), 'user', 32)
+    url_name = Alaveteli::Format.simplify_url_part(read_attribute(:name), 'user', 32)
     # For user with same name as others, add on arbitary numeric identifier
     unique_url_name = url_name
     suffix_num = 2 # as there's already one without numeric suffix

config/initializers/alaveteli.rb

@@ -66,6 +66,7 @@
 require 'safe_redirect'
 require 'alaveteli_pro/metrics_report'
 require 'alaveteli_pro/webhook_endpoints'
+require 'alaveteli_format'
 
 AlaveteliLocalization.set_locales(AlaveteliConfiguration::available_locales,
                                   AlaveteliConfiguration::default_locale)

lib/alaveteli_format.rb

@@ -0,0 +1,35 @@
+require 'cgi'
+require 'unicode'
+require 'unidecoder'
+
+module Alaveteli
+  module Format
+    # Simplified a name to something usable in a URL
+    def self.simplify_url_part(text, default_name, max_len = nil)
+      text = text.downcase # this also clones the string, if we use downcase! we modify the original
+      text = Unicode.normalize_KD(text)
+      text = text.to_ascii.downcase
+
+      text.gsub!(/(\s|-|_)/, "_")
+      text.gsub!(/[^a-z0-9_]/, "")
+      text.gsub!(/_+/, "_")
+      text.gsub!(/^_*/, "")
+      text.gsub!(/_*$/, "")
+
+      # If required, trim down to size
+      if not max_len.nil?
+        if text.size > max_len
+          text = text[0..(max_len-1)]
+        end
+        # removing trailing _
+        text.gsub!(/_*$/, "")
+      end
+      # Don't allow short (zero length!), or all numeric (clashes with identifiers)
+      if text.size < 1 || text.match(/^[0-9]+$/)
+        text = default_name # just do "user_1", "user_2" etc.
+      end
+
+      text
+    end
+  end
+end

lib/tasks/import.rake

@@ -31,7 +31,7 @@ namespace :import do
     reader.each do |row_array|
       row = Hash[headers.zip row_array]
       name = row['name']
-      url_part = MySociety::Format.simplify_url_part(name, 'body')
+      url_part = Alaveteli::Format.simplify_url_part(name, 'body')
       name_count[name] += 1
       url_part_count[url_part] += 1
       number_of_rows += 1