Add support for Azure blob storage (#1297)

This adds support for Azure blob storage using Paperclip, similar to
what's supported for S3 and other backends. It makes use of the
`paperclip-azure` plugin gem. As with S3, this is not loaded by default;
it's only used if the proper environment variables are set.

I'm using my fork of paperclip-azure because it incorporates a few new
features to make it feature-equivalent to S3 for Mastodon's purposes.

Gemfile

@@ -22,6 +22,7 @@ gem 'aws-sdk-s3', '~> 1.117', require: false
 gem 'fog-core', '<= 2.4.0'
 gem 'fog-openstack', '~> 0.3', require: false
 gem 'kt-paperclip', '~> 7.1'
+gem 'md-paperclip-azure', '~> 2.2', require: false
 gem 'blurhash', '~> 0.1'
 
 gem 'active_model_serializers', '~> 0.10'

Gemfile.lock

@@ -105,6 +105,14 @@ GEM
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.5.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    azure-storage-blob (2.0.3)
+      azure-storage-common (~> 2.0)
+      nokogiri (~> 1, >= 1.10.8)
+    azure-storage-common (2.0.4)
+      faraday (~> 1.0)
+      faraday_middleware (~> 1.0, >= 1.0.0.rc1)
+      net-http-persistent (~> 4.0)
+      nokogiri (~> 1, >= 1.10.8)
     bcrypt (3.1.17)
     better_errors (2.9.1)
       coderay (>= 1.0.0)
@@ -254,6 +262,8 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
+    faraday_middleware (1.2.0)
+      faraday (~> 1.0)
     fast_blank (1.0.1)
     fastimage (2.2.6)
     ffi (1.15.5)
@@ -399,6 +409,10 @@ GEM
     mario-redis-lock (1.2.1)
       redis (>= 3.0.5)
     matrix (0.4.2)
+    md-paperclip-azure (2.2.0)
+      addressable (~> 2.5)
+      azure-storage-blob (~> 2.0.1)
+      hashie (~> 5.0)
     memory_profiler (1.0.1)
     method_source (1.0.0)
     mime-types (3.4.1)
@@ -410,6 +424,8 @@ GEM
     msgpack (1.6.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
+    net-http-persistent (4.0.2)
+      connection_pool (~> 2.2)
     net-imap (0.3.4)
       date
       net-protocol
@@ -817,6 +833,7 @@ DEPENDENCIES
   lograge (~> 0.12)
   makara (~> 0.5)
   mario-redis-lock (~> 1.2)
+  md-paperclip-azure (~> 2.2)
   memory_profiler
   mime-types (~> 3.4.1)
   net-ldap (~> 0.17)

app/helpers/application_helper.rb

@@ -180,11 +180,11 @@ def cdn_host?
   end
 
   def storage_host
-    "https://#{ENV['S3_ALIAS_HOST'].presence || ENV['S3_CLOUDFRONT_HOST']}"
+    "https://#{ENV['S3_ALIAS_HOST'].presence || ENV['S3_CLOUDFRONT_HOST'].presence || ENV['AZURE_ALIAS_HOST']}"
   end
 
   def storage_host?
-    ENV['S3_ALIAS_HOST'].present? || ENV['S3_CLOUDFRONT_HOST'].present?
+    ENV['S3_ALIAS_HOST'].present? || ENV['S3_CLOUDFRONT_HOST'].present? || ENV['AZURE_ALIAS_HOST'].present?
   end
 
   def quote_wrap(text, line_width: 80, break_sequence: "\n")

config/initializers/content_security_policy.rb

@@ -13,6 +13,7 @@ def host_to_url(str)
 
 media_host   = host_to_url(ENV['S3_ALIAS_HOST'])
 media_host ||= host_to_url(ENV['S3_CLOUDFRONT_HOST'])
+media_host ||= host_to_url(ENV['AZURE_ALIAS_HOST'])
 media_host ||= host_to_url(ENV['S3_HOSTNAME']) if ENV['S3_ENABLED'] == 'true'
 media_host ||= assets_host
 

config/initializers/paperclip.rb

@@ -132,6 +132,26 @@ def copy_to_local_file(style, local_dest_path)
     fog_host: ENV['SWIFT_OBJECT_URL'],
     fog_public: true
   )
+elsif ENV['AZURE_ENABLED'] == 'true'
+  require 'paperclip-azure'
+
+  Paperclip::Attachment.default_options.merge!(
+    storage: :azure,
+    azure_options: {
+      protocol: 'https'
+    },
+    azure_credentials: {
+      storage_account_name: ENV['AZURE_STORAGE_ACCOUNT'],
+      storage_access_key:   ENV['AZURE_STORAGE_ACCESS_KEY'],
+      container:            ENV['AZURE_CONTAINER_NAME']
+    }
+  )
+  if ENV.has_key?('AZURE_ALIAS_HOST')
+    Paperclip::Attachment.default_options.merge!(
+      url: ':azure_alias_url',
+      azure_host_alias: ENV['AZURE_ALIAS_HOST']
+    )
+  end
 else
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,