From 395dd55d0d226f9870277c9e85da8d3b4c651a58 Mon Sep 17 00:00:00 2001 From: Sam Beam Date: Mon, 6 Jul 2015 17:36:10 -0400 Subject: [PATCH 1/2] gzip files before upload if gzip_compression is set since Sprockets 3 no longer gzips assets for us (https://github.com/sstephenson/sprockets/commit/d388ef7cc0e36e710539fb11799d7e5518609d61) , we can do it here (#304) prior to upload. Also fix the pre-detection of filename in upload_file so the cache_control and expires header get set properly on the gzipped files (#308). --- asset_sync.gemspec | 1 + lib/asset_sync/storage.rb | 25 +++++++++++++++++-- .../app/assets/javascripts/application.js | 11 +++++++- spec/integration/aws_integration_spec.rb | 18 +++++++------ spec/unit/storage_spec.rb | 6 +++-- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/asset_sync.gemspec b/asset_sync.gemspec index b815dc07..611c11fa 100644 --- a/asset_sync.gemspec +++ b/asset_sync.gemspec @@ -2,6 +2,7 @@ $:.push File.expand_path("../lib", __FILE__) require "asset_sync/version" +require "zlib" Gem::Specification.new do |s| s.name = "asset_sync" diff --git a/lib/asset_sync/storage.rb b/lib/asset_sync/storage.rb index c67691c4..3ed0f806 100644 --- a/lib/asset_sync/storage.rb +++ b/lib/asset_sync/storage.rb @@ -76,7 +76,7 @@ def get_local_files elsif File.exist?(self.config.manifest_path) log "Using: Manifest #{self.config.manifest_path}" yml = YAML.load(IO.read(self.config.manifest_path)) - + return yml.map do |original, compiled| # Upload font originals and compiled if original =~ /^.+(eot|svg|ttf|woff)$/ @@ -137,7 +137,9 @@ def upload_file(f) :content_type => mime } - if /-[0-9a-fA-F]{32}$/.match(File.basename(f,File.extname(f))) + uncompressed_filename = f.sub(/\.gz\z/, '') + basename = File.basename(uncompressed_filename, File.extname(uncompressed_filename)) + if /-[0-9a-fA-F]{32,64}$/.match(basename) file.merge!({ :cache_control => "public, max-age=#{one_year}", :expires => CGI.rfc1123_date(Time.now + one_year) @@ -207,6 +209,24 @@ def upload_file(f) file = bucket.files.create( file ) unless ignore end + def compress_files + self.local_files.each do |f| + unless File.extname(f) == '.gz' + file = File.join(path, f) + gz_path = file + ".gz" + unless File.exist?(gz_path) + log "Writing #{gz_path}" + Zlib::GzipWriter.open(gz_path, Zlib::BEST_COMPRESSION) do |gz| + gz.mtime = File.mtime(file) + gz.orig_name = file + gz.write(IO.binread(file)) + end + end + end + end + @local_files = nil # break memoization to include new .gz files + end + def upload_files # get a fresh list of remote files remote_files = ignore_existing_remote_files? ? [] : get_remote_files @@ -231,6 +251,7 @@ def upload_files def sync # fixes: https://github.com/rumblelabs/asset_sync/issues/19 log "AssetSync: Syncing." + compress_files if config.gzip? upload_files delete_extra_remote_files unless keep_existing_remote_files? log "AssetSync: Done." diff --git a/spec/dummy_app/app/assets/javascripts/application.js b/spec/dummy_app/app/assets/javascripts/application.js index ad86b2ec..6a07b55e 100644 --- a/spec/dummy_app/app/assets/javascripts/application.js +++ b/spec/dummy_app/app/assets/javascripts/application.js @@ -1 +1,10 @@ -console.log("hello"); \ No newline at end of file +var ipsum = "Lorem ipsum dolor sit amet, mel lobortis volutpat reformidans eu. Neglegentur mediocritatem pri eu, eu per ignota probatus. Vis facilisi posidonium et, id eum numquam sapientem. Ignota minimum id per, rebum persius nominati sed id. \ + \ +Ad mea simul perfecto patrioque. Id sumo etiam evertitur per, iisque lucilius ne mei. In pro mentitum deserunt. Per cu nibh nominavi, ne pro velit ponderum verterem, dolor perpetua sit eu. \ + \ +Ei nec sanctus vivendo. Saepe partiendo vix ne. Stet labitur ut his, ei tibique vivendum quo. His iuvaret qualisque ex, at rebum fierent prodesset eos. Sit enim soluta et. \ + \ +Eripuit abhorreant efficiantur at his. Ex duo putent aliquip adipisci. Ea albucius vivendum mel, ex alienum omittantur vim. Illum ridens utroque at usu, vix at adhuc simul, elit possim oblique sit id. Posse postea gubergren eum at, qui in wisi option splendide, liber assentior disputando ei vix. An quo clita definiebas, sed possit pericula cu, tamquam accusata at vim." + +console.log(ipsum); + diff --git a/spec/integration/aws_integration_spec.rb b/spec/integration/aws_integration_spec.rb index 5ccba42a..a2a32481 100644 --- a/spec/integration/aws_integration_spec.rb +++ b/spec/integration/aws_integration_spec.rb @@ -13,8 +13,9 @@ def bucket(name) def execute(command) app_path = File.expand_path("../../dummy_app", __FILE__) - Dir.chdir app_path - `#{command}` + Dir.chdir app_path do + puts `#{command}` + end end describe "AssetSync" do @@ -47,14 +48,10 @@ def execute(command) files = bucket(@prefix).files app_js_path = files.select{ |f| f.key =~ app_js_regex }.first - app_js_gz_path = files.select{ |f| f.key =~ app_js_gz_regex }.first app_js = files.get( app_js_path.key ) expect(app_js.content_type).to eq("text/javascript") - - app_js_gz = files.get( app_js_gz_path.key ) - expect(app_js_gz.content_type).to eq("text/javascript") - expect(app_js_gz.content_encoding).to eq("gzip") + expect(app_js.cache_control).to match("public") end it "sync with enabled=false" do @@ -64,11 +61,16 @@ def execute(command) it "sync with gzip_compression=true" do execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile" - # bucket(@prefix).files.size.should == 3 + expect(bucket(@prefix).files.size).to eq(1) app_js_path = files.select{ |f| f.key =~ app_js_regex }.first + + expect(app_js_path).not_to be_nil app_js = files.get( app_js_path.key ) expect(app_js.content_type).to eq("text/javascript") + expect(app_js.content_encoding).to eq("gzip") + expect(app_js.cache_control).to match("public") + expect(app_js.key).to match(/\.js$/) end end diff --git a/spec/unit/storage_spec.rb b/spec/unit/storage_spec.rb index 730f950b..b0c7ab8f 100644 --- a/spec/unit/storage_spec.rb +++ b/spec/unit/storage_spec.rb @@ -84,8 +84,8 @@ it 'should correctly set expire date' do - local_files = ['file1.jpg', 'file1-1234567890abcdef1234567890abcdef.jpg'] - local_files += ['dir1/dir2/file2.jpg', 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg'] + local_files = ['file1.jpg', 'file1-1234567890abcdef1234567890abcdef.jpg', 'file1-1234567890abcdef1234567890abcdef.jpg.gz'] + local_files += ['dir1/dir2/file2.jpg', 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg', 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg.gz'] remote_files = [] storage = AssetSync::Storage.new(@config) allow(storage).to receive(:local_files).and_return(local_files) @@ -99,7 +99,9 @@ def check_file(file) when 'dir1/dir2/file2.jpg' !expect(file).not_to include(:cache_control, :expires) when 'file1-1234567890abcdef1234567890abcdef.jpg' + when 'file1-1234567890abcdef1234567890abcdef.jpg.gz' when 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg' + when 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg.gz' expect(file).to include(:cache_control, :expires) else fail From d52e970c8965127cd8de778ff63070e5737ad1ba Mon Sep 17 00:00:00 2001 From: Sam Beam Date: Tue, 7 Jul 2015 15:25:52 -0400 Subject: [PATCH 2/2] compress only text files, set extensions via ENV improve Storage#compress_files to only compress files with extensions matching config.extensions_to_gzip. Add test for #compress_files --- lib/asset_sync/config.rb | 2 ++ lib/asset_sync/engine.rb | 1 + lib/asset_sync/storage.rb | 17 +++++++++-------- spec/unit/storage_spec.rb | 20 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/asset_sync/config.rb b/lib/asset_sync/config.rb index 88d88510..98f289d3 100644 --- a/lib/asset_sync/config.rb +++ b/lib/asset_sync/config.rb @@ -7,6 +7,7 @@ class Invalid < StandardError; end # AssetSync attr_accessor :existing_remote_files # What to do with your existing remote files? (keep or delete) attr_accessor :gzip_compression + attr_accessor :extensions_to_gzip attr_accessor :manifest attr_accessor :fail_silently attr_accessor :log_silently @@ -50,6 +51,7 @@ def initialize self.fog_region = nil self.existing_remote_files = 'keep' self.gzip_compression = false + self.extensions_to_gzip = 'css,js' self.manifest = false self.fail_silently = false self.log_silently = true diff --git a/lib/asset_sync/engine.rb b/lib/asset_sync/engine.rb index 0f6aa534..768961bc 100644 --- a/lib/asset_sync/engine.rb +++ b/lib/asset_sync/engine.rb @@ -32,6 +32,7 @@ class Engine < Rails::Engine config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep" config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION') + config.extensions_to_gzip = ENV['ASSET_SYNC_EXTENSIONS_TO_GZIP'] if ENV.has_key?('ASSET_SYNC_EXTENSIONS_TO_GZIP') config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') end diff --git a/lib/asset_sync/storage.rb b/lib/asset_sync/storage.rb index 3ed0f806..91c096dc 100644 --- a/lib/asset_sync/storage.rb +++ b/lib/asset_sync/storage.rb @@ -210,16 +210,17 @@ def upload_file(f) end def compress_files - self.local_files.each do |f| - unless File.extname(f) == '.gz' - file = File.join(path, f) - gz_path = file + ".gz" - unless File.exist?(gz_path) + extensions_to_gzip = Regexp.union(*config.extensions_to_gzip.split(',')) + self.local_files.each do |file| + if File.extname(file)[1..-1] =~ extensions_to_gzip + file_path = File.join(path, file) + if File.file?(file_path) + gz_path = file_path + ".gz" log "Writing #{gz_path}" Zlib::GzipWriter.open(gz_path, Zlib::BEST_COMPRESSION) do |gz| - gz.mtime = File.mtime(file) - gz.orig_name = file - gz.write(IO.binread(file)) + gz.mtime = File.mtime(file_path) + gz.orig_name = file_path + gz.write(IO.binread(file_path)) end end end diff --git a/spec/unit/storage_spec.rb b/spec/unit/storage_spec.rb index b0c7ab8f..d99b842f 100644 --- a/spec/unit/storage_spec.rb +++ b/spec/unit/storage_spec.rb @@ -193,4 +193,24 @@ def check_file(file) #Object.send(:remove_const, :MIME) if defined?(MIME) end end + + describe '#compress_files' do + before(:each) do + @config = AssetSync::Config.new + @config.public_path = 'public' + end + + it 'should compress text files' do + local_files = ['jquery.js', 'file1.jpg', 'application.css', 'logo.png', 'dir'] + storage = AssetSync::Storage.new(@config) + allow(storage).to receive(:local_files).and_return(local_files) + allow(File).to receive(:file?).and_return(true) + allow(File).to receive(:open).and_return(nil) + + expect(Zlib::GzipWriter).to receive(:open).with('public/application.css.gz', anything).and_return(nil) + expect(Zlib::GzipWriter).to receive(:open).with('public/jquery.js.gz', anything).and_return(nil) + storage.compress_files + end + + end end