From 83af3e594f561c4f1c3c35b59f34089b2843a887 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Mon, 17 Jun 2013 13:49:40 -0400 Subject: [PATCH 1/4] Use built-in rakelibdir feature Rake by default imports all .rake files in the rakelib dir, so we can use that rather than doing our own import loop. --- rakefile | 6 +----- {rakefiles => rakelib}/assets.rake | 0 {rakefiles => rakelib}/deploy.rake | 0 {rakefiles => rakelib}/deprecated.rake | 0 {rakefiles => rakelib}/django.rake | 0 {rakefiles => rakelib}/docs.rake | 0 {rakefiles => rakelib}/helpers.rb | 0 {rakefiles => rakelib}/i18n.rake | 0 {rakefiles => rakelib}/jasmine.rake | 0 {rakefiles => rakelib}/prereqs.rake | 2 -- {rakefiles => rakelib}/quality.rake | 0 {rakefiles => rakelib}/tests.rake | 0 {rakefiles => rakelib}/workspace.rake | 0 13 files changed, 1 insertion(+), 7 deletions(-) rename {rakefiles => rakelib}/assets.rake (100%) rename {rakefiles => rakelib}/deploy.rake (100%) rename {rakefiles => rakelib}/deprecated.rake (100%) rename {rakefiles => rakelib}/django.rake (100%) rename {rakefiles => rakelib}/docs.rake (100%) rename {rakefiles => rakelib}/helpers.rb (100%) rename {rakefiles => rakelib}/i18n.rake (100%) rename {rakefiles => rakelib}/jasmine.rake (100%) rename {rakefiles => rakelib}/prereqs.rake (98%) rename {rakefiles => rakelib}/quality.rake (100%) rename {rakefiles => rakelib}/tests.rake (100%) rename {rakefiles => rakelib}/workspace.rake (100%) diff --git a/rakefile b/rakefile index 20101a14db58..3fcd16f995ef 100644 --- a/rakefile +++ b/rakefile @@ -1,10 +1,6 @@ require 'json' require 'rake/clean' -require './rakefiles/helpers.rb' - -Dir['rakefiles/*.rake'].each do |rakefile| - import rakefile -end +require './rakelib/helpers.rb' # Build Constants REPO_ROOT = File.dirname(__FILE__) diff --git a/rakefiles/assets.rake b/rakelib/assets.rake similarity index 100% rename from rakefiles/assets.rake rename to rakelib/assets.rake diff --git a/rakefiles/deploy.rake b/rakelib/deploy.rake similarity index 100% rename from rakefiles/deploy.rake rename to rakelib/deploy.rake diff --git a/rakefiles/deprecated.rake b/rakelib/deprecated.rake similarity index 100% rename from rakefiles/deprecated.rake rename to rakelib/deprecated.rake diff --git a/rakefiles/django.rake b/rakelib/django.rake similarity index 100% rename from rakefiles/django.rake rename to rakelib/django.rake diff --git a/rakefiles/docs.rake b/rakelib/docs.rake similarity index 100% rename from rakefiles/docs.rake rename to rakelib/docs.rake diff --git a/rakefiles/helpers.rb b/rakelib/helpers.rb similarity index 100% rename from rakefiles/helpers.rb rename to rakelib/helpers.rb diff --git a/rakefiles/i18n.rake b/rakelib/i18n.rake similarity index 100% rename from rakefiles/i18n.rake rename to rakelib/i18n.rake diff --git a/rakefiles/jasmine.rake b/rakelib/jasmine.rake similarity index 100% rename from rakefiles/jasmine.rake rename to rakelib/jasmine.rake diff --git a/rakefiles/prereqs.rake b/rakelib/prereqs.rake similarity index 98% rename from rakefiles/prereqs.rake rename to rakelib/prereqs.rake index ff8b4b8784c0..e06d41143582 100644 --- a/rakefiles/prereqs.rake +++ b/rakelib/prereqs.rake @@ -1,5 +1,3 @@ -require './rakefiles/helpers.rb' - PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache') CLOBBER.include(PREREQS_MD5_DIR) diff --git a/rakefiles/quality.rake b/rakelib/quality.rake similarity index 100% rename from rakefiles/quality.rake rename to rakelib/quality.rake diff --git a/rakefiles/tests.rake b/rakelib/tests.rake similarity index 100% rename from rakefiles/tests.rake rename to rakelib/tests.rake diff --git a/rakefiles/workspace.rake b/rakelib/workspace.rake similarity index 100% rename from rakefiles/workspace.rake rename to rakelib/workspace.rake From d9268acd6bb9b19175315998de623818e0a799f1 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Mon, 17 Jun 2013 13:52:00 -0400 Subject: [PATCH 2/4] Provide instructions of ruby imports fail in rake `rake install_prereqs` requires a minimal level of ruby and rake already installed. If it doesn't exist, then print out a helpful message indicating next steps. --- CHANGELOG.rst | 2 ++ rakefile | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7fc07a3f19f9..9c7af1520be6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ These are notable changes in edx-platform. This is a rolling list of changes, in roughly chronological order, most recent first. Add your entries at or near the top. Include a label indicating the component affected. +Common: Make rake provide better error messages if packages are missing. + Common: Repairs development documentation generation by sphinx. LMS: Problem rescoring. Added options on the Grades tab of the diff --git a/rakefile b/rakefile index 3fcd16f995ef..96bd4c2e96c2 100644 --- a/rakefile +++ b/rakefile @@ -1,6 +1,12 @@ -require 'json' -require 'rake/clean' -require './rakelib/helpers.rb' +begin + require 'json' + require 'rake/clean' + require './rakelib/helpers.rb' +rescue LoadError => error + puts "Import faild (#{error})" + puts "Please run `bundle install` to bootstrap ruby dependencies" + exit 1 +end # Build Constants REPO_ROOT = File.dirname(__FILE__) From d99ad53ae90bebc0fb2c099b42a2ffa1c64ab4b8 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 19 Jun 2013 15:30:47 -0400 Subject: [PATCH 3/4] Add system and env arguments to asset tasks The preprocess task requires system and env arguments in order to correctly load up the django environment to preprocess sass files to inject themes. In order for that task to recieve the arguments, all tasks that depend on it also have to accept that same set of arguments. This will all go away once the next evolution of themes arrives, which will remove the preprocessing needed to inject theme names. --- rakelib/assets.rake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rakelib/assets.rake b/rakelib/assets.rake index 764d049a683e..0c58047bc202 100644 --- a/rakelib/assets.rake +++ b/rakelib/assets.rake @@ -55,8 +55,9 @@ def sass_cmd(watch=false, debug=false) "#{watch ? '--watch' : '--update'} -E utf-8 #{sass_watch_paths.join(' ')}" end +# This task takes arguments purely to pass them via dependencies to the preprocess task desc "Compile all assets" -multitask :assets => 'assets:all' +task :assets, [:system, :env] => 'assets:all' namespace :assets do @@ -80,8 +81,9 @@ namespace :assets do {:xmodule => [:install_python_prereqs], :coffee => [:install_node_prereqs, :'assets:coffee:clobber'], :sass => [:install_ruby_prereqs, :preprocess]}.each_pair do |asset_type, prereq_tasks| + # This task takes arguments purely to pass them via dependencies to the preprocess task desc "Compile all #{asset_type} assets" - task asset_type => prereq_tasks do + task asset_type, [:system, :env] => prereq_tasks do |t, args| cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false) if cmd.kind_of?(Array) cmd.each {|c| sh(c)} @@ -90,7 +92,8 @@ namespace :assets do end end - multitask :all => asset_type + # This task takes arguments purely to pass them via dependencies to the preprocess task + multitask :all, [:system, :env] => asset_type multitask :debug => "assets:#{asset_type}:debug" multitask :_watch => "assets:#{asset_type}:_watch" From 64912a774199e40a68cab3020d3e98ad9bb8c6e5 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 14 Jun 2013 21:17:08 -0400 Subject: [PATCH 4/4] Make assets watchers run as singletons Previously, multiple copies of the watchers started from the different shells would run simultaneously, which left the possiblity of zombie watchers, increased resource consumption, and incorrect results. This fixes that problem by only starting a watcher if that same command isn't already in the process list. Fixes LMS-499 --- CHANGELOG.rst | 3 +++ Gemfile | 1 + rakelib/assets.rake | 4 ++-- rakelib/helpers.rb | 12 ++++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c7af1520be6..18855e82aefc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ These are notable changes in edx-platform. This is a rolling list of changes, in roughly chronological order, most recent first. Add your entries at or near the top. Include a label indicating the component affected. +Common: Make asset watchers run as singletons (so they won't start if the +watcher is already running in another shell). + Common: Make rake provide better error messages if packages are missing. Common: Repairs development documentation generation by sphinx. diff --git a/Gemfile b/Gemfile index 7f7b14697854..1ad685c34d86 100644 --- a/Gemfile +++ b/Gemfile @@ -4,3 +4,4 @@ gem 'sass', '3.1.15' gem 'bourbon', '~> 1.3.6' gem 'colorize', '~> 0.5.8' gem 'launchy', '~> 2.1.2' +gem 'sys-proctable', '~> 0.9.3' diff --git a/rakelib/assets.rake b/rakelib/assets.rake index 0c58047bc202..c935b0d53b46 100644 --- a/rakelib/assets.rake +++ b/rakelib/assets.rake @@ -114,9 +114,9 @@ namespace :assets do task :_watch => (prereq_tasks + ["assets:#{asset_type}:debug"]) do cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true) if cmd.kind_of?(Array) - cmd.each {|c| background_process(c)} + cmd.each {|c| singleton_process(c)} else - background_process(cmd) + singleton_process(cmd) end end end diff --git a/rakelib/helpers.rb b/rakelib/helpers.rb index 4b10bef709e0..3373214a1930 100644 --- a/rakelib/helpers.rb +++ b/rakelib/helpers.rb @@ -1,4 +1,6 @@ require 'digest/md5' +require 'sys/proctable' +require 'colorize' def find_executable(exec) path = %x(which #{exec}).strip @@ -84,6 +86,16 @@ def background_process(*command) end end +# Runs a command as a background process, as long as no other processes +# tagged with the same tag are running +def singleton_process(*command) + if Sys::ProcTable.ps.select {|proc| proc.cmdline.include?(command.join(' '))}.empty? + background_process(*command) + else + puts "Process '#{command.join(' ')} already running, skipping".blue + end +end + def environments(system) Dir["#{system}/envs/**/*.py"].select{|file| ! (/__init__.py$/ =~ file)}.map do |env_file| env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')