From e46e97a1b560f3986c320c19bd3c886be8288066 Mon Sep 17 00:00:00 2001 From: Oren Mazor Date: Wed, 4 Feb 2015 16:24:25 +0000 Subject: [PATCH 1/2] initialize_variables_correctly. --- Capfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Capfile b/Capfile index b13a149f5cd2e..930d77992a285 100644 --- a/Capfile +++ b/Capfile @@ -41,12 +41,12 @@ namespace :deploy do end task :upload_to_hdfs, :roles => :uploader, :on_no_matching_servers => :continue do - target_sha = ENV['SHA'] || fetch(:sha, `git rev-parse HEAD`.gsub(/\s/,"")) + target_sha = ENV['SHA'] || fetch(:sha) run "hdfs dfs -copyFromLocal -f /u/apps/spark/current/lib/spark-assembly-*.jar hdfs://nn01.chi.shopify.com/user/sparkles/spark-assembly-#{target_sha}.jar" end task :test_spark_jar, :roles => :uploader, :on_no_master_servers => :continue do - target_sha = ENV['SHA'] || fetch(:sha, `git rev-parse HEAD`.gsub(/\s/,"")) + target_sha = ENV['SHA'] || fetch(:sha) run "sudo -u azkaban sh -c '. /u/virtualenvs/starscream/bin/activate && cd /u/apps/starscream/current && PYTHON_ENV=production SPARK_OPTS=\"spark.yarn.jar=hdfs://nn01.chi.shopify.com:8020/user/sparkles/spark-assembly-#{target_sha}.jar\" exec python shopify/tools/canary.py'" end @@ -79,6 +79,8 @@ namespace :deploy do after 'deploy:initialize_variables', 'deploy:prevent_gateway' # capistrano recipes packserv deploy always uses a gateway before 'deploy:symlink_current', 'deploy:symlink_shared' + before 'deploy:test_spark_jar', 'deploy:initialize_variables' + before 'deploy:upload_to_hdfs', 'deploy:initialize_variables' after 'deploy:download', 'deploy:upload_to_hdfs', 'deploy:test_spark_jar' after 'deploy:restart', 'deploy:cleanup' after 'deploy:cleanup', 'deploy:remind_us_to_update_starscream' From e19dff19e8e0d0ee5b2a899ad02f376360cf8402 Mon Sep 17 00:00:00 2001 From: Oren Mazor Date: Wed, 4 Feb 2015 16:26:05 +0000 Subject: [PATCH 2/2] fetch(:sha) actually already uses env[sha] --- Capfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Capfile b/Capfile index 930d77992a285..3739e6e4e3894 100644 --- a/Capfile +++ b/Capfile @@ -41,13 +41,11 @@ namespace :deploy do end task :upload_to_hdfs, :roles => :uploader, :on_no_matching_servers => :continue do - target_sha = ENV['SHA'] || fetch(:sha) - run "hdfs dfs -copyFromLocal -f /u/apps/spark/current/lib/spark-assembly-*.jar hdfs://nn01.chi.shopify.com/user/sparkles/spark-assembly-#{target_sha}.jar" + run "hdfs dfs -copyFromLocal -f /u/apps/spark/current/lib/spark-assembly-*.jar hdfs://nn01.chi.shopify.com/user/sparkles/spark-assembly-#{fetch(:sha)}.jar" end task :test_spark_jar, :roles => :uploader, :on_no_master_servers => :continue do - target_sha = ENV['SHA'] || fetch(:sha) - run "sudo -u azkaban sh -c '. /u/virtualenvs/starscream/bin/activate && cd /u/apps/starscream/current && PYTHON_ENV=production SPARK_OPTS=\"spark.yarn.jar=hdfs://nn01.chi.shopify.com:8020/user/sparkles/spark-assembly-#{target_sha}.jar\" exec python shopify/tools/canary.py'" + run "sudo -u azkaban sh -c '. /u/virtualenvs/starscream/bin/activate && cd /u/apps/starscream/current && PYTHON_ENV=production SPARK_OPTS=\"spark.yarn.jar=hdfs://nn01.chi.shopify.com:8020/user/sparkles/spark-assembly-#{fetch(:sha)}.jar\" exec python shopify/tools/canary.py'" end task :prevent_gateway do