From dfa8c4248e540676ef73999b2bc5782f6cf8a8bc Mon Sep 17 00:00:00 2001 From: Alex Angelini Date: Wed, 4 Feb 2015 18:00:52 -0500 Subject: [PATCH] Added support for HA namenode --- Capfile | 10 +++++----- conf/spark-defaults.conf | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Capfile b/Capfile index 3739e6e4e3894..2dc4cad8d1772 100644 --- a/Capfile +++ b/Capfile @@ -6,7 +6,7 @@ set :user, "deploy" set :shared_work_path, "/u/apps/spark/shared/work" set :shared_logs_path, "/u/apps/spark/shared/log" set :shared_conf_path, "/u/apps/spark/shared/conf" -set :spark_jar_path, "hdfs://nn01.chi.shopify.com/user/sparkles" +set :spark_jar_path, "hdfs://hadoop-production/user/sparkles" set :gateway, nil set :keep_releases, 5 @@ -41,11 +41,11 @@ namespace :deploy do end task :upload_to_hdfs, :roles => :uploader, :on_no_matching_servers => :continue do - run "hdfs dfs -copyFromLocal -f /u/apps/spark/current/lib/spark-assembly-*.jar hdfs://nn01.chi.shopify.com/user/sparkles/spark-assembly-#{fetch(:sha)}.jar" + run "hdfs dfs -copyFromLocal -f /u/apps/spark/current/lib/spark-assembly-*.jar hdfs://hadoop-production/user/sparkles/spark-assembly-#{fetch(:sha)}.jar" end task :test_spark_jar, :roles => :uploader, :on_no_master_servers => :continue do - run "sudo -u azkaban sh -c '. /u/virtualenvs/starscream/bin/activate && cd /u/apps/starscream/current && PYTHON_ENV=production SPARK_OPTS=\"spark.yarn.jar=hdfs://nn01.chi.shopify.com:8020/user/sparkles/spark-assembly-#{fetch(:sha)}.jar\" exec python shopify/tools/canary.py'" + run "sudo -u azkaban sh -c '. /u/virtualenvs/starscream/bin/activate && cd /u/apps/starscream/current && PYTHON_ENV=production SPARK_OPTS=\"spark.yarn.jar=hdfs://hadoop-production/user/sparkles/spark-assembly-#{fetch(:sha)}.jar\" exec python shopify/tools/canary.py'" end task :prevent_gateway do @@ -60,7 +60,7 @@ namespace :deploy do task :remind_us_to_update_starscream do puts "****************************************************************" - puts "*" + puts "*" puts "* Remember to update starscream/conf/config.yml" puts "*" puts "* spark_production" @@ -70,7 +70,7 @@ namespace :deploy do puts "*" puts "****************************************************************" end - + task :restart do end diff --git a/conf/spark-defaults.conf b/conf/spark-defaults.conf index 8d2f6112342d3..9387fea00ba3e 100644 --- a/conf/spark-defaults.conf +++ b/conf/spark-defaults.conf @@ -8,4 +8,4 @@ spark.cores.max 150 spark.ui.port 0 -spark.yarn.jar hdfs://nn01.chi.shopify.com:8020/user/sparkles/spark-assembly-latest.jar +spark.yarn.jar hdfs://hadoop-production/user/sparkles/spark-assembly-latest.jar