Add spark UI (#30)

* enable unauthenticated access to spark ui * add a cookie for client to authenticate
OSC · Jul 13, 2023 · 1b7e8cb · 1b7e8cb
1 parent 46aac3a
commit 1b7e8cb
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 11 deletions.
diff --git a/form.yml.erb b/form.yml.erb
@@ -80,6 +80,9 @@ attributes:
   spark_version:
     widget: "select"
     options:
+      - [
+          "3.4.1", "3.4.1",
+        ]
       - [
         "3.0.1", "3.0.1",
         ]

diff --git a/submit.yml.erb b/submit.yml.erb
@@ -21,7 +21,12 @@
 ---
 batch_connect:
   template: "basic"
-  conn_params: [ "tutorials_root" ]
+  conn_params:
+    - "tutorials_root"
+    - "spark_master_webui_port"
+    - "spark_master_host"
+    - "spark_ui_auth_token"
+    - "spark_version"
 script:
   accounting_id: "<%= account %>"
   native:

diff --git a/template/before.sh.erb b/template/before.sh.erb
@@ -31,23 +31,29 @@
     end
   end
 
+  def spark_ui_config
+    if context.spark_version >= '3.4.1'
+      {
+        "spark.ui.reverseProxy" => "true",
+        "spark.ui.reverseProxyUrl" => "/rnode/${SPARK_MASTER_HOST}/${SPARK_MASTER_WEBUI_PORT}",
+        "spark.ui.enabled" => "true",
+        "spark.ui.killEnabled" => "false",
+        "spark.ui.filters" => "edu.osc.spark.AuthFilter",
+      }
+    else
+      {}
+    end
+  end
+
   def exec_memory
     calculated = avail_mem/context.num_workers.to_i
     [60, calculated].min
   end
 
   spark_config = {
-    "spark.ui.reverseProxy" => "true",
-    #"spark.ui.reverseProxyUrl" => "https://ondemand.osc.edu/node/${SPARK_MASTER_HOST}/${SPARK_MASTER_WEBUI_PORT}",
     "spark.authenticate" => "true",
     "spark.authenticate.secret" => "${SPARK_SECRET}",
-    # Comment out below when reverse proxy and authentication are added
-    # This still starts the Web UI on the master/worker procs, but disables it for
-    # the Application driver
-    "spark.ui.enabled" => "false",
-    # So we need to disable the ability to kill applications
-    "spark.ui.killEnabled" => "false",
-  }.merge(extra_spark_config)
+  }.merge(extra_spark_config).merge(spark_ui_config)
 
   def pyspark_submit_args
     args = []
@@ -73,6 +79,9 @@ port=$(find_port ${host})
 SALT="$(create_passwd 16)"
 password="$(create_passwd 16)"
 PASSWORD_SHA1="$(echo -n "${password}${SALT}" | openssl dgst -sha1 | awk '{print $NF}')"
+export SPARK_UI_AUTH_TOKEN=$(uuid)
+export spark_ui_auth_token=$SPARK_UI_AUTH_TOKEN
+export spark_version=<%= context.spark_version %>
 
 # Notebook root directory
 export NOTEBOOK_ROOT="${NOTEBOOK_ROOT:-${HOME}}"
@@ -147,6 +156,8 @@ mkdir "${LOG_ROOT}"
 export SPARK_MASTER_HOST=${host}
 export SPARK_MASTER_PORT=$(find_port ${SPARK_MASTER_HOST})
 export SPARK_MASTER_WEBUI_PORT=$(find_port ${SPARK_MASTER_HOST})
+export spark_master_webui_port=$SPARK_MASTER_WEBUI_PORT
+export spark_master_host=$SPARK_MASTER_HOST
 
 # Generate Spark secret
 SPARK_SECRET="$(create_passwd)"

diff --git a/view.html.erb b/view.html.erb
@@ -5,15 +5,35 @@ Some helpful resources on using Apache Spark in a Jupyter notebook:
   (some work is required to translate the Python 2 examples here to Python 3)
 - <a href="https://github.com/jadianes/spark-py-notebooks/blob/master/nb2-rdd-basics/nb2-rdd-basics.ipynb" target="_blank">RDD Basics</a>
 
+<script type="text/javascript">
+  (function () {
+    let date = new Date();
+    date.setTime(date.getTime() + (7*24*60*60*1000));
+    let expires = `expires="${date.toUTCString()}"`;
+    let cookiePath =  'path=/rnode/<%= spark_master_host %>/<%= spark_master_webui_port %>';
+    let cookie = `spark_ui_auth_token=<%= spark_ui_auth_token %>;${cookiePath};${expires};samesite=strict;secure;`;
+    document.cookie = cookie;
+  })();
+  </script>
+
 <div class="row">
-  <div class="col-xs-<%= tutorials_root.blank? ? "12" : "6" %>">
+  <div class="col-md-<%= tutorials_root.blank? ? "6" : "4" %>">
     <form action="/node/<%= host %>/<%= port %>/login" method="post" target="_blank">
       <input type="hidden" name="password" value="<%= password %>">
       <button class="btn btn-primary" type="submit" style="margin-top: 6px;">
         <i class="fa fa-cogs"></i> Connect to Jupyter
       </button>
     </form>
   </div>
+
+  <%- if spark_version.to_s >= '3.4.1' -%>
+  <div class="col-md-<%= tutorials_root.blank? ? "6" : "4" %>">
+    <a class="btn btn-primary" target="_blank" href="/rnode/<%= spark_master_host %>/<%= spark_master_webui_port %>">
+      <i class="fa fa-star"></i> Connect to Spark WebUI
+    </a>
+  </div>
+  <%- end-%>
+
   <%- unless tutorials_root.blank? -%>
   <div class="ml-3 col-xs-6">
     <form action="/node/<%= host %>/<%= port %>/login?next=<%= url_encode "/node/#{host}/#{port}/tree#{tutorials_root}" %>" method="post" target="_blank" class="pull-right">