PPML: add k8s to Gramine Docker

ppml/trusted-big-data-ml/python/docker-gramine/bigdl-ppml-submit.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+SGX_ENABLED=false
+application_args=""
+input_args=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --master)
+      MASTER="$2"
+      input_args="$input_args $1 $2"
+      shift # past argument
+      shift # past value
+      ;;
+    --deploy-mode)
+      DEPLOY_MODE="$2"
+      input_args="$input_args $1 $2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-enabled)
+      SGX_ENABLED="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-log-level)
+      SGX_LOG_LEVEL="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-driver-memory)
+      SGX_DRIVER_MEM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-driver-jvm-memory)
+      SGX_DRIVER_JVM_MEM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-executor-memory)
+      SGX_EXECUTOR_MEM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --sgx-executor-jvm-memory)
+      SGX_EXECUTOR_JVM_MEM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --verbose)
+      input_args="$input_args $1"
+      shift # past argument
+      ;;
+    -*|--*)
+      input_args="$input_args $1 $2"
+      shift
+      shift
+      ;;
+    *)
+      application_args="${@}" # save positional arg
+      break
+      ;;
+  esac
+done
+
+echo "input_args $input_args"
+echo "app_args $application_args"
+echo $MASTER
+if [ "$MASTER" == k8s* ] && [ "$DEPLOY_MODE" = "" ]; then
+  echo "--deploy-mode should be specified for k8s cluster"
+  exit 1
+fi
+
+
+if [ "$SGX_ENABLED" = "true" ]; then
+  if [ "$SGX_DRIVER_MEM" = "" ] || [ "$SGX_DRIVER_JVM_MEM" = "" ] || [ "$SGX_EXECUTOR_MEM" = "" ] || [ "$SGX_EXECUTOR_JVM_MEM" = "" ] || [ "$SGX_LOG_LEVEL" = "" ]; then
+    echo "--sgx-driver-memory, --sgx-driver-jvm-memory, --sgx-executor-memory, --sgx-executor-jvm-memory, --sgx-log-level must be specified when sgx is enabled"
+    exit 1
+  else
+    sgx_commands="--conf spark.kubernetes.sgx.enabled=$SGX_ENABLED \
+        --conf spark.kubernetes.sgx.driver.mem=$SGX_DRIVER_MEM \
+        --conf spark.kubernetes.sgx.driver.jvm.mem=$SGX_DRIVER_JVM_MEM \
+        --conf spark.kubernetes.sgx.executor.mem=$SGX_EXECUTOR_MEM \
+        --conf spark.kubernetes.sgx.executor.jvm.mem=$SGX_EXECUTOR_JVM_MEM \
+        --conf spark.kubernetes.sgx.log.level=$SGX_LOG_LEVEL"
+  fi
+else
+  sgx_commands=""
+fi
+
+default_config="--conf spark.driver.host=$LOCAL_IP \
+        --conf spark.driver.port=$RUNTIME_DRIVER_PORT \
+        --conf spark.network.timeout=10000000 \
+        --conf spark.executor.heartbeatInterval=10000000 \
+        --conf spark.python.use.daemon=false \
+        --conf spark.python.worker.reuse=false \
+        --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+        --conf spark.kubernetes.driver.podTemplateFile=/ppml/trusted-big-data-ml/spark-driver-template.yaml \
+        --conf spark.kubernetes.executor.podTemplateFile=/ppml/trusted-big-data-ml/spark-executor-template.yaml \
+        --conf spark.kubernetes.executor.deleteOnTermination=false"
+
+if [ $secure_password ]; then
+   SSL="--conf spark.authenticate=true \
+    --conf spark.authenticate.secret=$secure_password \
+    --conf spark.kubernetes.executor.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret"  \
+    --conf spark.kubernetes.driver.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret"  \
+    --conf spark.authenticate.enableSaslEncryption=true \
+    --conf spark.network.crypto.enabled=true  \
+    --conf spark.network.crypto.keyLength=128  \
+    --conf spark.network.crypto.keyFactoryAlgorithm=PBKDF2WithHmacSHA1 \
+    --conf spark.io.encryption.enabled=true \
+    --conf spark.io.encryption.keySizeBits=128 \
+    --conf spark.io.encryption.keygen.algorithm=HmacSHA1 \
+    --conf spark.ssl.enabled=true \
+    --conf spark.ssl.port=8043 \
+    --conf spark.ssl.keyPassword=$secure_password \
+    --conf spark.ssl.keyStore=/ppml/trusted-big-data-ml/work/keys/keystore.jks  \
+    --conf spark.ssl.keyStorePassword=$secure_password \
+    --conf spark.ssl.keyStoreType=JKS \
+    --conf spark.ssl.trustStore=/ppml/trusted-big-data-ml/work/keys/keystore.jks \
+    --conf spark.ssl.trustStorePassword=$secure_password \
+    --conf spark.ssl.trustStoreType=JKS"
+else
+   SSL=""
+fi
+
+spark_submit_command="${JAVA_HOME}/bin/java \
+        -cp ${SPARK_HOME}/conf/:${SPARK_HOME}/jars/* \
+        -Xmx${RUNTIME_DRIVER_MEMORY} \
+        org.apache.spark.deploy.SparkSubmit \
+        $SSL \
+        $default_config \
+        $sgx_commands"
+
+set -x
+
+spark_submit_command="${spark_submit_command} ${input_args} ${application_args}"
+echo "spark_submit_command $spark_submit_command"
+if [ "$SGX_ENABLED" == "true" ] && [ "$DEPLOY_MODE" != "cluster" ]; then
+    ./clean.sh
+    gramine-argv-serializer bash -c "$spark_submit_command" > /ppml/trusted-big-data-ml/secured_argvs
+
+    ./init.sh
+    gramine-sgx bash 2>&1 | tee bigdl-ppml-submit.log
+else
+    $spark_submit_command 2>&1 | tee bigdl-ppml-submit.log
+fi

ppml/trusted-big-data-ml/python/docker-gramine/deploy-local-spark-sgx.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# KEYS_PATH means the absolute path to the keys folder
+# ENCLAVE_KEY_PATH means the absolute path to the "enclave-key.pem" file
+# LOCAL_IP means your local IP address.
+export SSL_KEYS_PATH=YOUR_LOCAL_SSL_KEYS_FOLDER_PATH
+export ENCLAVE_KEY_PATH=YOUR_LOCAL_ENCLAVE_KEY_PATH
+export LOCAL_IP=YOUR_LOCAL_IP
+export DOCKER_IMAGE=YOUR_DOCKER_IMAGE
+
+sudo docker run -itd \
+    --privileged \
+    --net=host \
+    --cpuset-cpus="0-5" \
+    --oom-kill-disable \
+    --device=/dev/gsgx \
+    --device=/dev/sgx/enclave \
+    --device=/dev/sgx/provision \
+    -v $ENCLAVE_KEY_PATH:/root/.config/gramine/enclave-key.pem \
+    -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \
+    -v $SSL_KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \
+    --name=gramine-test \
+    -e LOCAL_IP=$LOCAL_IP \
+    -e SGX_MEM_SIZE=64G \
+    $DOCKER_IMAGE bash

ppml/trusted-big-data-ml/python/docker-gramine/entrypoint.sh

@@ -0,0 +1,204 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# echo commands to the terminal output
+set -ex
+
+# Check whether there is a passwd entry for the container UID
+myuid=$(id -u)
+mygid=$(id -g)
+# turn off -e for getent because it will return error code in anonymous uid case
+set +e
+uidentry=$(getent passwd $myuid)
+set -e
+
+# If there is no passwd entry for the container UID, attempt to create one
+if [ -z "$uidentry" ] ; then
+    if [ -w /etc/passwd ] ; then
+        echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd
+    else
+        echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
+    fi
+fi
+
+SPARK_K8S_CMD="$1"
+echo "###################################### $SPARK_K8S_CMD"
+case "$SPARK_K8S_CMD" in
+    driver | driver-py | driver-r | executor)
+      shift 1
+      ;;
+    "")
+      ;;
+    *)
+      echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..."
+      exec /usr/bin/tini -s -- "$@"
+      ;;
+esac
+
+SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
+env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt
+readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
+
+if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
+  SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
+fi
+
+if [ -n "$PYSPARK_FILES" ]; then
+    PYTHONPATH="$PYTHONPATH:$PYSPARK_FILES"
+fi
+
+PYSPARK_ARGS=""
+if [ -n "$PYSPARK_APP_ARGS" ]; then
+    PYSPARK_ARGS="$PYSPARK_APP_ARGS"
+fi
+
+R_ARGS=""
+if [ -n "$R_APP_ARGS" ]; then
+    R_ARGS="$R_APP_ARGS"
+fi
+
+# Attestation
+if [ -z "$ATTESTATION" ]; then
+    echo "[INFO] Attestation is disabled!"
+    ATTESTATION="false"
+elif [ "$ATTESTATION" = "true" ]; then
+  echo "[INFO] Attestation is enabled!"
+  # Build ATTESTATION_COMMAND
+  if [ -z "$ATTESTATION_URL" ]; then
+    echo "[ERROR] Attestation is enabled, but ATTESTATION_URL is empty!"
+    echo "[INFO] PPML Application Exit!"
+    exit 1
+  fi
+  if [ -z "$ATTESTATION_ID" ]; then
+    echo "[ERROR] Attestation is enabled, but ATTESTATION_ID is empty!"
+    echo "[INFO] PPML Application Exit!"
+    exit 1
+  fi
+  if [ -z "$ATTESTATION_KEY" ]; then
+    echo "[ERROR] Attestation is enabled, but ATTESTATION_KEY is empty!"
+    echo "[INFO] PPML Application Exit!"
+    exit 1
+  fi
+  ATTESTATION_COMMAND="/opt/jdk8/bin/java -Xmx1g -cp $SPARK_CLASSPATH:$BIGDL_HOME/jars/* com.intel.analytics.bigdl.ppml.attestation.AttestationCLI -u ${ATTESTATION_URL} -i ${ATTESTATION_ID}  -k ${ATTESTATION_KEY}"
+fi
+
+
+if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then
+    pyv="$(python -V 2>&1)"
+    export PYTHON_VERSION="${pyv:7}"
+    export PYSPARK_PYTHON="python"
+    export PYSPARK_DRIVER_PYTHON="python"
+elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
+    pyv3="$(python3 -V 2>&1)"
+    export PYTHON_VERSION="${pyv3:7}"
+    export PYSPARK_PYTHON="python3"
+    export PYSPARK_DRIVER_PYTHON="python3"
+fi
+
+case "$SPARK_K8S_CMD" in
+  driver)
+    CMD=(
+      "$SPARK_HOME/bin/spark-submit"
+      --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --deploy-mode client
+      "$@"
+    )
+    echo $SGX_ENABLED && \
+    echo $SGX_DRIVER_MEM_SIZE && \
+    echo $SGX_DRIVER_JVM_MEM_SIZE && \
+    echo $SGX_EXECUTOR_MEM_SIZE && \
+    echo $SGX_EXECUTOR_JVM_MEM_SIZE && \
+    echo $SGX_LOG_LEVEL && \
+    echo $SPARK_DRIVER_MEMORY && \
+    unset PYTHONHOME && \
+    unset PYTHONPATH && \
+    if [ "$SGX_ENABLED" == "false" ]; then
+        $SPARK_HOME/bin/spark-submit --conf spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS --deploy-mode client "$@"
+    elif [ "$SGX_ENABLED" == "true" ]; then
+	export driverExtraClassPath=`cat /opt/spark/conf/spark.properties | grep -P -o "(?<=spark.driver.extraClassPath=).*"` && \
+        echo $driverExtraClassPath && \
+        export SGX_MEM_SIZE=$SGX_DRIVER_MEM_SIZE && \
+        export spark_commnd="/opt/jdk8/bin/java -Dlog4j.configurationFile=/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/log4j2.xml -Xms1G -Xmx$SGX_DRIVER_JVM_MEM_SIZE -cp "$SPARK_CLASSPATH:$driverExtraClassPath" org.apache.spark.deploy.SparkSubmit --conf spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS --deploy-mode client "$@"" && \
+        if [ "$ATTESTATION" = "true" ]; then
+          spark_commnd=$ATTESTATION_COMMAND" && "$spark_commnd
+        fi
+        echo $spark_commnd && \
+        gramine-argv-serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && export _SPARK_AUTH_SECRET=$_SPARK_AUTH_SECRET && $spark_commnd" > /ppml/trusted-big-data-ml/secured_argvs && \
+        ./init.sh && \
+        gramine-sgx bash  1>&2
+    fi
+    ;;
+  driver-py)
+    CMD=(
+      "$SPARK_HOME/bin/spark-submit"
+      --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --deploy-mode client
+      "$@" $PYSPARK_PRIMARY $PYSPARK_ARGS
+    )
+    ;;
+    driver-r)
+    CMD=(
+      "$SPARK_HOME/bin/spark-submit"
+      --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
+      --deploy-mode client
+      "$@" $R_PRIMARY $R_ARGS
+    )
+    ;;
+    executor)
+    echo $SGX_ENABLED && \
+    echo $SGX_DRIVER_MEM_SIZE && \
+    echo $SGX_DRIVER_JVM_MEM_SIZE && \
+    echo $SGX_EXECUTOR_MEM_SIZE && \
+    echo $SGX_EXECUTOR_JVM_MEM_SIZE && \
+    echo $SGX_LOG_LEVEL && \
+    echo $SPARK_EXECUTOR_MEMORY && \
+    unset PYTHONHOME && \
+    unset PYTHONPATH && \
+    if [ "$SGX_ENABLED" == "false" ]; then
+      /opt/jdk8/bin/java \
+        -Xms$SPARK_EXECUTOR_MEMORY \
+        -Xmx$SPARK_EXECUTOR_MEMORY \
+        "${SPARK_EXECUTOR_JAVA_OPTS[@]}" \
+        -cp "$SPARK_CLASSPATH" \
+        org.apache.spark.executor.CoarseGrainedExecutorBackend \
+        --driver-url $SPARK_DRIVER_URL \
+        --executor-id $SPARK_EXECUTOR_ID \
+        --cores $SPARK_EXECUTOR_CORES \
+        --app-id $SPARK_APPLICATION_ID \
+        --hostname $SPARK_EXECUTOR_POD_IP \
+        --resourceProfileId $SPARK_RESOURCE_PROFILE_ID
+    elif [ "$SGX_ENABLED" == "true" ]; then
+      export SGX_MEM_SIZE=$SGX_EXECUTOR_MEM_SIZE && \
+      export spark_commnd="/opt/jdk8/bin/java -Dlog4j.configurationFile=/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/log4j2.xml -Xms1G -Xmx$SGX_EXECUTOR_JVM_MEM_SIZE "${SPARK_EXECUTOR_JAVA_OPTS[@]}" -cp "$SPARK_CLASSPATH" org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP --resourceProfileId $SPARK_RESOURCE_PROFILE_ID" && \
+      if [ "$ATTESTATION" = "true" ]; then
+        spark_commnd=$ATTESTATION_COMMAND" && "$spark_commnd
+      fi
+      echo $spark_commnd && \
+      gramine-argv-serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && export _SPARK_AUTH_SECRET=$_SPARK_AUTH_SECRET && $spark_commnd" > /ppml/trusted-big-data-ml/secured_argvs && \
+      ./init.sh && \
+      gramine-sgx bash  1>&2
+    fi
+    ;;
+
+  *)
+    echo "Unknown command: $SPARK_K8S_CMD" 1>&2
+    exit 1
+esac
+
+# Execute the container CMD under tini for better hygiene
+#exec /usr/bin/tini -s -- "${CMD[@]}"