diff --git a/ppml/tdx/docker/client-image/Dockerfile b/ppml/tdx/docker/client-image/Dockerfile new file mode 100644 index 00000000000..c3e777299c6 --- /dev/null +++ b/ppml/tdx/docker/client-image/Dockerfile @@ -0,0 +1,7 @@ +FROM intelanalytics/bigdl-k8s + +ADD ./spark-submit-with-ppml-tdx-local.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-local.sh +ADD ./spark-submit-with-ppml-tdx-k8s.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-k8s.sh +ADD ./pod-template.yaml /opt/spark/work-dir/ppml-tdx/pod-template.yaml +ADD ./tdx-client.yaml /opt/spark/work-dir/ppml-tdx/tdx-client.yaml +ADD ./runtimeclass_kata.yaml /opt/spark/work-dir/ppml-tdx/runtimeclass_kata.yaml diff --git a/ppml/tdx/docker/client-image/README.md b/ppml/tdx/docker/client-image/README.md new file mode 100644 index 00000000000..da400eb9869 --- /dev/null +++ b/ppml/tdx/docker/client-image/README.md @@ -0,0 +1,8 @@ +## Build bigdl-tdx image +```bash +docker build \ + --build-arg http_proxy=.. \ + --build-arg https_proxy=.. \ + --build-arg no_proxy=.. \ + --rm --no-cache -t intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT . +``` diff --git a/ppml/tdx/docker/client-image/pod-template.yaml b/ppml/tdx/docker/client-image/pod-template.yaml new file mode 100644 index 00000000000..50aaf3c24d8 --- /dev/null +++ b/ppml/tdx/docker/client-image/pod-template.yaml @@ -0,0 +1,17 @@ +# example pod template for driver/executor +apiVersion: v1 +kind: Pod +spec: + # runtimeClassName: default is kata-cc + runtimeClassName: kata-cc + containers: + - name: spark-executor + securityContext: + privileged: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/config + volumes: + - name: kubeconf + hostPath: + path: /root/.kube/config diff --git a/ppml/tdx/docker/client-image/runtimeclass_kata.yaml b/ppml/tdx/docker/client-image/runtimeclass_kata.yaml new file mode 100644 index 00000000000..865044a5668 --- /dev/null +++ b/ppml/tdx/docker/client-image/runtimeclass_kata.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Define the "kata" runtime class +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-cc +handler: kata diff --git a/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-k8s.sh b/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-k8s.sh new file mode 100644 index 00000000000..b2f94773019 --- /dev/null +++ b/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-k8s.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Check environment variables +if [ -z "$SPARK_HOME" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +if [ -z "$RUNTIME_K8S_SERVICE_ACCOUNT" ]; then + echo "Please set RUNTIME_K8S_SERVICE_ACCOUNT environment variable" + exit 1 +fi + +if [ -z $RUNTIME_K8S_SPARK_IMAGE ]; then + echo "Please set RUNTIME_K8S_SPARK_IMAGE environment variable" + exit 1 +fi + +default_config="--conf spark.kubernetes.authenticate.driver.serviceAccountName=$RUNTIME_K8S_SERVICE_ACCOUNT \ + --conf spark.kubernetes.container.image=$RUNTIME_K8S_SPARK_IMAGE \ + --conf spark.kubernetes.executor.deleteOnTermination=false \ + --conf spark.python.use.daemon=false \ + --conf spark.python.worker.reuse=false" + +if [ $secure_password ]; then + if [ -z $KEYS_PATH ]; then + echo "Please set $KEYS_PATH environment variable" + exit 1 + fi + SSL="--conf spark.authenticate=true \ + --conf spark.authenticate.secret=$secure_password \ + --conf spark.kubernetes.executor.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \ + --conf spark.kubernetes.driver.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \ + --conf spark.authenticate.enableSaslEncryption=true \ + --conf spark.network.crypto.enabled=true \ + --conf spark.network.crypto.keyLength=128 \ + --conf spark.network.crypto.keyFactoryAlgorithm=PBKDF2WithHmacSHA1 \ + --conf spark.io.encryption.enabled=true \ + --conf spark.io.encryption.keySizeBits=128 \ + --conf spark.io.encryption.keygen.algorithm=HmacSHA1 \ + --conf spark.ssl.enabled=true \ + --conf spark.ssl.port=8043 \ + --conf spark.ssl.keyPassword=$secure_password \ + --conf spark.ssl.keyStore=$KEYS_PATH/keystore.jks \ + --conf spark.ssl.keyStorePassword=$secure_password \ + --conf spark.ssl.keyStoreType=JKS \ + --conf spark.ssl.trustStore=$KEYS_PATH/keystore.jks \ + --conf spark.ssl.trustStorePassword=$secure_password \ + --conf spark.ssl.trustStoreType=JKS" +else + SSL="" +fi + +set -x + +spark_submit_command="${SPARK_HOME}/bin/spark-submit \ + $default_config \ + $SSL \ + $*" + +echo "spark_submit_command $spark_submit_command" +bash -c "$spark_submit_command" diff --git a/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-local.sh b/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-local.sh new file mode 100644 index 00000000000..ce51bdec356 --- /dev/null +++ b/ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-local.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Check environment variables +if [ -z "${BIGDL_HOME}" ]; then + echo "Please set BIGDL_HOME environment variable" + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +#setup paths +export BIGDL_JAR_NAME=${BIGDL_HOME}/jars/* +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +# Check files +if [ ! -f ${BIGDL_CONF} ]; then + echo "Cannot find ${BIGDL_CONF}" + exit 1 +fi + +if [ ! -f $BIGDL_JAR ]; then + echo "Cannot find $BIGDL_JAR" + exit 1 +fi + + +${SPARK_HOME}/bin/spark-submit \ + --properties-file ${BIGDL_CONF} \ + --conf spark.driver.extraClassPath=${BIGDL_JAR} \ + --conf spark.executor.extraClassPath=${BIGDL_JAR} \ + $* diff --git a/ppml/tdx/docker/client-image/tdx-client.yaml b/ppml/tdx/docker/client-image/tdx-client.yaml new file mode 100644 index 00000000000..ed3da88e2ff --- /dev/null +++ b/ppml/tdx/docker/client-image/tdx-client.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: Pod +metadata: + name: bigdl-tdx-client +spec: + runtimeClassName: kata-cc + containers: + - name: spark-local-k8s-client-kata-cc + image: intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT + imagePullPolicy: Always + securityContext: + privileged: true + resources: + limits: + cpu: "2" + memory: "2G" + requests: + cpu: "2" + memory: "2G" + command: + - sh + - "-c" + - | + sleep 10000 + volumeMounts: + - name: kubeconfig + mountPath: /root/.kube/config + - name: nfs-pvc + mountPath: "/ppml/trusted-big-data-ml/work" + env: + - name: RUNTIME_SPARK_MASTER + value: "k8s://https://x.x.x.x:6443" + - name: RUNTIME_K8S_SERVICE_ACCOUNT + value: "spark" + - name: RUNTIME_K8S_SPARK_IMAGE + value: "intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT" + - name: RUNTIME_DRIVER_HOST + value: "x.x.x.x" + - name: RUNTIME_DRIVER_PORT + value: "54321" + - name: RUNTIME_EXECUTOR_INSTANCES + value: "1" + - name: RUNTIME_EXECUTOR_CORES + value: "16" + - name: RUNTIME_EXECUTOR_MEMORY + value: "32g" + - name: RUNTIME_TOTAL_EXECUTOR_CORES + value: "16" + - name: RUNTIME_DRIVER_CORES + value: "16" + - name: RUNTIME_DRIVER_MEMORY + value: "32g" + - name: LOCAL_IP + value: "x.x.x.x" + - name: http_proxy + value: http://.. + - name: https_proxy + value: http://.. + - name: JAVA_HOME + value: /opt/jdk + - name: SPARK_HOME + value: /opt/spark + volumes: + - name: kubeconfig + hostPath: + path: /root/.kube/config + - name: nfs-pvc + persistentVolumeClaim: + claimName: nfsvolumeclaim