tdx/docker/client-image (#5639)

ppml/tdx/docker/client-image/Dockerfile

@@ -0,0 +1,7 @@
+FROM intelanalytics/bigdl-k8s
+
+ADD ./spark-submit-with-ppml-tdx-local.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-local.sh
+ADD ./spark-submit-with-ppml-tdx-k8s.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-k8s.sh
+ADD ./pod-template.yaml /opt/spark/work-dir/ppml-tdx/pod-template.yaml
+ADD ./tdx-client.yaml /opt/spark/work-dir/ppml-tdx/tdx-client.yaml
+ADD ./runtimeclass_kata.yaml /opt/spark/work-dir/ppml-tdx/runtimeclass_kata.yaml

ppml/tdx/docker/client-image/README.md

@@ -0,0 +1,8 @@
+## Build bigdl-tdx image
+```bash
+docker build \
+  --build-arg http_proxy=.. \
+  --build-arg https_proxy=.. \
+　--build-arg no_proxy=.. \
+  --rm --no-cache -t intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT .
+```

ppml/tdx/docker/client-image/pod-template.yaml

@@ -0,0 +1,17 @@
+# example pod template for driver/executor
+apiVersion: v1
+kind: Pod
+spec:
+  # runtimeClassName: default is kata-cc
+  runtimeClassName: kata-cc
+  containers:
+  - name: spark-executor
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/config
+  volumes:
+  - name: kubeconf
+    hostPath:
+      path: /root/.kube/config

ppml/tdx/docker/client-image/runtimeclass_kata.yaml

@@ -0,0 +1,11 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Define the "kata" runtime class
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+  name: kata-cc
+handler: kata

ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-k8s.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Check environment variables
+if [ -z "$SPARK_HOME" ]; then
+    echo "Please set SPARK_HOME environment variable"
+    exit 1
+fi
+
+if [ -z "$RUNTIME_K8S_SERVICE_ACCOUNT" ]; then
+    echo "Please set RUNTIME_K8S_SERVICE_ACCOUNT environment variable"
+    exit 1
+fi
+
+if [ -z $RUNTIME_K8S_SPARK_IMAGE ]; then
+    echo "Please set RUNTIME_K8S_SPARK_IMAGE environment variable"
+    exit 1
+fi
+
+default_config="--conf spark.kubernetes.authenticate.driver.serviceAccountName=$RUNTIME_K8S_SERVICE_ACCOUNT \
+        --conf spark.kubernetes.container.image=$RUNTIME_K8S_SPARK_IMAGE \
+        --conf spark.kubernetes.executor.deleteOnTermination=false \
+        --conf spark.python.use.daemon=false \
+        --conf spark.python.worker.reuse=false"
+
+if [ $secure_password ]; then
+   if [ -z $KEYS_PATH ]; then
+       echo "Please set $KEYS_PATH environment variable"
+       exit 1
+   fi
+   SSL="--conf spark.authenticate=true \
+        --conf spark.authenticate.secret=$secure_password \
+        --conf spark.kubernetes.executor.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \
+        --conf spark.kubernetes.driver.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \
+        --conf spark.authenticate.enableSaslEncryption=true \
+        --conf spark.network.crypto.enabled=true \
+        --conf spark.network.crypto.keyLength=128 \
+        --conf spark.network.crypto.keyFactoryAlgorithm=PBKDF2WithHmacSHA1 \
+        --conf spark.io.encryption.enabled=true \
+        --conf spark.io.encryption.keySizeBits=128 \
+        --conf spark.io.encryption.keygen.algorithm=HmacSHA1 \
+        --conf spark.ssl.enabled=true \
+        --conf spark.ssl.port=8043 \
+        --conf spark.ssl.keyPassword=$secure_password \
+        --conf spark.ssl.keyStore=$KEYS_PATH/keystore.jks \
+        --conf spark.ssl.keyStorePassword=$secure_password \
+        --conf spark.ssl.keyStoreType=JKS \
+        --conf spark.ssl.trustStore=$KEYS_PATH/keystore.jks \
+        --conf spark.ssl.trustStorePassword=$secure_password \
+        --conf spark.ssl.trustStoreType=JKS"
+else
+   SSL=""
+fi
+
+set -x
+
+spark_submit_command="${SPARK_HOME}/bin/spark-submit \
+        $default_config \
+        $SSL \
+        $*"
+
+echo "spark_submit_command $spark_submit_command"
+bash -c "$spark_submit_command"

ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-local.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Check environment variables
+if [ -z "${BIGDL_HOME}" ]; then
+    echo "Please set BIGDL_HOME environment variable"
+    exit 1
+fi
+
+if [ -z "${SPARK_HOME}" ]; then
+    echo "Please set SPARK_HOME environment variable"
+    exit 1
+fi
+
+#setup paths
+export BIGDL_JAR_NAME=${BIGDL_HOME}/jars/*
+export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf
+
+# Check files
+if [ ! -f ${BIGDL_CONF} ]; then
+    echo "Cannot find ${BIGDL_CONF}"
+    exit 1
+fi
+
+if [ ! -f $BIGDL_JAR ]; then
+    echo "Cannot find $BIGDL_JAR"
+    exit 1
+fi
+
+
+${SPARK_HOME}/bin/spark-submit \
+  --properties-file ${BIGDL_CONF} \
+  --conf spark.driver.extraClassPath=${BIGDL_JAR} \
+  --conf spark.executor.extraClassPath=${BIGDL_JAR} \
+  $*

ppml/tdx/docker/client-image/tdx-client.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: bigdl-tdx-client
+spec:
+  runtimeClassName: kata-cc
+  containers:
+  - name: spark-local-k8s-client-kata-cc
+    image: intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT
+    imagePullPolicy: Always
+    securityContext:
+       privileged: true
+    resources:
+      limits:
+        cpu: "2"
+        memory: "2G"
+      requests:
+        cpu: "2"
+        memory: "2G"
+    command:
+    - sh
+    - "-c"
+    - |
+        sleep 10000
+    volumeMounts:
+    - name: kubeconfig
+      mountPath: /root/.kube/config
+    - name: nfs-pvc
+      mountPath: "/ppml/trusted-big-data-ml/work"
+    env:
+    - name: RUNTIME_SPARK_MASTER
+      value: "k8s://https://x.x.x.x:6443"
+    - name: RUNTIME_K8S_SERVICE_ACCOUNT
+      value: "spark"
+    - name: RUNTIME_K8S_SPARK_IMAGE
+      value: "intelanalytics/bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT"
+    - name: RUNTIME_DRIVER_HOST
+      value: "x.x.x.x"
+    - name: RUNTIME_DRIVER_PORT
+      value: "54321"
+    - name: RUNTIME_EXECUTOR_INSTANCES
+      value: "1"
+    - name: RUNTIME_EXECUTOR_CORES
+      value: "16"
+    - name: RUNTIME_EXECUTOR_MEMORY
+      value: "32g"
+    - name: RUNTIME_TOTAL_EXECUTOR_CORES
+      value: "16"
+    - name: RUNTIME_DRIVER_CORES
+      value: "16"
+    - name: RUNTIME_DRIVER_MEMORY
+      value: "32g"
+    - name: LOCAL_IP
+      value: "x.x.x.x"
+    - name: http_proxy
+      value: http://..
+    - name: https_proxy
+      value: http://..
+    - name: JAVA_HOME
+      value: /opt/jdk
+    - name: SPARK_HOME
+      value: /opt/spark
+  volumes:
+  - name: kubeconfig
+    hostPath:
+      path: /root/.kube/config
+  - name: nfs-pvc
+    persistentVolumeClaim:
+        claimName: nfsvolumeclaim