tdx/docker/client-image

ppml/tdx/docker/client-image/Dockerfile

@@ -0,0 +1,8 @@
+FROM intelanalytics/bigdl-k8s
+
+ADD ./spark-submit-with-ppml-tdx-local.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-local.sh
+ADD ./spark-submit-with-ppml-tdx-k8s.sh /opt/spark/work-dir/ppml-tdx/spark-submit-with-ppml-tdx-k8s.sh
+ADD ./pod-template.yaml /opt/spark/work-dir/ppml-tdx/pod-template.yaml
+ADD ./tdx-client.yaml /opt/spark/work-dir/ppml-tdx/tdx-client.yaml
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]

ppml/tdx/docker/client-image/README.md

@@ -0,0 +1,8 @@
+## Build bigdl-tdx image
+```bash
+docker build \
+  --build-arg http_proxy=.. \
+  --build-arg https_proxy=.. \
+　--build-arg no_proxy=.. \
+  --rm --no-cache -t bigdl-tdx-client-spark-3.1.2:2.1.0-SNAPSHOT .
+```

ppml/tdx/docker/client-image/pod-template.yaml

@@ -0,0 +1,34 @@
+# example pod template for driver/executor
+apiVersion: v1
+kind: Pod
+spec:
+  # runtimeClassName: default is kata-cc
+  runtimeClassName: kata-cc
+  containers:
+  - name: spark-executor
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - name: data
+      mountPath: /ppml/trusted-big-data-ml/work/data
+    - name: kubeconf
+      mountPath: /root/.kube/config
+    - name: tpch-1g
+      mountPath: /TPCH-1G
+    volumeDevices:
+    - devicePath: "/dev/sdd"
+      name: datapath
+    command: ["/opt/entrypoint.sh"]
+  volumes:
+  - name: data
+    hostPath:
+      path: /home/data
+  - name: tpch-1g
+    hostPath:
+      path: /home/1G
+  - name: kubeconf
+    hostPath:
+      path: /root/.kube/config
+  - name: datapath
+    persistentVolumeClaim:
+      claimName: busybox-lvm-block-pvc-pre-2

ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-k8s.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Check environment variables
+if [ -z "$SPARK_HOME" ]; then
+    echo "Please set SPARK_HOME environment variable"
+    exit 1
+fi
+
+if [ -z "$RUNTIME_K8S_SERVICE_ACCOUNT" ]; then
+    echo "Please set BIGDL_HOME environment variable"
+    exit 1
+fi
+
+if [ -z $RUNTIME_K8S_SPARK_IMAGE ]; then
+    echo "Please set BIGDL_HOME environment variable"
+    exit 1
+fi
+
+default_config="--conf spark.kubernetes.authenticate.driver.serviceAccountName=$RUNTIME_K8S_SERVICE_ACCOUNT \
+        --conf spark.kubernetes.container.image=$RUNTIME_K8S_SPARK_IMAGE \
+        --conf spark.kubernetes.executor.deleteOnTermination=false \
+        --conf spark.network.timeout=10000000 \
+        --conf spark.executor.heartbeatInterval=10000000 \
+        --conf spark.python.use.daemon=false \
+        --conf spark.python.worker.reuse=false"
+
+if [ $secure_password ]; then
+   SSL="--conf spark.authenticate=true \
+        --conf spark.authenticate.secret=$secure_password \
+        --conf spark.kubernetes.executor.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \
+        --conf spark.kubernetes.driver.secretKeyRef.SPARK_AUTHENTICATE_SECRET="spark-secret:secret" \
+        --conf spark.authenticate.enableSaslEncryption=true \
+        --conf spark.network.crypto.enabled=true \
+        --conf spark.network.crypto.keyLength=128 \
+        --conf spark.network.crypto.keyFactoryAlgorithm=PBKDF2WithHmacSHA1 \
+        --conf spark.io.encryption.enabled=true \
+        --conf spark.io.encryption.keySizeBits=128 \
+        --conf spark.io.encryption.keygen.algorithm=HmacSHA1 \
+        --conf spark.ssl.enabled=true \
+        --conf spark.ssl.port=8043 \
+        --conf spark.ssl.keyPassword=$secure_password \
+        --conf spark.ssl.keyStore=/opt/spark/work-dir/keys/keystore.jks \
+        --conf spark.ssl.keyStorePassword=$secure_password \
+        --conf spark.ssl.keyStoreType=JKS \
+        --conf spark.ssl.trustStore=/opt/spark/work-dir/keys/keystore.jks \
+        --conf spark.ssl.trustStorePassword=$secure_password \
+        --conf spark.ssl.trustStoreType=JKS"
+else
+   SSL=""
+fi
+
+set -x
+
+spark_submit_command="${SPARK_HOME}/bin/spark-submit \
+        $default_config \
+        $SSL \
+        $*"
+
+echo "spark_submit_command $spark_submit_command"
+bash -c "$spark_submit_command"

ppml/tdx/docker/client-image/spark-submit-with-ppml-tdx-local.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Check environment variables
+if [ -z "${BIGDL_HOME}" ]; then
+    echo "Please set BIGDL_HOME environment variable"
+    exit 1
+fi
+
+if [ -z "${SPARK_HOME}" ]; then
+    echo "Please set SPARK_HOME environment variable"
+    exit 1
+fi
+
+#setup paths
+export BIGDL_JAR_NAME=${BIGDL_HOME}/jars/*
+export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf
+
+# Check files
+if [ ! -f ${BIGDL_CONF} ]; then
+    echo "Cannot find ${BIGDL_CONF}"
+    exit 1
+fi
+
+if [ ! -f $BIGDL_JAR ]; then
+    echo "Cannot find $BIGDL_JAR"
+    exit 1
+fi
+
+
+${SPARK_HOME}/bin/spark-submit \
+  --properties-file ${BIGDL_CONF} \
+  --conf spark.driver.extraClassPath=${BIGDL_JAR} \
+  --conf spark.executor.extraClassPath=${BIGDL_JAR} \
+  $*

ppml/tdx/docker/client-image/tdx-client.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: Pod
+
+metadata:
+  # name: default is bigdl-tdx-client
+  name: bigdl-tdx-client
+
+spec:
+  # hostNetwork: true
+  # runtimeClassName: default is kata-cc
+  runtimeClassName: kata-cc
+
+  # default config to create container
+  containers:
+  - name: spark-local-k8s-client-kata-cc
+    image: intelanalytics/bigdl-tdx-client-spark-3.1.2:latest
+    imagePullPolicy: Always
+    securityContext:
+       privileged: true
+    resources:
+      limits:
+        cpu: "2"
+        memory: "2G"
+      requests:
+        cpu: "2"
+        memory: "2G"
+    command:
+    - sh
+    - "-c"
+    - |
+        mkdir -p /run/data &&
+        mount /dev/sdd /run/data
+        sleep 10000
+
+    volumeMounts:
+    - name: kubeconfig
+      mountPath: /root/.kube/config
+    - name: sda-data
+      mountPath: /ppml/trusted-big-data-ml/work/data
+    - name: sda-tpch-1g
+      mountPath: /TPCH-1G
+    volumeDevices:
+    - devicePath: "/dev/sdd"
+      name: data
+    env:
+    - name: RUNTIME_SPARK_MASTER
+      value: "k8s://https://x.x.x.x:6443"
+    - name: RUNTIME_K8S_SERVICE_ACCOUNT
+      value: "spark"
+    - name: RUNTIME_K8S_SPARK_IMAGE
+      value: "intelanalytics/bigdl-tdx-client-spark-3.1.2:latest"
+    - name: RUNTIME_DRIVER_HOST
+      value: "x.x.x.x"
+    - name: RUNTIME_DRIVER_PORT
+      value: "54321"
+    - name: RUNTIME_EXECUTOR_INSTANCES
+      value: "1"
+    - name: RUNTIME_EXECUTOR_CORES
+      value: "16"
+    - name: RUNTIME_EXECUTOR_MEMORY
+      value: "32g"
+    - name: RUNTIME_TOTAL_EXECUTOR_CORES
+      value: "16"
+    - name: RUNTIME_DRIVER_CORES
+      value: "16"
+    - name: RUNTIME_DRIVER_MEMORY
+      value: "32g"
+    - name: LOCAL_IP
+      value: "x.x.x.x"
+    - name: http_proxy
+      value: http://..
+    - name: https_proxy
+      value: http://..
+    - name: JAVA_HOME
+      value: /opt/jdk
+    - name: SPARK_HOME
+      value: /opt/spark
+  volumes:
+  - name: kubeconfig
+    hostPath:
+      path: /root/.kube/config
+  - name: nvme-data
+    hostPath:
+      path: /mnt/nvme0n1/data
+  - name: nvme-tpch-1g
+    hostPath:
+      path: /mnt/nvme0n1/TPCH-1G
+  - name: sda-tpch-1g
+    hostPath:
+      path: /home/1G
+  - name: sda-data
+    hostPath:
+      path: /home/data
+  - name: data
+    persistentVolumeClaim:
+      claimName: busybox-lvm-block-pvc-pre-1