update graphene ppml examples to main (intel-analytics#5804)

* update graphene ppml examples * update graphene ppml examples Co-authored-by: Jin Hanyu <[email protected]>
ForJadeForest · Sep 20, 2022 · 7967e39 · 7967e39
1 parent 441442d
commit 7967e39
Show file tree

Hide file tree

Showing 11 changed files with 211 additions and 75 deletions.
diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-pyspark-sql-examples.sh b/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-pyspark-sql-examples.sh
@@ -2,11 +2,13 @@
 status_2_k8s_pyspark_sql_basic=1
 status_3_k8s_pyspark_sql_e2e=1
 
-SPARK_LOCAL_IP=192.168.0.112
+SPARK_LOCAL_IP=$LOCAL_IP
 DB_PATH=/ppml/trusted-big-data-ml/work/data/sqlite_example/100w.db
 
 if [ $status_2_k8s_pyspark_sql_basic -ne 0 ]; then
-SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
   export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && \
   /opt/jdk8/bin/java \
     -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
@@ -36,12 +38,16 @@ SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
     --conf spark.kubernetes.sgx.mem=32g \
     --conf spark.kubernetes.sgx.jvm.mem=16g \
     --verbose \
-    /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/src/main/python/sql/basic.py" 2>&1 > k8s-pyspark-sql-basic-sgx.log
+    /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/src/main/python/sql/basic.py" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee k8s-pyspark-sql-basic-sgx.log
 fi
 status_2_k8s_pyspark_sql_basic=$(echo $?)
 
 if [ $status_3_k8s_pyspark_sql_e2e -ne 0 ]; then
-SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
   export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && \
   /opt/jdk8/bin/java \
     -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' -Xmx10g \
@@ -70,7 +76,9 @@ SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
     --conf spark.kubernetes.sgx.jvm.mem=16g \
     --verbose \
     local:///ppml/trusted-big-data-ml/work/examples/customer_profile.py \
-    --db_path $DB_PATH" 2>&1 > k8s-pyspark-sql-e2e-100w-sgx.log
+    --db_path $DB_PATH" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee k8s-pyspark-sql-e2e-100w-sgx.log
 fi
 status_3_k8s_pyspark_sql_e2e=$(echo $?)
 

diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-spark-pi.sh b/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-spark-pi.sh
@@ -1,9 +1,11 @@
 #!/bin/bash
 status_1_k8s_spark_pi=1
-SPARK_LOCAL_IP=192.168.0.112
+SPARK_LOCAL_IP=$LOCAL_IP
 
 if [ $status_1_k8s_spark_pi -ne 0 ]; then
-  SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
+  cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
   export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && \
   /opt/jdk8/bin/java \
     -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
@@ -34,7 +36,9 @@ if [ $status_1_k8s_spark_pi -ne 0 ]; then
     --conf spark.kubernetes.sgx.jvm.mem=16g \
     --class org.apache.spark.examples.SparkPi \
     --verbose \
-    local:///ppml/trusted-big-data-ml/work/spark-3.1.2/examples/jars/spark-examples_2.12-3.1.2.jar" 2>&1 > k8s-spark-pi-sgx.log
+    local:///ppml/trusted-big-data-ml/work/spark-3.1.2/examples/jars/spark-examples_2.12-3.1.2.jar" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee k8s-spark-pi-sgx.log
 fi
 status_1_k8s_spark_pi=$(echo $?)
 

diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-spark-sql-examples.sh b/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/k8s-spark-sql-examples.sh
@@ -2,11 +2,13 @@
 status_4_k8s_spark_sql_example=1
 status_5_k8s_spark_sql_e2e=1
 
-SPARK_LOCAL_IP=192.168.0.112
+SPARK_LOCAL_IP=$LOCAL_IP
 DB_PATH=/ppml/trusted-big-data-ml/work/data/sqlite_example/100w.db
 
 if [ $status_4_k8s_spark_sql_example -ne 0 ]; then
-SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
   export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && \
   /opt/jdk8/bin/java \
     -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
@@ -36,12 +38,16 @@ SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
     --conf spark.kubernetes.sgx.mem=32g \
     --conf spark.kubernetes.sgx.jvm.mem=16g \
     --class org.apache.spark.examples.sql.SparkSQLExample \
-    /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/jars/spark-examples_2.12-3.1.2.jar" 2>&1 > k8s-spark-sql-example-sgx.log
+    /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/jars/spark-examples_2.12-3.1.2.jar" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee k8s-spark-sql-example-sgx.log
 fi
 status_4_k8s_spark_sql_example=$(echo $?)
 
 if [ $status_5_k8s_spark_sql_e2e -ne 0 ]; then
-SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
   export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && \
   /opt/jdk8/bin/java \
     -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*:/ppml/trusted-big-data-ml/work/data/sqlite_example/spark-example-sql-e2e.jar' -Xmx10g \
@@ -70,7 +76,9 @@ SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && \
     --conf spark.kubernetes.sgx.jvm.mem=16g \
     --class test.SqlExample \
     /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/jars/spark-example-sql-e2e.jar \
-    $DB_PATH" 2>&1 > k8s-spark-sql-e2e-100w-sgx.log
+    $DB_PATH" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee k8s-spark-sql-e2e-100w-sgx.log
 fi
 status_5_k8s_spark_sql_e2e=$(echo $?)
 

diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/pyspark-e2e-example.sh b/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/pyspark-e2e-example.sh
@@ -1,12 +1,14 @@
 #!/bin/bash
 status_8_local_spark_customer_profile=1
-SPARK_LOCAL_IP=192.168.0.112
+SPARK_LOCAL_IP=$LOCAL_IP
 DB_PATH=/ppml/trusted-big-data-ml/work/data/sqlite_example/100w.db
 
 # attention to SPARK_LOCAL_IP env change into targeted ip
 if [ $status_8_local_spark_customer_profile -ne 0 ]; then
 echo "example.8 local spark, Custom profile"
-SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && /opt/jdk8/bin/java \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && export SPARK_LOCAL_IP=$SPARK_LOCAL_IP && /opt/jdk8/bin/java \
   -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
   -Xmx1g \
   org.apache.spark.deploy.SparkSubmit \
@@ -17,7 +19,9 @@ SGX=1 ./pal_loader bash -c "export TF_MKL_ALLOC_MAX_BYTES=10737418240 && export
     --conf spark.python.use.daemon=false \
     --conf spark.python.worker.reuse=false \
     /ppml/trusted-big-data-ml/work/examples/customer_profile.py \
-    --db_path $DB_PATH" 2>&1 > customer_profile-sgx.log
+    --db_path $DB_PATH" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee customer_profile-sgx.log
 status_8_local_spark_customer_profile=$(echo $?)
 fi
 

diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/pyspark-simple-examples.sh b/ppml/trusted-big-data-ml/python/docker-graphene/test-suites/pyspark-simple-examples.sh
@@ -4,28 +4,36 @@ status_4_local_spark_wordcount=1
 
 if [ $status_3_local_spark_pi -ne 0 ]; then
 echo "example.3 local spark, pi"
-SGX=1 ./pal_loader bash -c "/opt/jdk8/bin/java \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "/opt/jdk8/bin/java \
    -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
    -Xmx1g org.apache.spark.deploy.SparkSubmit \
    --master 'local[4]' \
    --conf spark.python.use.daemon=false \
    --conf spark.python.worker.reuse=false \
-   /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/src/main/python/pi.py" 2>&1 > test-pi-sgx.log && \
+   /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/src/main/python/pi.py" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee test-pi-sgx.log && \
    cat test-pi-sgx.log | egrep 'roughly'
 status_3_local_spark_pi=$(echo $?)
 fi
 
 
 if [ $status_4_local_spark_wordcount -ne 0 ]; then
 echo "example.4 local spark, test-wordcount"
-SGX=1 ./pal_loader bash -c "export PYSPARK_PYTHON=/usr/bin/python && /opt/jdk8/bin/java \
+cd /ppml/trusted-big-data-ml
+./clean.sh
+/graphene/Tools/argv_serializer bash -c "export PYSPARK_PYTHON=/usr/bin/python && /opt/jdk8/bin/java \
    -cp '/ppml/trusted-big-data-ml/work/spark-3.1.2/conf/:/ppml/trusted-big-data-ml/work/spark-3.1.2/jars/*' \
    -Xmx1g org.apache.spark.deploy.SparkSubmit \
    --master 'local[4]' \
    --conf spark.python.use.daemon=false \
    --conf spark.python.worker.reuse=false \
    /ppml/trusted-big-data-ml/work/spark-3.1.2/examples/src/main/python/wordcount.py \
-   /ppml/trusted-big-data-ml/work/examples/helloworld.py" 2>&1 > test-wordcount-sgx.log && \
+   /ppml/trusted-big-data-ml/work/examples/helloworld.py" > /ppml/trusted-big-data-ml/secured-argvs
+./init.sh
+SGX=1 ./pal_loader bash 2>&1 | tee test-wordcount-sgx.log && \
    cat test-wordcount-sgx.log | egrep 'print'
 status_4_local_spark_wordcount=$(echo $?)
 fi