From 33a06581ecff77c3e53b90536af2fb7db705097b Mon Sep 17 00:00:00 2001 From: Jiaming Date: Tue, 2 Aug 2022 11:21:57 +0800 Subject: [PATCH 1/7] add assembly --- python/ppml/scripts/start-fl-server.py | 1 + scala/ppml/pom.xml | 14 ++++++++++++ scala/ppml/src/assembly/ppml-assembly.xml | 26 +++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 scala/ppml/src/assembly/ppml-assembly.xml diff --git a/python/ppml/scripts/start-fl-server.py b/python/ppml/scripts/start-fl-server.py index c1c399f335f..529c5623543 100644 --- a/python/ppml/scripts/start-fl-server.py +++ b/python/ppml/scripts/start-fl-server.py @@ -51,3 +51,4 @@ fl_server.start() fl_server.wait_for_termination() + diff --git a/scala/ppml/pom.xml b/scala/ppml/pom.xml index 61c22b44646..8e0ad945c37 100644 --- a/scala/ppml/pom.xml +++ b/scala/ppml/pom.xml @@ -414,6 +414,20 @@ single + + assembly + false + package + + single + + + + ${project.basedir}/src/assembly/ppml-assembly.xml + + + + diff --git a/scala/ppml/src/assembly/ppml-assembly.xml b/scala/ppml/src/assembly/ppml-assembly.xml new file mode 100644 index 00000000000..a49dc80b54f --- /dev/null +++ b/scala/ppml/src/assembly/ppml-assembly.xml @@ -0,0 +1,26 @@ + + ppml-assembly + + dir + zip + + false + + + /fl + ../${project.basedir}/../python/ppml/scripts + + + + + /fl/lib + ../${project.basedir}/ppml/target + + bigdl-ppml-*-jar-with-dependencies.jar + bigdl-ppml-*-python-api.zip + + + + From 83801427e31aa0acffa46148809d2ebb24560925 Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Fri, 19 Aug 2022 13:35:04 +0800 Subject: [PATCH 2/7] Enable FL Sever in SGX --- python/ppml/scripts/start-fl-server.py | 1 - scala/ppml/pom.xml | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/python/ppml/scripts/start-fl-server.py b/python/ppml/scripts/start-fl-server.py index 529c5623543..c1c399f335f 100644 --- a/python/ppml/scripts/start-fl-server.py +++ b/python/ppml/scripts/start-fl-server.py @@ -51,4 +51,3 @@ fl_server.start() fl_server.wait_for_termination() - diff --git a/scala/ppml/pom.xml b/scala/ppml/pom.xml index 8e0ad945c37..61c22b44646 100644 --- a/scala/ppml/pom.xml +++ b/scala/ppml/pom.xml @@ -414,20 +414,6 @@ single - - assembly - false - package - - single - - - - ${project.basedir}/src/assembly/ppml-assembly.xml - - - - From 9817c2ad7c78eadb81d73b218ff123a900d87dfc Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Fri, 19 Aug 2022 13:38:06 +0800 Subject: [PATCH 3/7] Add start-fl-server.py to Docker Image --- .../python/docker-graphene/Dockerfile | 1 + .../examples/start-fl-server.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile index e215c8054e1..7cf37e1294d 100644 --- a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile +++ b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile @@ -284,6 +284,7 @@ ADD ./test-suites /ppml/trusted-big-data-ml/work/test-suites ADD ./_dill.py.patch ./_dill.py.patch ADD ./python-uuid.patch ./python-uuid.patch ADD ./python-pslinux.patch ./python-pslinux.patch +ADD ../../../../python/ppml/scripts/start-fl-server.py /ppml/trusted-big-data-ml/work/examples RUN zip -u ${BIGDL_HOME}/jars/xgboost4j_2.12-1.1.2.jar ./tracker.py && \ patch /usr/local/lib/python3.7/dist-packages/dill/_dill.py ./_dill.py.patch && \ diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py b/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py new file mode 100644 index 00000000000..c72ac47ec34 --- /dev/null +++ b/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py @@ -0,0 +1,62 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import os +import fnmatch +import getopt + +for files in os.listdir('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/'): + if fnmatch.fnmatch(files, 'bigdl-ppml-*-python-api.zip'): + sys.path.append('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/' + files) + sys.path.append('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/' + files + '/bigdl/ppml/fl/nn/generated') + +if '/usr/lib/python3.6' in sys.path: + sys.path.remove('/usr/lib/python3.6') +if '/usr/lib/python3.6/lib-dynload' in sys.path: + sys.path.remove('/usr/lib/python3.6/lib-dynload') +if '/usr/local/lib/python3.6/dist-packages' in sys.path: + sys.path.remove('/usr/local/lib/python3.6/dist-packages') +if '/usr/lib/python3/dist-packages' in sys.path: + sys.path.remove('/usr/lib/python3/dist-packages') + +from bigdl.ppml.fl.nn.fl_server import FLServer + +if __name__ == '__main__': + + client_num = 2 + port = 8980 + + try: + opts, args = getopt.getopt(sys.argv[1:], "hc:p:", ["client-num=", "port="]) + except getopt.GetoptError: + print("start_fl_server.py -c -p ") + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print("start_fl_server.py -c -p ") + elif opt in ("-c", "--client-num"): + client_num = arg + elif opt in ("-p", "--port"): + port = arg + + fl_server = FLServer(client_num) + fl_server.set_port(port) + fl_server.build() + fl_server.start() + + fl_server.wait_for_termination() From 541a1e0b5fe02e2a7e5adbfec8d670536ab10ff9 Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Fri, 19 Aug 2022 17:07:40 +0800 Subject: [PATCH 4/7] rm ppml-assebly.xml --- scala/ppml/src/assembly/ppml-assembly.xml | 26 ----------------------- 1 file changed, 26 deletions(-) delete mode 100644 scala/ppml/src/assembly/ppml-assembly.xml diff --git a/scala/ppml/src/assembly/ppml-assembly.xml b/scala/ppml/src/assembly/ppml-assembly.xml deleted file mode 100644 index a49dc80b54f..00000000000 --- a/scala/ppml/src/assembly/ppml-assembly.xml +++ /dev/null @@ -1,26 +0,0 @@ - - ppml-assembly - - dir - zip - - false - - - /fl - ../${project.basedir}/../python/ppml/scripts - - - - - /fl/lib - ../${project.basedir}/ppml/target - - bigdl-ppml-*-jar-with-dependencies.jar - bigdl-ppml-*-python-api.zip - - - - From 7b27e362bc72d0772d8d4212dcad6d4838eeee12 Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Fri, 19 Aug 2022 17:15:35 +0800 Subject: [PATCH 5/7] Add start-python-fl-server-sgx.sh to Docker Image --- .../examples/start-fl-server.py | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py b/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py deleted file mode 100644 index c72ac47ec34..00000000000 --- a/ppml/trusted-big-data-ml/python/docker-graphene/examples/start-fl-server.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -import os -import fnmatch -import getopt - -for files in os.listdir('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/'): - if fnmatch.fnmatch(files, 'bigdl-ppml-*-python-api.zip'): - sys.path.append('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/' + files) - sys.path.append('/ppml/trusted-big-data-ml/work/bigdl-2.1.0-SNAPSHOT/python/' + files + '/bigdl/ppml/fl/nn/generated') - -if '/usr/lib/python3.6' in sys.path: - sys.path.remove('/usr/lib/python3.6') -if '/usr/lib/python3.6/lib-dynload' in sys.path: - sys.path.remove('/usr/lib/python3.6/lib-dynload') -if '/usr/local/lib/python3.6/dist-packages' in sys.path: - sys.path.remove('/usr/local/lib/python3.6/dist-packages') -if '/usr/lib/python3/dist-packages' in sys.path: - sys.path.remove('/usr/lib/python3/dist-packages') - -from bigdl.ppml.fl.nn.fl_server import FLServer - -if __name__ == '__main__': - - client_num = 2 - port = 8980 - - try: - opts, args = getopt.getopt(sys.argv[1:], "hc:p:", ["client-num=", "port="]) - except getopt.GetoptError: - print("start_fl_server.py -c -p ") - sys.exit(2) - - for opt, arg in opts: - if opt == '-h': - print("start_fl_server.py -c -p ") - elif opt in ("-c", "--client-num"): - client_num = arg - elif opt in ("-p", "--port"): - port = arg - - fl_server = FLServer(client_num) - fl_server.set_port(port) - fl_server.build() - fl_server.start() - - fl_server.wait_for_termination() From 35600168fc9b41171bc34ab7497b60fd97072e14 Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Mon, 22 Aug 2022 13:57:20 +0800 Subject: [PATCH 6/7] Put start-fl-server.py into assembly --- ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile index 7cf37e1294d..e215c8054e1 100644 --- a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile +++ b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile @@ -284,7 +284,6 @@ ADD ./test-suites /ppml/trusted-big-data-ml/work/test-suites ADD ./_dill.py.patch ./_dill.py.patch ADD ./python-uuid.patch ./python-uuid.patch ADD ./python-pslinux.patch ./python-pslinux.patch -ADD ../../../../python/ppml/scripts/start-fl-server.py /ppml/trusted-big-data-ml/work/examples RUN zip -u ${BIGDL_HOME}/jars/xgboost4j_2.12-1.1.2.jar ./tracker.py && \ patch /usr/local/lib/python3.7/dist-packages/dill/_dill.py ./_dill.py.patch && \ From d266cb9fbb6e9c85c773bd5b8f176d261de9a747 Mon Sep 17 00:00:00 2001 From: Jin Hanyu <476099001@qq.com> Date: Sat, 27 Aug 2022 17:21:04 +0800 Subject: [PATCH 7/7] Change the Dockerfile for better --- .../python/docker-graphene/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile index e215c8054e1..b573953d68b 100644 --- a/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile +++ b/ppml/trusted-big-data-ml/python/docker-graphene/Dockerfile @@ -245,10 +245,6 @@ RUN apt install software-properties-common -y && \ apt-get install -y python3-minimal build-essential python3.7-distutils python3-apt python3.7 python3-setuptools python3-dev python3-pip libpython3.7 && \ rm /usr/bin/python3 && \ ln -s /usr/bin/python3.7 /usr/bin/python3 && \ - rm -rf /usr/lib/python3/dist-packages/protobuf-3.6.1.egg-info && \ - rm -rf /usr/lib/python3/dist-packages/google/protobuf && \ - ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 && \ - pip install pyyaml && \ pip3 install --upgrade pip && \ pip install setuptools==58.4.0 && \ pip install --no-cache-dir numpy scipy && \ @@ -266,7 +262,11 @@ RUN apt install software-properties-common -y && \ pip install --no-cache-dir psutil && \ pip install --no-cache-dir dill==0.3.4 && \ python3 -m pip install toml==0.10.2 click jinja2 && \ - python3 -m ipykernel.kernelspec + python3 -m ipykernel.kernelspec && \ + rm -rf /usr/lib/python3/dist-packages/protobuf-3.6.1.egg-info && \ + rm -rf /usr/lib/python3/dist-packages/google/protobuf && \ + ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 && \ + pip install --no-cache-dir pyyaml ADD ./bash.manifest.template /ppml/trusted-big-data-ml/bash.manifest.template ADD ./Makefile /ppml/trusted-big-data-ml/Makefile