From b5bbbe400234fc113eb8d6f539b5f1ceb16939c7 Mon Sep 17 00:00:00 2001 From: fernchen Date: Thu, 9 Jul 2020 13:48:32 +0800 Subject: [PATCH 1/5] [CI][Caffe Frontend] add caffe environment --- docker/Dockerfile.ci_cpu | 4 ++ docker/install/ubuntu_install_caffe.sh | 93 ++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 docker/install/ubuntu_install_caffe.sh diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu index 42067b2cf38b..305fec7ccba7 100644 --- a/docker/Dockerfile.ci_cpu +++ b/docker/Dockerfile.ci_cpu @@ -78,3 +78,7 @@ RUN bash /install/ubuntu_install_tensorflow.sh # Arm(R) Compute Library COPY install/ubuntu_install_arm_compute_lib.sh /install/ubuntu_install_arm_compute_lib.sh RUN bash /install/ubuntu_install_arm_compute_lib.sh + +# Caffe deps +COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh +RUN bash /install/ubuntu_install_caffe.sh diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh new file mode 100644 index 000000000000..ea175e22921d --- /dev/null +++ b/docker/install/ubuntu_install_caffe.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -u +set -o pipefail + +# Prerequisite +apt-get update --fix-missing + +export DEBIAN_FRONTEND=noninteractive +apt-get install -y tzdata +ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +dpkg-reconfigure --frontend noninteractive tzdata + +apt-get install libprotobuf-dev -y +apt-get install libleveldb-dev -y +apt-get install libsnappy-dev -y +apt-get install libopencv-dev -y +apt-get install libhdf5-serial-dev -y +apt-get install protobuf-compiler -y +apt-get install libgflags-dev -y +apt-get install libgoogle-glog-dev -y +apt-get install liblmdb-dev -y +apt-get install libatlas-base-dev -y +apt-get install --no-install-recommends libboost-all-dev -y +apt-get install gfortran -y + +cd / +mkdir caffe +cd caffe +wget https://github.com/weiliu89/caffe/archive/ssd.zip -O ssd.zip +unzip ssd.zip +rm ssd.zip +cd caffe-ssd + +echo "CPU_ONLY := 1" >> Makefile.config +echo "OPENCV_VERSION := 3" >> Makefile.config +echo "BLAS := open" >> Makefile.config +echo "PYTHON_LIBRARIES := boost_python3 python3.6m" >> Makefile.config +echo "PYTHON_INCLUDE := /usr/include/python3.6m /usr/lib/python3.6/dist-packages/numpy/core/include /usr/local/lib/python3.6/dist-packages/numpy/core/include" >> Makefile.config +echo "PYTHON_LIB := /usr/lib" >> Makefile.config +echo "WITH_PYTHON_LAYER := 1" >> Makefile.config +echo INCLUDE_DIRS := $\(PYTHON_INCLUDE\) /usr/local/include /usr/include/hdf5/serial >> Makefile.config +echo LIBRARY_DIRS := $\(PYTHON_LIB\) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial >> Makefile.config +echo "BUILD_DIR := build" >> Makefile.config +echo "DISTRIBUTE_DIR := distribute" >> Makefile.config +echo "Q ?= @" >> Makefile.config + + +make -j8 +make test -j8 +make runtest -j8 + +echo export PYTHONPATH=/caffe/caffe-ssd/python >> /etc/profile +cd ./python + +rm requirements.txt +echo "Cython>=0.19.2" >> requirements.txt +echo "numpy>=1.7.1" >> requirements.txt +echo "scipy>=0.13.2" >> requirements.txt +echo "scikit-image>=0.9.3" >> requirements.txt +echo "h5py>=2.2.0" >> requirements.txt +echo "leveldb>=0.191" >> requirements.txt +echo "networkx>=1.8.1" >> requirements.txt +echo "nose>=1.3.0" >> requirements.txt +echo "pandas>=0.12.0" >> requirements.txt +echo "python-dateutil>=2.6.0" >> requirements.txt +echo "protobuf>=2.5.0" >> requirements.txt +echo "python-gflags>=2.0" >> requirements.txt +echo "pyyaml>=3.10" >> requirements.txt +echo "Pillow>=2.3.0" >> requirements.txt +echo "six>=1.1.0" >> requirements.txt + +for req in $(cat requirements.txt); do pip3 install $req -i https://pypi.tuna.tsinghua.edu.cn/simple some-package; done + +cd .. +make pycaffe From dee15c49f2fbf193d52523a8b74cbec7cafdb1d0 Mon Sep 17 00:00:00 2001 From: fernchen Date: Mon, 20 Jul 2020 20:48:30 +0800 Subject: [PATCH 2/5] [CI][Caffe Frontend] change the caffe deps into BVLC distribution. --- docker/Dockerfile.ci_gpu | 3 ++ docker/install/ubuntu_install_caffe.sh | 66 ++------------------------ 2 files changed, 6 insertions(+), 63 deletions(-) diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu index 6233cc53211f..b8a60fc36968 100644 --- a/docker/Dockerfile.ci_gpu +++ b/docker/Dockerfile.ci_gpu @@ -83,6 +83,9 @@ RUN bash /install/ubuntu_install_dgl.sh COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh RUN bash /install/ubuntu_install_vulkan.sh +COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh +RUN bash /install/ubuntu_install_caffe.sh + # AutoTVM deps COPY install/ubuntu_install_redis.sh /install/ubuntu_install_redis.sh RUN bash /install/ubuntu_install_redis.sh diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index ea175e22921d..7bc49780676f 100644 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -20,74 +20,14 @@ set -e set -u set -o pipefail -# Prerequisite apt-get update --fix-missing +# avoid manually selecting the time zone export DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime dpkg-reconfigure --frontend noninteractive tzdata - -apt-get install libprotobuf-dev -y -apt-get install libleveldb-dev -y -apt-get install libsnappy-dev -y -apt-get install libopencv-dev -y -apt-get install libhdf5-serial-dev -y -apt-get install protobuf-compiler -y -apt-get install libgflags-dev -y -apt-get install libgoogle-glog-dev -y -apt-get install liblmdb-dev -y -apt-get install libatlas-base-dev -y -apt-get install --no-install-recommends libboost-all-dev -y -apt-get install gfortran -y -cd / -mkdir caffe -cd caffe -wget https://github.com/weiliu89/caffe/archive/ssd.zip -O ssd.zip -unzip ssd.zip -rm ssd.zip -cd caffe-ssd +apt install caffe-cpu -y -echo "CPU_ONLY := 1" >> Makefile.config -echo "OPENCV_VERSION := 3" >> Makefile.config -echo "BLAS := open" >> Makefile.config -echo "PYTHON_LIBRARIES := boost_python3 python3.6m" >> Makefile.config -echo "PYTHON_INCLUDE := /usr/include/python3.6m /usr/lib/python3.6/dist-packages/numpy/core/include /usr/local/lib/python3.6/dist-packages/numpy/core/include" >> Makefile.config -echo "PYTHON_LIB := /usr/lib" >> Makefile.config -echo "WITH_PYTHON_LAYER := 1" >> Makefile.config -echo INCLUDE_DIRS := $\(PYTHON_INCLUDE\) /usr/local/include /usr/include/hdf5/serial >> Makefile.config -echo LIBRARY_DIRS := $\(PYTHON_LIB\) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial >> Makefile.config -echo "BUILD_DIR := build" >> Makefile.config -echo "DISTRIBUTE_DIR := distribute" >> Makefile.config -echo "Q ?= @" >> Makefile.config - - -make -j8 -make test -j8 -make runtest -j8 - -echo export PYTHONPATH=/caffe/caffe-ssd/python >> /etc/profile -cd ./python - -rm requirements.txt -echo "Cython>=0.19.2" >> requirements.txt -echo "numpy>=1.7.1" >> requirements.txt -echo "scipy>=0.13.2" >> requirements.txt -echo "scikit-image>=0.9.3" >> requirements.txt -echo "h5py>=2.2.0" >> requirements.txt -echo "leveldb>=0.191" >> requirements.txt -echo "networkx>=1.8.1" >> requirements.txt -echo "nose>=1.3.0" >> requirements.txt -echo "pandas>=0.12.0" >> requirements.txt -echo "python-dateutil>=2.6.0" >> requirements.txt -echo "protobuf>=2.5.0" >> requirements.txt -echo "python-gflags>=2.0" >> requirements.txt -echo "pyyaml>=3.10" >> requirements.txt -echo "Pillow>=2.3.0" >> requirements.txt -echo "six>=1.1.0" >> requirements.txt - -for req in $(cat requirements.txt); do pip3 install $req -i https://pypi.tuna.tsinghua.edu.cn/simple some-package; done - -cd .. -make pycaffe +pip install --upgrade scikit-image From 6691e77354c294dc88a3d0ad1cfb201aef44069a Mon Sep 17 00:00:00 2001 From: fernchen Date: Tue, 21 Jul 2020 09:58:41 +0800 Subject: [PATCH 3/5] [CI][Caffe Fronted] simplify configuration while installing tzdata for precompiled caffe. --- docker/install/ubuntu_install_caffe.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index 7bc49780676f..b46e64a7f895 100644 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -25,8 +25,6 @@ apt-get update --fix-missing # avoid manually selecting the time zone export DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata -ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime -dpkg-reconfigure --frontend noninteractive tzdata apt install caffe-cpu -y From a1e78f41b0bf2512812b5e6f5d29243c84606533 Mon Sep 17 00:00:00 2001 From: fernchen Date: Tue, 21 Jul 2020 11:36:24 +0800 Subject: [PATCH 4/5] [CI][Caffe Frontend] add more information about tzdata. --- docker/install/ubuntu_install_caffe.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index b46e64a7f895..5176e53186b3 100644 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -22,7 +22,10 @@ set -o pipefail apt-get update --fix-missing -# avoid manually selecting the time zone +# The precompiled caffe dependents on tzdata. +# While installing tzdata in docker, we need set the time zone manually, +# which will cause the container to hang during installation. +# So in order to avoid manually selecting the time zone, set as following: export DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata From f03a8327eb6dcb366586ae17515627bf34e3b8b4 Mon Sep 17 00:00:00 2001 From: fernchen Date: Tue, 28 Jul 2020 12:25:37 +0800 Subject: [PATCH 5/5] [CI][CaffeFrontend]remove the ci for gpu env and change to pip3 env --- docker/Dockerfile.ci_gpu | 3 --- docker/install/ubuntu_install_caffe.sh | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu index b8a60fc36968..6233cc53211f 100644 --- a/docker/Dockerfile.ci_gpu +++ b/docker/Dockerfile.ci_gpu @@ -83,9 +83,6 @@ RUN bash /install/ubuntu_install_dgl.sh COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh RUN bash /install/ubuntu_install_vulkan.sh -COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh -RUN bash /install/ubuntu_install_caffe.sh - # AutoTVM deps COPY install/ubuntu_install_redis.sh /install/ubuntu_install_redis.sh RUN bash /install/ubuntu_install_redis.sh diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh index 5176e53186b3..21f7d09b1bf3 100644 --- a/docker/install/ubuntu_install_caffe.sh +++ b/docker/install/ubuntu_install_caffe.sh @@ -29,6 +29,6 @@ apt-get update --fix-missing export DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata -apt install caffe-cpu -y +apt-get install caffe-cpu -y -pip install --upgrade scikit-image +pip3 install --upgrade scikit-image