Thinner docker image

.gitignore

@@ -1,5 +1,7 @@
 *.DS_Store
-build/
+build/paddle/math
+build/paddle/utils
+build/paddle/gserver
 *.user
 
 .vscode

paddle/scripts/deb/build_scripts/build.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+ARCH=$(uname -i)
 apt-get update
 apt-get install -y dh-make
 cd ~
@@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu
 mkdir -p ~/dist/cpu-noavx
 mkdir -p ~/dist/gpu-noavx
 cd paddle
-mkdir build
+
+# clean build dir and third_party dir cache
+rm -rf build third_party
+mkdir -p build
 cd build
-cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON
+cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 make -j `nproc`
+# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything?
 cpack -D CPACK_GENERATOR='DEB' ..
 mv *.deb ~/dist/cpu
 
 rm -rf *
-cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/
+ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
+cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 make -j `nproc`
 cpack -D CPACK_GENERATOR='DEB' ..
 mv *.deb ~/dist/gpu
 
 
 rm -rf *
-cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF
+rm -f /usr/lib/libcudnn.so
+cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 make -j `nproc`
 cpack -D CPACK_GENERATOR='DEB' ..
 mv *.deb ~/dist/cpu-noavx
 
 rm -rf *
-cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/
+ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
+cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 make -j `nproc`
 cpack -D CPACK_GENERATOR='DEB' ..
 mv *.deb ~/dist/gpu-noavx

paddle/scripts/docker/README.md

@@ -0,0 +1,9 @@
+# Build docker image
+
+We use a docker environment to build paddle binaries and put it into a runtime image `paddle-core` for uses of most cases
+
+***Notice***: do **not** run in this directory, run under the top level of this project like:
+
+```
+sh paddle/scripts/docker/buildall.sh
+```

paddle/scripts/docker/buildall.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+BUILD_DIR=$PWD/build
+DEB_DIST_DIR=$PWD/dist
+VERSION=latest
+
+function build_in_docker() {
+  if [ ! -d $BUILD_DIR ]; then
+    mkdir -p $BUILD_DIR
+  fi
+  if [ ! -d $DEB_DIST_DIR ]; then
+    mkdir -p $DEB_DIST_DIR
+  fi
+  docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile
+  # FIXME: need to wait a signal not sleeping
+  BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600)
+  # NOTICE: build deb files for real paddle image
+  docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh"
+
+  docker stop $BUILDER && docker rm $BUILDER
+}
+
+function build_paddle_core() {
+  docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile
+  docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu
+  docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx
+  docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx
+}
+
+build_in_docker
+build_paddle_core

paddle/scripts/docker/paddle-core/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:2.7.13-slim
+MAINTAINER PaddlePaddle Authors <[email protected]>
+
+# ENV variables
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV WITH_GPU=OFF
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+
+# Use Fix locales to en_US.UTF-8
+
+RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y libgfortran3 && \
+    apt-get clean -y && \
+    pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0'
+RUN pip install numpy
+# Use different deb file when building different type of images
+ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/
+RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb
+
+ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
+# default command shows the paddle version and exit
+CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.gpu

@@ -0,0 +1,32 @@
+FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
+MAINTAINER PaddlePaddle Authors <[email protected]>
+
+# ENV variables
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV WITH_GPU=OFF
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+
+# Use Fix locales to en_US.UTF-8
+
+RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y python python-pip libgfortran3 && \
+    apt-get clean -y && \
+    pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0'
+RUN pip install numpy
+# Use different deb file when building different type of images
+ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/
+RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb
+
+ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
+# default command shows the paddle version and exit
+CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx

@@ -0,0 +1,33 @@
+FROM python:2.7.13-slim
+MAINTAINER PaddlePaddle Authors <[email protected]>
+
+# ENV variables
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV WITH_GPU=OFF
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+
+# Use Fix locales to en_US.UTF-8
+
+RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y libgfortran3 && \
+    apt-get clean -y && \
+    pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0'
+RUN pip install numpy
+# Use different deb file when building different type of images
+ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/
+RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb
+
+
+ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
+# default command shows the paddle version and exit
+CMD ["paddle", "version"]

paddle/scripts/docker/paddle-core/Dockerfile.noavx

@@ -0,0 +1,32 @@
+FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
+MAINTAINER PaddlePaddle Authors <[email protected]>
+
+# ENV variables
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV WITH_GPU=OFF
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+
+# Use Fix locales to en_US.UTF-8
+
+RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y python python-pip libgfortran3 && \
+    apt-get clean -y && \
+    pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0'
+RUN pip install numpy
+# Use different deb file when building different type of images
+ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/
+RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb
+
+ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
+# default command shows the paddle version and exit
+CMD ["paddle", "version"]

paddle/scripts/docker/paddle-dev/Dockerfile

@@ -0,0 +1,53 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
+MAINTAINER PaddlePaddle Authors <[email protected]>
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG UBUNTU_MIRROR
+RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
+
+# ENV variables
+ARG BUILD_WOBOQ
+ARG BUILD_AND_INSTALL
+ARG WITH_AVX
+ARG WITH_DOC
+ARG WITH_STYLE_CHECK
+
+ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
+ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
+ENV WITH_GPU=OFF
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
+ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
+
+ENV HOME /root
+
+RUN apt-get update && \
+    apt-get install -y git python-pip python-dev openssh-server bison && \
+    apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
+    apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
+    apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
+    apt-get install -y automake locales clang-format-3.8 && \
+    apt-get clean -y
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+RUN pip install --upgrade pip && \
+    pip install -U 'protobuf==3.1.0' && \
+    pip install -U wheel pillow BeautifulSoup && \
+    pip install -U docopt PyYAML sphinx && \
+    pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \
+    pip install -U pre-commit 'requests==2.9.2' jupyter
+
+RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
+    cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
+    cd .. && rm -rf cmake-3.4.1
+
+RUN apt-get install -y swig
+# FIXME: wait a long time is OK
+CMD ["sleep", "3600"]