-
Notifications
You must be signed in to change notification settings - Fork 94
/
Dockerfile-runtime-multi-gpu
149 lines (118 loc) · 5.15 KB
/
Dockerfile-runtime-multi-gpu
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
ARG docker_name
FROM $docker_name
ARG use_miniconda
MAINTAINER H2o.ai <[email protected]>
ENV HOME=/root
ENV CUDA_HOME=/usr/local/cuda
ENV PATH=/usr/local/cuda/bin:$PATH
ENV CUDADIR=/usr/local/cuda/include/
ENV LD_LIBRARY_PATH=/usr/lib64:/usr/local/lib:$LD_LIBRARY_PATH
ENV MINICONDA_VERSION=4.8.3
# Setup gcc etc.
RUN yum install -y epel-release
RUN yum install -y gcc gcc-c++ libgcc libstdc++ libgomp glibc
RUN yum install -y \
make \
ncurses-devel \
zlib-devel \
epel-release \
zeromq-devel \
wget \
blas-devel \
openblas-devel \
libpng-devel \
freetype-devel \
bzip2 \
libffi-devel
# Arrow
RUN bash -c 'if [ `arch` = "ppc64le" ]; then yum install -y git boost-devel cmake3 autoconf libjpeg-devel; fi'
RUN mkdir -p /opt/h2oai/h2o4gpu
RUN wget https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai-thirdparty-deps/1.0-master-22/`arch`-centos7/llvm.tar.bz2 && \
tar xvf llvm.tar.bz2 && \
cp -r llvm/* /opt/h2oai/h2o4gpu/ && \
rm -rf llvm*
ENV LLVM4=/opt/h2oai/h2o4gpu
ENV PATH=$LLVM4/bin:$PATH
ENV LD_LIBRARY_PATH=$LLVM4/lib:$LD_LIBRARY_PATH
ENV LLVM_CONFIG=$LLVM4/bin/llvm-config
ARG python_version
# ln -sf /usr/bin/python36 /usr/bin/python breaks yum
RUN if [ "$use_miniconda" = "1" ] ; then wget https://repo.anaconda.com/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-`arch`.sh && \
bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-`arch`.sh -b -p /opt/h2oai/h2o4gpu/python && \
/opt/h2oai/h2o4gpu/python/bin/conda install -y python=${python_version} pip; else \
yum install -y python36 python36-devel python36-pip && \
ln -sf /usr/bin/python36 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip; \
fi
ENV PATH=/opt/h2oai/h2o4gpu/python/bin:$PATH
RUN pip install -U pip==23.3.2
#
# PPC64 specific - certain libs/whl don't support PPC64LE
#
WORKDIR $HOME
# Arrow
# Need to enable c99 manually https://github.com/numpy/numpy/issues/14147
RUN bash -c 'if [ `arch` = "ppc64le" ]; then \
ln -s /usr/bin/cmake3 /usr/bin/cmake && \
git clone https://github.com/apache/arrow.git && \
cd $HOME/arrow/cpp && \
git checkout tags/apache-arrow-0.17.1 && \
CFLAGS=-std=c99 pip install numpy==1.19.2 cython==0.29.14 scipy==1.5.2 && \
export ARROW_BUILD_TYPE=release && \
export ARROW_HOME=/usr/local && \
cmake -DARROW_CXXFLAGS="-lutil" -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_PYTHON=on -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE -DARROW_BUILD_TESTS=OFF && make -j && make install && \
cd $HOME/arrow/python && \
python setup.py install; \
fi'
WORKDIR /
# Add requirements
COPY src/interface_py/requirements_runtime_demos_multi_gpu.txt requirements_runtime_demos_multi_gpu.txt
COPY src/interface_py/requirements_test.txt requirements_test.txt
RUN \
chmod a+rwx requirements*.txt && \
python -m pip install -r requirements_runtime_demos_multi_gpu.txt && \
python -m pip install -r requirements_test.txt
RUN mkdir -p /etc/OpenCL/vendors && \
echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
RUN (localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || true)
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LD_LIBRARY_PATH_CUDA=$CUDA_HOME/lib64/:$CUDA_HOME/lib/:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
# Add a canned jupyter notebook demo to the container
RUN \
mkdir -p jupyter/demos
COPY examples/py/demos/H2O4GPU_Ridge.ipynb /jupyter/demos/H2O4GPU_Ridge.ipynb
COPY examples/py/demos/H2O4GPU_LinearRegression.ipynb /jupyter/demos/H2O4GPU_LinearRegression.ipynb
COPY examples/py/demos/H2O4GPU_GBM.ipynb /jupyter/demos/H2O4GPU_GBM.ipynb
COPY examples/py/demos/H2O4GPU_GLM.ipynb /jupyter/demos/H2O4GPU_GLM.ipynb
COPY examples/py/demos/H2O4GPU_Lasso.ipynb /jupyter/demos/H2O4GPU_Lasso.ipynb
COPY examples/py/demos/H2O4GPU_KMeans_Images.ipynb /jupyter/demos/H2O4GPU_KMeans_Images.ipynb
COPY examples/py/demos/H2O4GPU_KMeans_Quantization.ipynb /jupyter/demos/H2O4GPU_KMeans_Quantization.ipynb
COPY examples/py/demos/Multi-GPU-H2O-GLM-simple.ipynb /jupyter/demos/Multi-GPU-H2O-GLM-simple.ipynb
COPY examples/py/demos/H2O4GPU_TruncatedSVD.ipynb /jupyter/demos/H2O4GPU_TruncatedSVD.ipynb
COPY examples/py/demos/H2O4GPU_PCA.ipynb /jupyter/demos/H2O4GPU_PCA.ipynb
COPY examples/py/demos/H2O4GPU_Daal_LinearRegression.ipynb /jupyter/demos/H2O4GPU_Daal_LinearRegression.ipynb
COPY examples/py/demos/figures /jupyter/demos/figures
RUN \
cd /jupyter/demos && \
chmod -R a+rwx /jupyter && \
mkdir /scikit_learn_data && \
chmod -R a+rwx /scikit_learn_data && \
mkdir -p /jupyter/scikit_learn_data/covertype && \
chmod -R a+rwx /jupyter/scikit_learn_data/covertype && \
mkdir -p /jupyter/scikit_learn_data/lfw_home && \
chmod -R a+rwx /jupyter/scikit_learn_data/lfw_home && \
HOME=/jupyter jupyter notebook --generate-config && \
sed -i "s/#c.NotebookApp.token = '<generated>'/c.NotebookApp.token = 'h2o'/" /jupyter/.jupyter/jupyter_notebook_config.py && \
chmod -R a+rwx /jupyter/.jupyter
# Add shell wrapper
COPY scripts/run.sh /run.sh
RUN \
chmod a+rwx run.sh
ARG h2o4gpu_VERSION
ARG h2o4gpu_COMMIT
ARG DOCKER_VERSION_TAG
LABEL \
h2o4gpu_commit="$h2o4gpu_COMMIT" \
docker_version_tag="$DOCKER_VERSION_TAG"
ENTRYPOINT ["./run.sh"]
EXPOSE 8888