[ML] Upgrade to PyTorch 1.11 on Linux (#2238)

Together with #2233 and #2235 this completes the upgrade to PyTorch 1.11 for all platforms. A small change to the way things are done is that we now copy all MKL libraries into the Docker images, then copy the required subset into our release distribution. This means we can change the subset of MKL libraries that we redistribute without having to rebuild our Docker images. Fixes #2126
elastic · Mar 30, 2022 · c7c3683 · c7c3683
1 parent 64aa225
commit c7c3683
Show file tree

Hide file tree

Showing 15 changed files with 33 additions and 34 deletions.
diff --git a/3rd_party/3rd_party.sh b/3rd_party/3rd_party.sh
@@ -70,7 +70,7 @@ case `uname` in
                 MKL_LOCATION=/usr/local/gcc103/lib
                 MKL_EXTENSION=.so
                 MKL_PREFIX=libmkl_
-                MKL_LIBRARIES=`cd "$MKL_LOCATION" && ls $MKL_PREFIX*$MKL_EXTENSION`
+                MKL_LIBRARIES='avx avx2 avx512 avx512_mic core def gnu_thread intel_lp64 mc3 vml_avx vml_avx2 vml_avx512 vml_avx512_mic vml_cmpt vml_def vml_mc3'
             fi
             BOOST_EXTENSION=mt-${BOOST_ARCH}-1_77.so.1.77.0
             BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread unit_test_framework'
@@ -265,11 +265,11 @@ fi
 if [ ! -z "$MKL_LOCATION" ] ; then
     if [ ! -z "$MKL_LIBRARIES" ] ; then
         if [ -n "$INSTALL_DIR" ] ; then
+            rm -f $INSTALL_DIR/$MKL_PREFIX*$MKL_EXTENSION
             for LIBRARY in $MKL_LIBRARIES
             do
-                rm -f $INSTALL_DIR/$LIBRARY
-                cp $MKL_LOCATION/$LIBRARY $INSTALL_DIR
-                chmod u+wx $INSTALL_DIR/$LIBRARY
+                cp $MKL_LOCATION/$MKL_PREFIX$LIBRARY$MKL_EXTENSION $INSTALL_DIR
+                chmod u+wx $INSTALL_DIR/$MKL_PREFIX$LIBRARY$MKL_EXTENSION
             done
         fi
     else

diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv
@@ -1,2 +1,2 @@
 name,version,revision,url,license,copyright,sourceURL
-PyTorch,1.9.0,d69c22dd61a2f006dcfe1e3ea8468a3ecaf931aa,https://pytorch.org,BSD-3-Clause,,
+PyTorch,1.11.0,bc2c6edaf163b1a1330e37a6e34caf8c553e4755,https://pytorch.org,BSD-3-Clause,,
diff --git a/build-setup/linux.md b/build-setup/linux.md
@@ -274,18 +274,13 @@ sudo yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-m
 sudo yum -y install intel-mkl-2020.4-912
 ```
 
-Then copy the required libraries to the system directory:
+Then copy the shared libraries to the system directory:
 
 ```
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+sudo cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 ```
 
-### PyTorch 1.9.0
+### PyTorch 1.11.0
 
 PyTorch requires that certain Python modules are installed. Install these modules with `pip` using the same Python version you will build PyTorch with. If you followed the instructions above and built Python from source use `python3.7`:
 
@@ -302,7 +297,7 @@ sudo /usr/local/gcc103/bin/python3.7 -m pip install install numpy pyyaml setupto
 Then obtain the PyTorch code:
 
 ```
-git clone --depth=1 --branch=v1.9.0 [email protected]:pytorch/pytorch.git
+git clone --depth=1 --branch=v1.11.0 [email protected]:pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -330,7 +325,8 @@ export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
 [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF
-export PYTORCH_BUILD_VERSION=1.9.0
+[ $(uname -m) != x86_64 ] && export USE_BREAKPAD=OFF
+export PYTORCH_BUILD_VERSION=1.11.0
 export PYTORCH_BUILD_NUMBER=1
 /usr/local/gcc103/bin/python3.7 setup.py install
 ```

diff --git a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh
@@ -22,7 +22,7 @@
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-cross-build
-VERSION=6
+VERSION=7
 
 set -e
 

diff --git a/dev-tools/docker/build_linux_aarch64_native_build_image.sh b/dev-tools/docker/build_linux_aarch64_native_build_image.sh
@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-native-build
-VERSION=6
+VERSION=7
 
 set -e
 

diff --git a/dev-tools/docker/build_linux_build_image.sh b/dev-tools/docker/build_linux_build_image.sh
@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-build
-VERSION=20
+VERSION=21
 
 set -e
 

diff --git a/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile b/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:7
 
 MAINTAINER David Roberts <[email protected]>
 

diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile
@@ -24,7 +24,7 @@ RUN \
 RUN \
   mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \
   cd /usr/local/sysroot-aarch64-linux-gnu/usr && \
-  wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-aarch64-linux-gnu-6.tar.bz2 | tar jxf - && \
+  wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-aarch64-linux-gnu-7.tar.bz2 | tar jxf - && \
   cd .. && \
   ln -s usr/lib lib && \
   ln -s usr/lib64 lib64

diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:7
 
 MAINTAINER David Roberts <[email protected]>
 

diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile
@@ -127,7 +127,7 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -140,7 +140,8 @@ RUN \
   export USE_MKLDNN=ON && \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export USE_BREAKPAD=OFF && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:7
 
 MAINTAINER David Roberts <[email protected]>
 

diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <[email protected]>
 

diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile
@@ -128,18 +128,13 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 RUN \
   yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo && \
   yum -y install intel-mkl-2020.4-912 && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+  cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 
 # Clone PyTorch and build LibTorch
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -153,7 +148,7 @@ RUN \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
   export USE_XNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <[email protected]>
 

diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc
@@ -28,6 +28,13 @@
 
 //=== Regressions
 
+== {es} version 8.3.0
+
+=== Enhancements
+
+* Upgrade PyTorch to version 1.11. (See {ml-pull}2233[#2233], {ml-pull}2235[#2235]
+  and {ml-pull}2238[#2238].)
+
 == {es} version 8.2.0
 
 === Enhancements