[2.0] Bump Python to >= 3.8 (#20593)

* Bump Python to >= 3.8, NumPy to >= 1.21.0

* remove pillow in requirements

* update python

* fix some tests

.github/workflows/link_check.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.8'
           architecture: x64
 
       - name: Install Dependencies

.github/workflows/os_x_mklbuild.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.8'
           architecture: x64
 
       - name: Install Dependencies

.github/workflows/os_x_staticbuild.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.8'
           architecture: x64
 
       - name: Install Dependencies

cd/python/docker/Dockerfile

@@ -27,8 +27,8 @@ RUN apt-get update && \
     apt-get install -y software-properties-common && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get update && \
-    apt-get install -y python3.7-dev python3.7-distutils virtualenv wget && \
-    ln -sf /usr/bin/python3.7 /usr/local/bin/python3 && \
+    apt-get install -y python3.8-dev python3.8-distutils virtualenv wget && \
+    ln -sf /usr/bin/python3.8 /usr/local/bin/python3 && \
     wget -nv https://bootstrap.pypa.io/get-pip.py && \
     python3 get-pip.py
 

ci/docker/Dockerfile.build.centos7

@@ -54,7 +54,7 @@ RUN yum -y check-update || true && \
         # CentOS Software Collections https://www.softwarecollections.org
         devtoolset-7 \
         devtoolset-8 \
-        rh-python36 \
+        rh-python38 \
         rh-maven35 \
         # Libraries
         # Provide clbas headerfiles
@@ -82,9 +82,9 @@ RUN mkdir ~/openblas && \
     CXX="g++ -fPIC" CC="gcc -fPIC" make -j DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 && \
     make PREFIX=/usr/local install
 
-# Make Python 3.6 and Maven 3.3 Software Collections available by default during
+# Make Python 3.8 and Maven 3.3 Software Collections available by default during
 # the following build steps in this Dockerfile
-SHELL [ "/usr/bin/scl", "enable", "devtoolset-7", "rh-python36", "rh-maven35" ]
+SHELL [ "/usr/bin/scl", "enable", "devtoolset-7", "rh-python38", "rh-maven35" ]
 
 # Install minimum required cmake version
 RUN cd /usr/local/src && \

ci/docker/runtime_functions.sh

@@ -767,7 +767,7 @@ sanity_python() {
 # $1 -> mxnet_variant: The variant of the libmxnet.so library
 cd_unittest_ubuntu() {
     set -ex
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     export PYTHONPATH=./python/
     export MXNET_ONEDNN_DEBUG=0  # Ignored if not present
     export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
@@ -925,7 +925,7 @@ unittest_cpp() {
 
 unittest_centos7_cpu() {
     set -ex
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     cd /work/mxnet
     export DMLC_LOG_STACK_TRACE_DEPTH=100
     OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
@@ -937,7 +937,7 @@ unittest_centos7_cpu() {
 
 unittest_centos7_gpu() {
     set -ex
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     cd /work/mxnet
     export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
     export DMLC_LOG_STACK_TRACE_DEPTH=100
@@ -1368,7 +1368,7 @@ build_static_libmxnet() {
     set -ex
     pushd .
     source /opt/rh/devtoolset-8/enable
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
     export CXXFLAGS="-fabi-version=11 -fabi-compat-version=7"
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
@@ -1393,7 +1393,7 @@ cd_package_pypi() {
     set -ex
     pushd .
     source /opt/rh/devtoolset-8/enable
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
     export CXXFLAGS="-fabi-version=11 -fabi-compat-version=7"
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
@@ -1404,7 +1404,7 @@ cd_package_pypi() {
 # Sanity checks wheel file
 cd_integration_test_pypi() {
     set -ex
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
 
     # install mxnet wheel package
     pip3 install --user ./wheel_build/dist/*.whl
@@ -1438,7 +1438,7 @@ build_static_python_cpu() {
     pushd .
     export mxnet_variant=cpu
     source /opt/rh/devtoolset-8/enable
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
     export CXXFLAGS="-fabi-version=11 -fabi-compat-version=7"
     ./ci/publish/python/build.sh
@@ -1450,7 +1450,7 @@ build_static_python_cu102() {
     pushd .
     export mxnet_variant=cu102
     source /opt/rh/devtoolset-8/enable
-    source /opt/rh/rh-python36/enable
+    source /opt/rh/rh-python38/enable
     # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
     export CXXFLAGS="-fabi-version=11 -fabi-compat-version=7"
     ./ci/publish/python/build.sh

docs/python_docs/requirements

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+numpy>=1.17,<1.20.0
 jupyter
 Jinja2==2.11.3
 sphinx==2.4.0

tests/python/dnnl/test_dnnl.py

@@ -44,17 +44,22 @@ def test_dnnl_ndarray_slice():
     # trigger computation on ndarray slice
     assert_almost_equal(y[0].asnumpy()[0, 0, 0], np.array(0.056331709))
 
+
+# In python3.8, functions are only pickable if they are defined in
+# the top-level of a module
+# https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled
+class Dummy(gluon.data.Dataset):
+    def __len__(self):
+        return 2
+    def __getitem__(self, key):
+        return key, np.ones((3, 224, 224)), np.ones((10, ))
+
 @use_np
 @pytest.mark.seed(1234)
 def test_dnnl_engine_threading():
     net = gluon.nn.HybridSequential()
     net.add(gluon.nn.Conv2D(channels=32, kernel_size=3, activation=None))
     net.initialize(ctx=mx.cpu())
-    class Dummy(gluon.data.Dataset):
-        def __len__(self):
-            return 2
-        def __getitem__(self, key):
-            return key, np.ones((3, 224, 224)), np.ones((10, ))
 
     loader = gluon.data.DataLoader(Dummy(), batch_size=2, num_workers=1)