[BUILD] Upgrade build system to default python3 (#1260)

Jenkinsfile

@@ -106,7 +106,7 @@ stage('Build') {
            cp ../cmake/config.cmake .
            echo set\\(USE_OPENCL ON\\) >> config.cmake
            echo set\\(USE_ROCM ON\\) >> config.cmake
-           echo set\\(USE_VULKAN ON\\) >> config.cmake
+           echo set\\(USE_VULKAN OFF\\) >> config.cmake
            """
         make('gpu', 'build2', '-j2')
       }

Makefile

@@ -48,18 +48,18 @@ build/libtvm_web_runtime.js: build/libtvm_web_runtime.bc
 
 # Lint scripts
 cpplint:
-	python dmlc-core/scripts/lint.py topi cpp topi/include;
-	python dmlc-core/scripts/lint.py nnvm cpp nnvm/include nnvm/src;
-	python dmlc-core/scripts/lint.py tvm cpp include src verilog\
+	python3 dmlc-core/scripts/lint.py topi cpp topi/include;
+	python3 dmlc-core/scripts/lint.py nnvm cpp nnvm/include nnvm/src;
+	python3 dmlc-core/scripts/lint.py tvm cpp include src verilog\
 	 examples/extension/src examples/graph_executor/src
 
 pylint:
-	pylint python/tvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
-	pylint topi/python/topi --rcfile=$(ROOTDIR)/tests/lint/pylintrc
-	pylint nnvm/python/nnvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
+	python3 -m pylint python/tvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
+	python3 -m pylint topi/python/topi --rcfile=$(ROOTDIR)/tests/lint/pylintrc
+	python3 -m pylint nnvm/python/nnvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
 
 jnilint:
-	python dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src
+	python3 dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src
 
 lint: cpplint pylint jnilint
 

docs/Makefile

@@ -3,15 +3,10 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
+SPHINXBUILD   = python3 -m sphinx
 PAPER         =
 BUILDDIR      = _build
 
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter

docs/install/index.rst

@@ -38,7 +38,7 @@ Our goal is to build the shared libraries:
 The minimal building requirements are
 
 - A recent c++ compiler supporting C++ 11 (g++-4.8 or higher)
-- CMake 3.7 or higher
+- CMake 3.5 or higher
 - We highly recommend to build with LLVM to enable all the features.
 - It is possible to build without llvm dependency if we only want to use CUDA/OpenCL
 
@@ -49,6 +49,7 @@ The configuration of tvm can be modified by `config.cmake`.
 - First, check the cmake in your system, you do not have cmake
   you can obtain the latest version from `official website <https://cmake.org/download/>`_
 - First create a build directory, copy the ``cmake/config.cmake`` to the directory.
+
   .. code:: bash
 
       mkdir build
@@ -76,6 +77,7 @@ The configuration of tvm can be modified by `config.cmake`.
 - We can then build tvm and related libraries.
 
   .. code:: bash
+
       cd build
       cmake ..
       make -j4

nnvm/python/nnvm/frontend/coreml.py

@@ -5,6 +5,7 @@
 
 import tvm
 from .. import symbol as _sym
+from .._base import string_types
 from .common import SymbolTable
 
 __all__ = ['from_coreml']
@@ -281,7 +282,7 @@ def coreml_op_to_nnvm(op, inname, outname, symtab):
     classname = type(op).__name__
     if classname not in _convert_map:
         raise NotImplementedError("%s is not supported" % (classname))
-    if isinstance(inname, (str, unicode)):
+    if isinstance(inname, string_types):
         insym = symtab.get_var(inname)
     else:
         insym = [symtab.get_var(i) for i in inname]

nnvm/python/nnvm/frontend/keras.py

@@ -487,14 +487,15 @@ def from_keras(model):
         import keras
     except ImportError:
         raise ImportError('Keras must be installed')
+
     assert isinstance(model, keras.engine.training.Model)
     if keras.backend.image_data_format() != 'channels_last':
         raise ValueError("Keras frontend currently supports data_format = channels_last only.")
     _check_unsupported_layers(model)
 
     symtab = SymbolTable()
     for keras_layer in model.layers:
-        if isinstance(keras_layer, keras.engine.topology.InputLayer):
+        if isinstance(keras_layer, keras.engine.InputLayer):
             symtab.get_var(keras_layer.name, must_contain=False)
         else:
             inbound_nodes = keras_layer.inbound_nodes if hasattr(keras_layer, 'inbound_nodes') \
@@ -512,7 +513,7 @@ def from_keras(model):
                 # would confuse users, so we should keep them as far as possible.  Fortunately,
                 # they are named uniquely to input_1, input_2, input_3 ... by default.
                 for pred_idx, pred in zip(node.node_indices, node.inbound_layers):
-                    if isinstance(pred, keras.engine.topology.InputLayer):
+                    if isinstance(pred, keras.engine.InputLayer):
                         _sym = symtab.get_var(pred.name, must_contain=True)
                     else:
                         _sym = symtab.get_var(pred.name + ':' + str(pred_idx), must_contain=True)
@@ -522,6 +523,6 @@ def from_keras(model):
                     insym = insym[0]
                 keras_op_to_nnvm(insym, keras_layer, keras_layer.name + ':' + str(my_idx), symtab)
 
-    outsym = symtab.get_var(model.output_layers[0].name + ':0')
+    outsym = symtab.get_var(model._output_layers[0].name + ':0')
     tvmparams = {k:tvm.nd.array(np.array(v, dtype=np.float32)) for k, v in symtab.params.items()}
     return outsym, tvmparams

nnvm/tests/python/frontend/keras/test_forward.py

@@ -15,9 +15,9 @@
 
 def verify_keras_frontend(keras_model):
     in_shapes = []
-    for layer in keras_model.input_layers:
+    for layer in keras_model._input_layers:
         in_shapes.append(tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape))
-    out_shape = [dim.value if dim.value is not None else 1 for dim in keras_model.output_layers[0].output.shape]
+    out_shape = [dim.value if dim.value is not None else 1 for dim in keras_model._output_layers[0].output.shape]
 
     def get_keras_output(xs, dtype='float32'):
         return keras_model.predict(xs)
@@ -41,7 +41,7 @@ def get_tvm_output(xs, target, ctx, dtype='float32'):
         tvm_out = get_tvm_output([x.transpose([0,3,1,2]) for x in xs], target, ctx)
         np.testing.assert_allclose(keras_out, tvm_out, rtol=1e-5, atol=1e-5)
 
-    
+
 def test_forward_elemwise_add():
     r = []
     data = keras.layers.Input(shape=(32,32,3))
@@ -74,7 +74,7 @@ def test_forward_dense():
 def test_forward_transpose_conv():
     data = keras.layers.Input(shape=(32,32,3))
     x = keras.layers.Conv2D(filters=10, kernel_size=(3,3), strides=(2,2), padding='same')(data)
-    x = keras.applications.mobilenet.DepthwiseConv2D(kernel_size=(3,3), padding='same')(x)
+    x = keras.layers.DepthwiseConv2D(kernel_size=(3,3), padding='same')(x)
     x = keras.layers.Conv2DTranspose(filters=64, kernel_size=(3,3), padding='valid')(x)
     x = keras.layers.GlobalMaxPooling2D()(x)
     keras_model = keras.models.Model(data, x)
@@ -136,7 +136,6 @@ def test_forward_activations():
     act_funcs = [keras.layers.Activation('softmax'),
                  keras.layers.Activation('softplus'),
                  keras.layers.LeakyReLU(alpha=0.3),
-                 keras.layers.Activation(keras.applications.mobilenet.relu6),
                  keras.layers.PReLU(weights=weights, alpha_initializer="zero"),
                  keras.layers.ELU(alpha=0.5),
                  keras.layers.Activation('selu'),

nnvm/tests/python/frontend/mxnet/test_forward.py

@@ -100,8 +100,7 @@ def test_forward_rrelu():
 def test_forward_prelu():
     data = mx.sym.var('data')
     data = mx.sym.concat(data, -data, dim=1)  # negative part explicitly
-    gamma = mx.sym.zeros(shape=(6,))
-    mx_sym = mx.sym.LeakyReLU(data, gamma=gamma, act_type='prelu')
+    mx_sym = mx.sym.LeakyReLU(data, act_type='prelu')
     verify_mxnet_frontend_impl(mx_sym, (1, 3, 100, 100), (1, 6, 100, 100))
 
 def test_forward_softrelu():

tests/ci_build/Dockerfile.gpu

@@ -23,7 +23,7 @@ RUN bash /install/ubuntu_install_sphinx.sh
 
 # Fix recommonmark to latest version
 RUN git clone https://github.com/rtfd/recommonmark
-RUN cd recommonmark; python setup.py install
+RUN cd recommonmark; python3 setup.py install
 
 # Enable doxygen for c++ doc build
 RUN apt-get update && apt-get install -y doxygen graphviz libprotobuf-dev protobuf-compiler
@@ -40,9 +40,6 @@ RUN bash /install/ubuntu_install_rocm.sh
 COPY install/ubuntu_install_opengl.sh /install/ubuntu_install_opengl.sh
 RUN bash /install/ubuntu_install_opengl.sh
 
-COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh
-RUN bash /install/ubuntu_install_vulkan.sh
-
 
 # DL Frameworks
 COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
@@ -60,7 +57,11 @@ RUN bash /install/ubuntu_install_darknet.sh
 COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh
 RUN bash /install/ubuntu_install_onnx.sh
 
-RUN pip install Pillow
+RUN pip3 install Pillow
+
+# disable vulkan for now
+# COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh
+# RUN bash /install/ubuntu_install_vulkan.sh
 
 # Environment variables
 ENV PATH=/usr/local/nvidia/bin:${PATH}

tests/ci_build/Dockerfile.lint

@@ -1,6 +1,8 @@
 # For lint test
 FROM ubuntu:16.04
 
-RUN apt-get update && apt-get install -y python-pip sudo
+RUN apt-get update && apt-get install -y sudo wget
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
 RUN apt-get install -y doxygen graphviz
-RUN pip install cpplint pylint
+RUN pip3 install cpplint pylint

tests/ci_build/install/ubuntu_install_coreml.sh

@@ -1 +1 @@
-pip2 install coremltools
+pip3 install coremltools

tests/ci_build/install/ubuntu_install_keras.sh

@@ -1 +1,2 @@
 pip2 install keras tensorflow h5py
+pip3 install keras tensorflow h5py

tests/ci_build/install/ubuntu_install_mxnet.sh

@@ -1,2 +1 @@
-pip2 install mxnet
 pip3 install mxnet

tests/ci_build/install/ubuntu_install_python.sh

@@ -1,5 +1,12 @@
 # install python and pip, don't modify this, modify install_python_package.sh
-apt-get update && apt-get install -y python-pip python-dev python3-dev
+apt-get update && apt-get install -y python-dev
 
-# the version of the pip shipped with ubuntu may be too lower, install a recent version here
-cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py
+# python 3.6
+apt-get update && yes | apt-get install software-properties-common
+add-apt-repository ppa:jonathonf/python-3.6 &&\
+    apt-get update && apt-get install -y python-pip python-dev python3.6 python3.6-dev
+
+rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
+
+# Install pip
+cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python2 get-pip.py && python3.6 get-pip.py

tests/ci_build/install/ubuntu_install_python_package.sh

@@ -1,3 +1,3 @@
 # install libraries for python package on ubuntu
 pip2 install nose pylint numpy nose-timer cython decorator scipy tornado
-pip3 install nose pylint numpy nose-timer cython decorator scipy tornado
+pip3 install nose pylint numpy nose-timer cython decorator scipy tornado typed_ast

tests/ci_build/install/ubuntu_install_sphinx.sh

@@ -1 +1 @@
-pip install sphinx sphinx-gallery sphinx_rtd_theme matplotlib Image commonmark>=0.7.3 docutils>=0.11
+pip3 install sphinx sphinx-gallery sphinx_rtd_theme matplotlib Image commonmark>=0.7.3 docutils>=0.11

tests/scripts/task_clean.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 echo "Cleanup data..."
-rm -rf $1
+cd $1 && rm -rf Cmake* && cd ..

tests/scripts/task_python_nnvm.sh

@@ -11,10 +11,10 @@ python -m nose -v nnvm/tests/python/compiler || exit -1
 python3 -m nose -v nnvm/tests/python/compiler || exit -1
 
 echo "Running ONNX frontend test..."
-python -m nose -v nnvm/tests/python/frontend/onnx || exit -1
+python3 -m nose -v nnvm/tests/python/frontend/onnx || exit -1
 
 echo "Running MXNet frontend test..."
-python -m nose -v nnvm/tests/python/frontend/mxnet || exit -1
+python3 -m nose -v nnvm/tests/python/frontend/mxnet || exit -1
 
 echo "Running Keras frontend test..."
-python -m nose -v nnvm/tests/python/frontend/keras || exit -1
+python3 -m nose -v nnvm/tests/python/frontend/keras || exit -1