[MXNet-744] Fix website build pipeline Python 3 issues (#12195)

* Fix website build pipeline Python 3 issues (#12195)

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -61,6 +61,7 @@ COPY install/ubuntu_onnx.sh /work/
 RUN /work/ubuntu_onnx.sh
 
 COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
 RUN /work/ubuntu_docs.sh
 
 ARG USER_ID=0

ci/docker/Dockerfile.build.ubuntu_gpu

@@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
 RUN /work/ubuntu_onnx.sh
 
 COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
 RUN /work/ubuntu_docs.sh
 
 COPY install/ubuntu_tutorials.sh /work/

ci/docker/Dockerfile.build.ubuntu_nightly_cpu

@@ -58,6 +58,7 @@ COPY install/ubuntu_onnx.sh /work/
 RUN /work/ubuntu_onnx.sh
 
 COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
 RUN /work/ubuntu_docs.sh
 
 COPY install/ubuntu_nightly_tests.sh /work/

ci/docker/Dockerfile.build.ubuntu_nightly_gpu

@@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
 RUN /work/ubuntu_onnx.sh
 
 COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
 RUN /work/ubuntu_docs.sh
 
 COPY install/ubuntu_tutorials.sh /work/

docs/build_version_doc/requirements.txt → ci/docker/install/docs_requirements

@@ -1,15 +1,16 @@
-beautifulsoup4
-breathe
+beautifulsoup4==4.6.3
+breathe==4.10.0
 cpplint==1.3.0
 CommonMark==0.5.4
 h5py==2.8.0rc1
-mock==1.0.1
-nose
-nose-timer
+mock==2.0.0
+nose==1.3.7
+nose-timer==0.7.3
 numpy<1.15.0,>=1.8.2
 pylint==1.8.3
-pypandoc
-recommonmark==0.4.0 
+pypandoc==1.4
+recommonmark==0.4.0
 requests<2.19.0,>=2.18.4
 scipy==1.0.1
+six==1.11.0
 sphinx==1.5.6

ci/docker/install/ubuntu_docs.sh

@@ -27,15 +27,7 @@ apt-get install -y \
     doxygen \
     pandoc
 
-echo 'Installing python packages...'
-pip install --upgrade pip && pip install \
-    beautifulsoup4 \
-    breathe \
-    CommonMark==0.5.4 \
-    h5py \
-    mock==1.0.1 \
-    pypandoc \
-    recommonmark==0.4.0 \
-    sphinx==1.5.6
+pip3 install -r /work/docs_requirements
+pip2 install -r /work/docs_requirements
 
 echo 'Dependency installation complete.'

docs/Jenkinsfile-dev

@@ -0,0 +1,54 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 60
+
+node('mxnetlinux-cpu') {
+  // Loading the utilities requires a node context unfortunately
+  checkout scm
+  utils = load('ci/Jenkinsfile_utils.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+  stage('Build Docs') {
+    node(NODE_LINUX_CPU) {
+      ws('workspace/docs') {
+        utils.init_git()
+        timeout(time: max_time, unit: 'MINUTES') {
+            sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh build_docs ${params.tags_to_build} ${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}"
+            archiveArtifacts 'docs/build_version_doc/artifacts.tgz'
+            build 'test-website-publish'
+        }
+      }
+    }
+  }
+}
+,
+failure_handler: {
+  if (currentBuild.result == "FAILURE") {
+    // Do nothing.
+  }
+}
+)

docs/build_version_doc/setup_docs_ubuntu.sh

@@ -110,10 +110,10 @@ sudo apt-get update && sudo apt-get install -y \
 wget -nv https://bootstrap.pypa.io/get-pip.py
 echo "Installing for Python 3..."
 sudo python3 get-pip.py
-pip3 install --user -r requirements.txt
+pip3 install --user -r ../../ci/docker/install/docs_requirements
 echo "Installing for Python 2..."
 sudo python2 get-pip.py
-pip2 install --user -r requirements.txt
+pip2 install --user -r ../../ci/docker/install/docs_requirements
 
 
 cd ../../

docs/mxdoc.py

@@ -23,16 +23,16 @@
 import sys
 from recommonmark import transform
 import pypandoc
-# import StringIO from io for python3 compatibility
-from io import StringIO
 import contextlib
-from ConfigParser import SafeConfigParser
+# Use six for Python 2 / 3 compatibility
+from six import StringIO
+from six.moves import configparser
 
 _BUILD_VER = os.getenv('BUILD_VER', 'default')
 print("Building version {}".format(_BUILD_VER))
 _DOC_SET = 'document_sets_' + _BUILD_VER
 
-parser = SafeConfigParser()
+parser = configparser.SafeConfigParser()
 parser.read('settings.ini')
 
 if _DOC_SET not in parser.sections():
@@ -41,8 +41,7 @@
 for section in [ _DOC_SET ]:
     print("Document sets to generate:")
     for candidate in [ 'scala_docs', 'clojure_docs', 'doxygen_docs', 'r_docs' ]:
-        print '%-12s  : %s' % (candidate, parser.get(section, candidate))
-    print
+        print('%-12s  : %s' % (candidate, parser.get(section, candidate)))
 
 _MXNET_DOCS_BUILD_MXNET = parser.getboolean('mxnet', 'build_mxnet')
 _SCALA_DOCS = parser.getboolean(_DOC_SET, 'scala_docs')