WIP: Julia CI build (#8175)

* Delete workspace before build. (#8195) * Delete workspace before build. * Move 'deleteDir' outside the retry. * Jenkins: fix Jenkinsfile indentation * add Julia package as a submodule and enable CPU build
apache · Oct 13, 2017 · 1b234fb · 1b234fb
1 parent 301052b
commit 1b234fb
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 32 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -16,3 +16,6 @@
 [submodule "cub"]
 	path = cub
 	url = https://github.com/dmlc/cub
+[submodule "julia-package"]
+	path = julia-package
+	url = https://github.com/dmlc/MXNet.jl.git
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -19,7 +19,7 @@ def init_git() {
       timeout(time: 2, unit: 'MINUTES') {
         checkout scm
         sh 'git submodule update --init'
-        sh 'git clean -d -f'        
+        sh 'git clean -d -f'
       }
     } catch (exc) {
       error "Failed to fetch source codes"
@@ -35,7 +35,7 @@ def init_git_win() {
       timeout(time: 2, unit: 'MINUTES') {
         checkout scm
         bat 'git submodule update --init'
-        bat 'git clean -d -f'        
+        bat 'git clean -d -f'
       }
     } catch (exc) {
       error "Failed to fetch source codes"
@@ -405,7 +405,22 @@ try {
           }
         }
       },
-      'Python 2: CPU Win':{
+      'Julia 0.6: CPU': {
+        node('mxnetlinux') {
+          ws('workspace/ut-julia-cpu') {
+            init_git()
+            unpack_lib('cpu')
+            withEnv(['MXNET_HOME=/workspace/ut-julia-cpu']) {
+              timeout(time: max_timem unit: 'MINUTES') {
+                sh "${docker_run} cpu julia -e 'Pkg.clone(\"julia-package\", \"MXNet\")'"
+                sh "${docker_run} cpu julia -e 'Pkg.build(\"MXNet\")'"
+                sh "${docker_run} cpu julia -e 'Pkg.test(\"MXNet\")'"
+              }
+            }
+          }
+        }
+      },
+      'Python 2: CPU Win': {
         node('mxnetwindows') {
           ws('workspace/ut-python-cpu') {
             init_git_win()
@@ -419,56 +434,56 @@ try {
     del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
     C:\\mxnet\\test_cpu.bat"""
           }
-         }
-       },
-       'Python 3: CPU Win': {
-          node('mxnetwindows') {
+        }
+      },
+      'Python 3: CPU Win': {
+        node('mxnetwindows') {
           ws('workspace/ut-python-cpu') {
             init_git_win()
             unstash 'vc14_cpu'
             bat '''rmdir /s/q pkg_vc14_cpu
     7z x -y vc14_cpu.7z'''
-          bat """xcopy C:\\mxnet\\data data /E /I /Y
+            bat """xcopy C:\\mxnet\\data data /E /I /Y
     xcopy C:\\mxnet\\model model /E /I /Y
     call activate py3
     set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
     del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
     C:\\mxnet\\test_cpu.bat"""
           }
-         }
-       },
-       'Python 2: GPU Win':{
-         node('mxnetwindows') {
-           ws('workspace/ut-python-gpu') {
-             init_git_win()
-             unstash 'vc14_gpu'
-             bat '''rmdir /s/q pkg_vc14_gpu
+        }
+      },
+      'Python 2: GPU Win': {
+        node('mxnetwindows') {
+          ws('workspace/ut-python-gpu') {
+            init_git_win()
+            unstash 'vc14_gpu'
+            bat '''rmdir /s/q pkg_vc14_gpu
     7z x -y vc14_gpu.7z'''
-             bat """xcopy C:\\mxnet\\data data /E /I /Y
+            bat """xcopy C:\\mxnet\\data data /E /I /Y
     xcopy C:\\mxnet\\model model /E /I /Y
     call activate py2
     set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
     del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
     C:\\mxnet\\test_gpu.bat"""
            }
          }
-       },
-       'Python 3: GPU Win':{
-         node('mxnetwindows') {
-           ws('workspace/ut-python-gpu') {
-             init_git_win()
-             unstash 'vc14_gpu'
-             bat '''rmdir /s/q pkg_vc14_gpu
+      },
+      'Python 3: GPU Win': {
+        node('mxnetwindows') {
+          ws('workspace/ut-python-gpu') {
+            init_git_win()
+            unstash 'vc14_gpu'
+            bat '''rmdir /s/q pkg_vc14_gpu
     7z x -y vc14_gpu.7z'''
-             bat """xcopy C:\\mxnet\\data data /E /I /Y
+            bat """xcopy C:\\mxnet\\data data /E /I /Y
     xcopy C:\\mxnet\\model model /E /I /Y
     call activate py3
     set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
     del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
     C:\\mxnet\\test_gpu.bat"""
-           }
-         }
+          }
         }
+      }
     }
 
     stage('Integration Test') {

diff --git a/julia-package b/julia-package
diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
@@ -10,3 +10,5 @@ COPY install/ubuntu_install_r.sh /install/
 RUN /install/ubuntu_install_r.sh
 COPY install/ubuntu_install_perl.sh /install/
 RUN /install/ubuntu_install_perl.sh
+COPY install/install_julia.sh /install/
+RUN /install/install_julia.sh
diff --git a/tests/ci_build/install/install_julia.sh b/tests/ci_build/install/install_julia.sh
@@ -20,10 +20,10 @@
 
 set -e
 
-wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
-mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
-tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz
-rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz
+wget https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.0-linux-x86_64.tar.gz
+mv julia-0.6.0-linux-x86_64.tar.gz /tmp/
+tar xfvz /tmp/julia-0.6.0-linux-x86_64.tar.gz
+rm -f /tmp/julia-0.6.0-linux-x86_64.tar.gz
 
 # tar extracted in current directory
-ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
+ln -s -f ${PWD}/julia-903644385b/bin/julia /usr/bin/julia
diff --git a/tools/license_header.py b/tools/license_header.py
@@ -59,6 +59,7 @@
                'cub/',
                'dlpack/',
                'dmlc-core/',
+               'julia-package/',
                'mshadow/',
                'nnvm',
                'ps-lite',