diff --git a/.appveyor.yml b/.appveyor.yml index 53bedccae..0795d3153 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -49,7 +49,7 @@ install: # - appveyor DownloadFile %LAPACK_URL% dlls\lapack.dll build_script: - - nimble.exe refresh + - nimble.exe install -y test_script: - nimble.exe test_no_lapack diff --git a/.travis.yml b/.travis.yml index 6b39f6b39..834547ceb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ install: - choosenim $CHANNEL script: - - nimble refresh + - nimble install -y - if [[$BLIS]]; then nimble test_blis; else nimble test; diff --git a/arraymancer.nimble b/arraymancer.nimble index 369144b64..e4c5fc244 100644 --- a/arraymancer.nimble +++ b/arraymancer.nimble @@ -40,55 +40,52 @@ srcDir = "src" ## TODO: auto detection or at least check in common directories ## Note: It is import to gate compiler flags like -march=native behind Xcompiler "-Xcompiler -march=native" -template cudaSwitches() = - switch("cincludes", "/opt/cuda/include") - switch("cc", "gcc") # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372 - switch("gcc.exe", "/opt/cuda/bin/nvcc") - switch("gcc.linkerexe", "/opt/cuda/bin/nvcc") - switch("gcc.cpp.exe", "/opt/cuda/bin/nvcc") - switch("gcc.cpp.linkerexe", "/opt/cuda/bin/nvcc") +template cudaSwitches(switches: var string) = + switches.add " --cincludes:/opt/cuda/include" + switches.add " --cc:gcc" # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372 + switches.add " --gcc.exe:/opt/cuda/bin/nvcc" + switches.add " --gcc.linkerexe:/opt/cuda/bin/nvcc" + switches.add " --gcc.cpp.exe:/opt/cuda/bin/nvcc" + switches.add " --gcc.cpp.linkerexe:/opt/cuda/bin/nvcc" # Due to the __ldg intrinsics in kernels # we only support compute capabilities 3.5+ # See here: http://docs.nvidia.com/cuda/pascal-compatibility-guide/index.html # And wikipedia for GPU capabilities: https://en.wikipedia.org/wiki/CUDA - switch("gcc.options.always", "-arch=sm_61 --x cu") # Interpret .c files as .cu - switch("gcc.cpp.options.always", "-arch=sm_61 --x cu -Xcompiler -fpermissive") # Interpret .c files as .cu, gate fpermissive behind Xcompiler - switch("define", "cudnn") - -when defined(cuda): - cudaSwitches - -template mkl_threadedSwitches() = - switch("define","openmp") - switch("stackTrace","off") - switch("define","blas=mkl_intel_lp64") - switch("define","lapack=mkl_intel_lp64") - switch("clibdir", "/opt/intel/mkl/lib/intel64") - switch("passl", "/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a") - switch("passl", "-lmkl_core") - switch("passl", "-lmkl_gnu_thread") - switch("passl", "-lgomp") - switch("dynlibOverride","mkl_intel_lp64") - -template mkl_singleSwitches() = - switch("define","blas=mkl_intel_lp64") - switch("define","lapack=mkl_intel_lp64") - switch("clibdir", "/opt/intel/mkl/lib/intel64") - switch("passl", "/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a") - switch("passl", "-lmkl_core") - switch("passl", "-lmkl_sequential") - switch("dynlibOverride","mkl_intel_lp64") - -template cuda_mkl_openmp() = - mkl_threadedSwitches() - switch("cincludes", "/opt/cuda/include") - switch("cc", "gcc") # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372 - switch("gcc.exe", "/opt/cuda/bin/nvcc") - switch("gcc.linkerexe", "/opt/cuda/bin/nvcc") - switch("gcc.cpp.exe", "/opt/cuda/bin/nvcc") - switch("gcc.cpp.linkerexe", "/opt/cuda/bin/nvcc") - switch("gcc.options.always", "-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native") - switch("gcc.cpp.options.always", "-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native") + switches.add " --gcc.options.always:\"-arch=sm_61 --x cu\"" # Interpret .c files as .cu + switches.add " --gcc.cpp.options.always:\"-arch=sm_61 --x cu -Xcompiler -fpermissive\"" # Interpret .c files as .cu, gate fpermissive behind Xcompiler + switches.add " -d:cudnn" + +template mkl_threadedSwitches(switches: var string) = + switches.add " --d:openmp" + switches.add " --stackTrace:off" + switches.add " --d:blas=mkl_intel_lp64" + switches.add " --d:lapack=mkl_intel_lp64" + switches.add " --clibdir:/opt/intel/mkl/lib/intel64" + switches.add " --passl:/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a" + switches.add " --passl:-lmkl_core" + switches.add " --passl:-lmkl_gnu_thread" + switches.add " --passl:-lgomp" + switches.add " --dynlibOverride:mkl_intel_lp64" + +template mkl_singleSwitches(switches: var string) = + switches.add " --d:blas=mkl_intel_lp64" + switches.add " --d:lapack=mkl_intel_lp64" + switches.add " --clibdir:/opt/intel/mkl/lib/intel64" + switches.add " --passl:/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a" + switches.add " --passl:-lmkl_core" + switches.add " --passl:-lmkl_sequential" + switches.add " --dynlibOverride:mkl_intel_lp64" + +template cuda_mkl_openmp(switches: var string) = + switches.mkl_threadedSwitches() + switches.add " --cincludes:/opt/cuda/include" + switches.add " --cc:gcc" # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372 + switches.add " --gcc.exe:/opt/cuda/bin/nvcc" + switches.add " --gcc.linkerexe:/opt/cuda/bin/nvcc" + switches.add " --gcc.cpp.exe:/opt/cuda/bin/nvcc" + switches.add " --gcc.cpp.linkerexe:/opt/cuda/bin/nvcc" + switches.add " --gcc.options.always:\"-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native\"" + switches.add " --gcc.cpp.options.always:\"-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native\"" ######################################################## # Optimization @@ -105,83 +102,86 @@ template cuda_mkl_openmp() = ########################################################################## ## Testing tasks -proc test(name: string, lang: string = "c") = +proc test(name, switches = "", split = false, lang = "c") = if not dirExists "build": mkDir "build" - if not dirExists "nimcache": - mkDir "nimcache" - --run - switch("out", ("./build/" & name)) - setCommand lang, "tests/" & name & ".nim" + if not split: + exec "nim " & lang & " -o:build/" & name & switches & " -r tests/" & name & ".nim" + else: + exec "nim " & lang & " -o:build/" & name & switches & " -r tests/_split_tests/" & name & ".nim" task all_tests, "Run all tests - Intel MKL + OpenMP + Cuda + march=native + release": - switch("define","cuda") - cuda_mkl_openmp - test "full_test_suite", "cpp" + var switches = " -d:cuda" + switches.cuda_mkl_openmp() + test "full_test_suite", switches, split=false, lang="cpp" task test, "Run all tests - Default BLAS & Lapack": - test "tests_cpu" + test "tests_tensor_part01", "", split = true + test "tests_tensor_part02", "", split = true + test "tests_tensor_part03", "", split = true + test "tests_tensor_part04", "", split = true + test "tests_tensor_part05", "", split = true + test "tests_cpu_remainder", "", split = true task test_no_lapack, "Run all tests - Default BLAS without lapack": - switch("define", "no_lapack") - test "tests_cpu" + let switch = " -d:no_lapack" + test "tests_tensor_part01", switch, split = true + test "tests_tensor_part02", switch, split = true + test "tests_tensor_part03", switch, split = true + test "tests_tensor_part04", switch, split = true + test "tests_tensor_part05", switch, split = true + test "tests_cpu_remainder", switch, split = true task test_cpp, "Run all tests - Cpp codegen": - test "tests_cpu", "cpp" + test "tests_cpu", "", split = false, "cpp" task test_cuda, "Run all tests - Cuda backend with CUBLAS and CuDNN": - switch("define","cuda") - switch("define","cudnn") - cudaSwitches # Unfortunately the "switch" line doesn't also trigger - # the "when defined(cuda)" part of this nimble file - # hence the need to call cudaSwitches explicitly - test "tests_cuda", "cpp" + var switches = " -d:cuda -d:cudnn" + switches.cudaSwitches() + test "tests_cuda", switches, split = false, "cpp" task test_opencl, "Run all OpenCL backend tests": - switch("define", "opencl") - test "tests_opencl" + test "tests_opencl", " -d:opencl" # task test_deprecated, "Run all tests on deprecated procs": # test "tests_cpu_deprecated" task test_openblas, "Run all tests - OpenBLAS": - switch("define","blas=openblas") - switch("define","lapack=openblas") + var switches = " -d:blas=openblas -d:lapack=openblas" when defined(macosx): ## Should work but somehow Nim doesn't find libopenblas.dylib on MacOS - switch("clibdir", "/usr/local/opt/openblas/lib") - switch("cincludes", "/usr/local/opt/openblas/include") - test "tests_cpu" + switches.add " --clibdir:/usr/local/opt/openblas/lib" + switches.add " --cincludes:/usr/local/opt/openblas/include" + test "tests_cpu", switches task test_blis, "Run all tests - BLIS": - switch("define","blis") - test "tests_cpu" + test "tests_cpu", " -d:blis" task test_native, "Run all tests - march=native": - switch("define","native") - test "tests_cpu" + test "tests_cpu", " -d:native" task test_openmp, "Run all tests - OpenMP": - switch("define","openmp") - switch("stackTrace","off") # stacktraces interfere with OpenMP + var switches = " -d:openmp" + switches.add " --stackTrace:off" # stacktraces interfere with OpenMP when defined(macosx): # Default compiler on Mac is clang without OpenMP and gcc is an alias to clang. # Use Homebrew GCC instead for OpenMP support. GCC (v7), must be properly linked via `brew link gcc` - switch("cc", "gcc") - switch("gcc.exe", "/usr/local/bin/gcc-7") - switch("gcc.linkerexe", "/usr/local/bin/gcc-7") - test "tests_cpu" + switches.add " --cc:gcc" + switches.add " --gcc.exe:/usr/local/bin/gcc-7" + switches.add " --gcc.linkerexe:/usr/local/bin/gcc-7" + test "tests_cpu", switches task test_mkl, "Run all tests - Intel MKL - single threaded": - mkl_singleSwitches - test "tests_cpu" + var switches: string + switches.mkl_singleSwitches() + test "tests_cpu", switches task test_mkl_omp, "Run all tests - Intel MKL + OpenMP": - mkl_threadedSwitches - test "tests_cpu" + var switches: string + switches.mkl_threadedSwitches() + test "tests_cpu", switches task test_release, "Run all tests - Release mode": - switch("define","release") - test "tests_cpu" + test "tests_cpu", " -d:release" task gen_doc, "Generate Arraymancer documentation": # TODO: Industrialize: something more robust that only check nim files (and not .DS_Store ...) diff --git a/tests/_split_tests/tests_cpu_remainder.nim b/tests/_split_tests/tests_cpu_remainder.nim new file mode 100644 index 000000000..8bd0efef7 --- /dev/null +++ b/tests/_split_tests/tests_cpu_remainder.nim @@ -0,0 +1,46 @@ +# Copyright 2017 the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tests were split to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import ../io/test_csv, + ../io/test_numpy, + ../datasets/test_mnist, + ../datasets/test_imdb, + ../nn_primitives/test_nnp_numerical_gradient, + ../nn_primitives/test_nnp_convolution, + ../nn_primitives/test_nnp_loss, + ../nn_primitives/test_nnp_maxpool, + ../nn_primitives/test_nnp_gru, + ../nn_primitives/test_nnp_embedding, + ../autograd/test_gate_basic, + ../autograd/test_gate_blas, + ../autograd/test_gate_hadamard, + ../autograd/test_gate_shapeshifting, + ../ml/test_metrics, + ../test_bugtracker + +when not defined(windows) and not sizeof(int) == 4: + # STB image does not work on windows 32-bit, https://github.com/mratsim/Arraymancer/issues/358 + import ../io/test_image + +when not defined(no_lapack): + import ../linear_algebra/test_linear_algebra, + ../ml/test_dimensionality_reduction, + ../ml/test_clustering + +import ../stability_tests/test_stability_openmp, + # /end_to_end/examples_compile + ../end_to_end/examples_run diff --git a/tests/_split_tests/tests_tensor_part01.nim b/tests/_split_tests/tests_tensor_part01.nim new file mode 100644 index 000000000..56c23e864 --- /dev/null +++ b/tests/_split_tests/tests_tensor_part01.nim @@ -0,0 +1,23 @@ +# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Split tests to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import + ../tensor/test_init, + ../tensor/test_operators_comparison, + ../tensor/test_accessors, + ../tensor/test_accessors_slicer, + ../tensor/test_display diff --git a/tests/_split_tests/tests_tensor_part02.nim b/tests/_split_tests/tests_tensor_part02.nim new file mode 100644 index 000000000..e2636444f --- /dev/null +++ b/tests/_split_tests/tests_tensor_part02.nim @@ -0,0 +1,19 @@ +# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Split tests to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import + ../tensor/test_operators_blas diff --git a/tests/_split_tests/tests_tensor_part03.nim b/tests/_split_tests/tests_tensor_part03.nim new file mode 100644 index 000000000..9a7bd4f4a --- /dev/null +++ b/tests/_split_tests/tests_tensor_part03.nim @@ -0,0 +1,21 @@ +# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Split tests to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import + ../tensor/test_math_functions, + ../tensor/test_higherorder, + ../tensor/test_aggregate diff --git a/tests/_split_tests/tests_tensor_part04.nim b/tests/_split_tests/tests_tensor_part04.nim new file mode 100644 index 000000000..23e397994 --- /dev/null +++ b/tests/_split_tests/tests_tensor_part04.nim @@ -0,0 +1,20 @@ +# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Split tests to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import + ../tensor/test_shapeshifting, + ../tensor/test_broadcasting diff --git a/tests/_split_tests/tests_tensor_part05.nim b/tests/_split_tests/tests_tensor_part05.nim new file mode 100644 index 000000000..1a169eac5 --- /dev/null +++ b/tests/_split_tests/tests_tensor_part05.nim @@ -0,0 +1,22 @@ +# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Split tests to save on memory +# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895) + +import + ../tensor/test_ufunc, + ../tensor/test_filling_data, + ../tensor/test_optimization, + ../tensor/test_exporting diff --git a/tests/nn_primitives/test_nnp_loss.nim b/tests/nn_primitives/test_nnp_loss.nim index 203400132..f64aac0b8 100644 --- a/tests/nn_primitives/test_nnp_loss.nim +++ b/tests/nn_primitives/test_nnp_loss.nim @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ../../src/arraymancer, unittest +import ../../src/arraymancer, unittest, random +# Fix random seed for reproducibility +randomize(1234) suite "[NN primitives] Loss functions": proc `~=`[T: SomeFloat](a, b: T): bool =