Add LibTorch support

conda/Dockerfile

@@ -65,17 +65,29 @@ RUN wget -q https://developer.nvidia.com/compute/cuda/9.0/Prod/patches/2/cuda_9.
 
 # install CUDA 9.0 CuDNN
 # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-RUN curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.1.2/cudnn-9.0-linux-x64-v7.1.tgz -O && \
-    tar --no-same-owner -xzf cudnn-9.0-linux-x64-v7.1.tgz -C /usr/local && \
-    rm cudnn-9.0-linux-x64-v7.1.tgz && \
+RUN mkdir tmp_cudnn && cd tmp_cudnn && \
+    wget -q http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7-dev_7.4.1.5-1+cuda9.0_amd64.deb && \
+    wget -q http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7_7.4.1.5-1+cuda9.0_amd64.deb && \
+    ar -x libcudnn7-dev_7.4.1.5-1+cuda9.0_amd64.deb && tar -xvf data.tar.xz && \
+    ar -x libcudnn7_7.4.1.5-1+cuda9.0_amd64.deb && tar -xvf data.tar.xz && \
+    mkdir -p cuda/include && mkdir -p cuda/lib64 && \
+    cp -a usr/include/x86_64-linux-gnu/cudnn_v7.h cuda/include/cudnn.h && \
+    cp -a usr/lib/x86_64-linux-gnu/libcudnn* cuda/lib64 && \
+    mv cuda/lib64/libcudnn_static_v7.a cuda/lib64/libcudnn_static.a && \
+    ln -s libcudnn.so.7 cuda/lib64/libcudnn.so && \
+    chmod +x cuda/lib64/*.so && \
+    cp -a cuda/include/* /usr/local/cuda/include/ && \
+    cp -a cuda/lib64/* /usr/local/cuda/lib64/ && \
+    cd .. && \
+    rm -rf tmp_cudnn && \
     ldconfig
 
 # NCCL2 license: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html
-RUN wget -q https://s3.amazonaws.com/pytorch/nccl_2.2.13-1%2Bcuda9.0_x86_64.txz && \
-    tar --no-same-owner -xvf nccl_2.2.13-1+cuda9.0_x86_64.txz && \
-    mv nccl_2.2.13-1+cuda9.0_x86_64/include/* /usr/local/cuda/include/ && \
-    cp -P nccl_2.2.13-1+cuda9.0_x86_64/lib/libnccl* /usr/local/cuda/lib64/ && \
-    rm -rf nccl_2.2.13-1+cuda9.0_x86_64* && \
+RUN wget -q https://s3.amazonaws.com/pytorch/nccl_2.3.7-1%2Bcuda9.0_x86_64.txz && \
+    tar --no-same-owner -xvf nccl_2.3.7-1+cuda9.0_x86_64.txz && \
+    mv nccl_2.3.7-1+cuda9.0_x86_64/include/* /usr/local/cuda/include/ && \
+    cp -P nccl_2.3.7-1+cuda9.0_x86_64/lib/libnccl* /usr/local/cuda/lib64/ && \
+    rm -rf nccl_2.3.7-1+cuda9.0_x86_64* && \
     ldconfig
 
 # install CUDA 9.2 in the same container
@@ -86,20 +98,31 @@ RUN wget -q https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers
 
 # install CUDA 9.2 CuDNN
 # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
-RUN curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.1.4/cudnn-9.2-linux-x64-v7.1.tgz -O && \
-    tar --no-same-owner -xzf cudnn-9.2-linux-x64-v7.1.tgz -C /usr/local && \
-    rm cudnn-9.2-linux-x64-v7.1.tgz && \
+RUN mkdir tmp_cudnn && cd tmp_cudnn && \
+    wget -q http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7-dev_7.4.1.5-1+cuda9.2_amd64.deb && \
+    wget -q http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/libcudnn7_7.4.1.5-1+cuda9.2_amd64.deb && \
+    ar -x libcudnn7-dev_7.4.1.5-1+cuda9.2_amd64.deb && tar -xvf data.tar.xz && \
+    ar -x libcudnn7_7.4.1.5-1+cuda9.2_amd64.deb && tar -xvf data.tar.xz && \
+    mkdir -p cuda/include && mkdir -p cuda/lib64 && \
+    cp -a usr/include/x86_64-linux-gnu/cudnn_v7.h cuda/include/cudnn.h && \
+    cp -a usr/lib/x86_64-linux-gnu/libcudnn* cuda/lib64 && \
+    mv cuda/lib64/libcudnn_static_v7.a cuda/lib64/libcudnn_static.a && \
+    ln -s libcudnn.so.7 cuda/lib64/libcudnn.so && \
+    chmod +x cuda/lib64/*.so && \
+    cp -a cuda/include/* /usr/local/cuda/include/ && \
+    cp -a cuda/lib64/* /usr/local/cuda/lib64/ && \
+    cd .. && \
+    rm -rf tmp_cudnn && \
     ldconfig
 
 # NCCL2 license: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html
-RUN wget -q https://s3.amazonaws.com/pytorch/nccl_2.2.13-1%2Bcuda9.2_x86_64.txz && \
-    tar --no-same-owner -xvf nccl_2.2.13-1+cuda9.2_x86_64.txz && \
-    mv nccl_2.2.13-1+cuda9.2_x86_64/include/* /usr/local/cuda/include/ && \
-    cp -P nccl_2.2.13-1+cuda9.2_x86_64/lib/libnccl* /usr/local/cuda/lib64/ && \
-    rm -rf nccl_2.2.13-1+cuda9.2_x86_64* && \
+RUN wget -q https://s3.amazonaws.com/pytorch/nccl_2.3.7-1%2Bcuda9.2_x86_64.txz && \
+    tar --no-same-owner -xvf nccl_2.3.7-1+cuda9.2_x86_64.txz && \
+    mv nccl_2.3.7-1+cuda9.2_x86_64/include/* /usr/local/cuda/include/ && \
+    cp -P nccl_2.3.7-1+cuda9.2_x86_64/lib/libnccl* /usr/local/cuda/lib64/ && \
+    rm -rf nccl_2.3.7-1+cuda9.2_x86_64* && \
     ldconfig
 
-
 # prune static libs
 ARG GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61"
 ARG CUDA_ROOT="/usr/local/cuda-8.0"
@@ -111,16 +134,14 @@ RUN ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep
 ARG GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70"
 ARG CUDA_ROOT="/usr/local/cuda-9.0"
 ARG CUDA_LIB_DIR="/usr/local/cuda-9.0/lib64"
-RUN ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | \
-    xargs -I {} bash -c "echo {} && $CUDA_ROOT/bin/nvprune $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" && \
-    $CUDA_ROOT/bin/nvprune $GENCODE -gencode code=compute_30 $CUDA_LIB_DIR/libcudnn_static.a -o $CUDA_LIB_DIR/libcudnn_static.a
+RUN ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | \
+    xargs -I {} bash -c "echo {} && $CUDA_ROOT/bin/nvprune $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
 
 ARG GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70"
 ARG CUDA_ROOT="/usr/local/cuda-9.2"
 ARG CUDA_LIB_DIR="/usr/local/cuda-9.2/lib64"
-RUN ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "cusolver" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | \
-    xargs -I {} bash -c "echo {} && $CUDA_ROOT/bin/nvprune $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" && \
-    $CUDA_ROOT/bin/nvprune $GENCODE -gencode code=compute_30 $CUDA_LIB_DIR/libcudnn_static.a -o $CUDA_LIB_DIR/libcudnn_static.a
+RUN ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "cusolver" | grep -v "culibos" | grep -v "cudart" | \
+    xargs -I {} bash -c "echo {} && $CUDA_ROOT/bin/nvprune $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
 
 
 # Anaconda

conda/magma-cuda90-2.4.0/cmakelists.patch

@@ -0,0 +1,140 @@
+--- CMakeLists.txt    2016-11-20 20:20:02.000000000 -0500
++++ CMakeLists.txt    2017-07-17 01:10:32.161739159 -040062a63
+62a63
+> set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++ -fno-exceptions")
+82,89c83,90
+< find_package( OpenMP )
+< if ( OPENMP_FOUND )
+< 	message( STATUS "Found OpenMP" )
+< 	message( STATUS "    OpenMP_C_FLAGS   ${OpenMP_C_FLAGS}" )
+< 	message( STATUS "    OpenMP_CXX_FLAGS ${OpenMP_CXX_FLAGS}" )
+< 	set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
+< 	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" )
+< endif()
+---
+> # find_package( OpenMP )
+> # if ( OPENMP_FOUND )
+> # 	message( STATUS "Found OpenMP" )
+> # 	message( STATUS "    OpenMP_C_FLAGS   ${OpenMP_C_FLAGS}" )
+> # 	message( STATUS "    OpenMP_CXX_FLAGS ${OpenMP_CXX_FLAGS}" )
+> # 	set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
+> # 	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" )
+> # endif()
+238a240,246
+> 
+> 	if ( ${GPU_TARGET} MATCHES "All")
+> 	  set( MIN_ARCH 350)
+> 	  SET( NV_SM "-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 ")
+> 	  SET( NV_COMP "")
+> 	endif()
+>       
+244c252,253
+< 	set( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DHAVE_CUBLAS ${NV_SM} ${NV_COMP} ${FORTRAN_CONVENTION}" )
+---
+> 	set( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DHAVE_CUBLAS -Xfatbin -compress-all -Xcompiler -fPIC ${NV_SM} ${NV_COMP} ${FORTRAN_CONVENTION}" )
+> 	MESSAGE(STATUS "CUDA_NVCC_FLAGS: ${CUDA_NVCC_FLAGS}")
+255c264
+< 	find_package( LAPACK )
+---
+> 	# find_package( LAPACK )
+302c311,312
+< set( LIBS tester lapacktest magma )
+---
+> # set( LIBS tester lapacktest magma )
+> set( LIBS magma)
+410a421
+> set_target_properties(magma PROPERTIES POSITION_INDEPENDENT_CODE ON)
+414c425
+< add_library( tester ${libtest_all} )
+---
+> # add_library( tester ${libtest_all} )
+421,436c432,447
+< if ( USE_FORTRAN )
+< 	foreach( f ${liblapacktest_all} )
+< 		if ( ${f} MATCHES "\\.(f|f90)$" )
+< 			list( APPEND liblapacktest_all_f ${f} )
+< 		endif()
+< 	endforeach()
+< 	add_library( lapacktest ${liblapacktest_all_f} )
+< else()
+< 	# alternatively, use only C/C++/CUDA files, including magma_[sdcz]_no_fortran.cpp
+< 	foreach( f ${liblapacktest_all} )
+< 		if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+< 			list( APPEND liblapacktest_all_cpp ${f} )
+< 		endif()
+< 	endforeach()
+< 	add_library( lapacktest ${liblapacktest_all_cpp} )
+< endif()
+---
+> # if ( USE_FORTRAN )
+> # 	foreach( f ${liblapacktest_all} )
+> # 		if ( ${f} MATCHES "\\.(f|f90)$" )
+> # 			list( APPEND liblapacktest_all_f ${f} )
+> # 		endif()
+> # 	endforeach()
+> # 	add_library( lapacktest ${liblapacktest_all_f} )
+> # else()
+> # 	# alternatively, use only C/C++/CUDA files, including magma_[sdcz]_no_fortran.cpp
+> # 	foreach( f ${liblapacktest_all} )
+> # 		if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+> # 			list( APPEND liblapacktest_all_cpp ${f} )
+> # 		endif()
+> # 	endforeach()
+> # 	add_library( lapacktest ${liblapacktest_all_cpp} )
+> # endif()
+456c467
+< 
+---
+> set_target_properties(magma_sparse PROPERTIES POSITION_INDEPENDENT_CODE ON)
+462,464c473,475
+< set( CMAKE_RUNTIME_OUTPUT_DIRECTORY testing )
+< set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY testing_lib )
+< set( CMAKE_LIBRARY_OUTPUT_DIRECTORY testing_lib )
+---
+> # set( CMAKE_RUNTIME_OUTPUT_DIRECTORY testing )
+> # set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY testing_lib )
+> # set( CMAKE_LIBRARY_OUTPUT_DIRECTORY testing_lib )
+467,478c478,489
+< foreach( f ${testing_all} )
+< 	if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+< 		list( APPEND testing_all_cpp ${f} )
+< 	endif()
+< endforeach()
+< foreach( TEST ${testing_all_cpp} )
+< 	string( REGEX REPLACE "\\.(cpp|f90|F90)" "" EXE ${TEST} )
+< 	string( REGEX REPLACE "testing/" "" EXE ${EXE} )
+< 	#message( "${TEST} --> ${EXE}" )
+< 	add_executable( ${EXE} ${TEST} )
+< 	target_link_libraries( ${EXE} ${LIBS} )
+< endforeach()
+---
+> # foreach( f ${testing_all} )
+> # 	if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+> # 		list( APPEND testing_all_cpp ${f} )
+> # 	endif()
+> # endforeach()
+> # foreach( TEST ${testing_all_cpp} )
+> # 	string( REGEX REPLACE "\\.(cpp|f90|F90)" "" EXE ${TEST} )
+> # 	string( REGEX REPLACE "testing/" "" EXE ${EXE} )
+> # 	#message( "${TEST} --> ${EXE}" )
+> # 	add_executable( ${EXE} ${TEST} )
+> # 	target_link_libraries( ${EXE} ${LIBS} )
+> # endforeach()
+483,490c494,501
+< set( CMAKE_RUNTIME_OUTPUT_DIRECTORY sparse/testing )
+< foreach( TEST ${sparse_testing_all} )
+< 	string( REGEX REPLACE "\\.(cpp|f90|F90)"     "" EXE ${TEST} )
+< 	string( REGEX REPLACE "sparse/testing/" "" EXE ${EXE} )
+< 	#message( "${TEST} --> ${EXE}" )
+< 	add_executable( ${EXE} ${TEST} )
+< 	target_link_libraries( ${EXE} ${LIBS_SPARSE} )
+< endforeach()
+---
+> # set( CMAKE_RUNTIME_OUTPUT_DIRECTORY sparse/testing )
+> # foreach( TEST ${sparse_testing_all} )
+> # 	string( REGEX REPLACE "\\.(cpp|f90|F90)"     "" EXE ${TEST} )
+> # 	string( REGEX REPLACE "sparse/testing/" "" EXE ${EXE} )
+> # 	#message( "${TEST} --> ${EXE}" )
+> # 	add_executable( ${EXE} ${TEST} )
+> # 	target_link_libraries( ${EXE} ${LIBS_SPARSE} )
+> # endforeach()

conda/magma-cuda90-2.4.0/meta.yaml

@@ -0,0 +1,17 @@
+package:
+  name: magma-cuda90
+  version: 2.4.0
+
+source:
+   url: http://icl.cs.utk.edu/projectsfiles/magma/downloads/magma-2.4.0.tar.gz
+   patches:
+     - cmakelists.patch
+     - thread_queue.patch
+
+build:
+  number: 1
+
+about:
+  home: http://icl.cs.utk.edu/magma/software/index.html
+  license: BSD
+  license_file: COPYRIGHT

conda/magma-cuda92-2.4.0/cmakelists.patch

@@ -0,0 +1,140 @@
+--- CMakeLists.txt    2016-11-20 20:20:02.000000000 -0500
++++ CMakeLists.txt    2017-07-17 01:10:32.161739159 -040062a63
+62a63
+> set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++ -fno-exceptions")
+82,89c83,90
+< find_package( OpenMP )
+< if ( OPENMP_FOUND )
+< 	message( STATUS "Found OpenMP" )
+< 	message( STATUS "    OpenMP_C_FLAGS   ${OpenMP_C_FLAGS}" )
+< 	message( STATUS "    OpenMP_CXX_FLAGS ${OpenMP_CXX_FLAGS}" )
+< 	set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
+< 	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" )
+< endif()
+---
+> # find_package( OpenMP )
+> # if ( OPENMP_FOUND )
+> # 	message( STATUS "Found OpenMP" )
+> # 	message( STATUS "    OpenMP_C_FLAGS   ${OpenMP_C_FLAGS}" )
+> # 	message( STATUS "    OpenMP_CXX_FLAGS ${OpenMP_CXX_FLAGS}" )
+> # 	set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
+> # 	set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" )
+> # endif()
+238a240,246
+> 
+> 	if ( ${GPU_TARGET} MATCHES "All")
+> 	  set( MIN_ARCH 350)
+> 	  SET( NV_SM "-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 ")
+> 	  SET( NV_COMP "")
+> 	endif()
+>       
+244c252,253
+< 	set( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DHAVE_CUBLAS ${NV_SM} ${NV_COMP} ${FORTRAN_CONVENTION}" )
+---
+> 	set( CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -DHAVE_CUBLAS -Xfatbin -compress-all -Xcompiler -fPIC ${NV_SM} ${NV_COMP} ${FORTRAN_CONVENTION}" )
+> 	MESSAGE(STATUS "CUDA_NVCC_FLAGS: ${CUDA_NVCC_FLAGS}")
+255c264
+< 	find_package( LAPACK )
+---
+> 	# find_package( LAPACK )
+302c311,312
+< set( LIBS tester lapacktest magma )
+---
+> # set( LIBS tester lapacktest magma )
+> set( LIBS magma)
+410a421
+> set_target_properties(magma PROPERTIES POSITION_INDEPENDENT_CODE ON)
+414c425
+< add_library( tester ${libtest_all} )
+---
+> # add_library( tester ${libtest_all} )
+421,436c432,447
+< if ( USE_FORTRAN )
+< 	foreach( f ${liblapacktest_all} )
+< 		if ( ${f} MATCHES "\\.(f|f90)$" )
+< 			list( APPEND liblapacktest_all_f ${f} )
+< 		endif()
+< 	endforeach()
+< 	add_library( lapacktest ${liblapacktest_all_f} )
+< else()
+< 	# alternatively, use only C/C++/CUDA files, including magma_[sdcz]_no_fortran.cpp
+< 	foreach( f ${liblapacktest_all} )
+< 		if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+< 			list( APPEND liblapacktest_all_cpp ${f} )
+< 		endif()
+< 	endforeach()
+< 	add_library( lapacktest ${liblapacktest_all_cpp} )
+< endif()
+---
+> # if ( USE_FORTRAN )
+> # 	foreach( f ${liblapacktest_all} )
+> # 		if ( ${f} MATCHES "\\.(f|f90)$" )
+> # 			list( APPEND liblapacktest_all_f ${f} )
+> # 		endif()
+> # 	endforeach()
+> # 	add_library( lapacktest ${liblapacktest_all_f} )
+> # else()
+> # 	# alternatively, use only C/C++/CUDA files, including magma_[sdcz]_no_fortran.cpp
+> # 	foreach( f ${liblapacktest_all} )
+> # 		if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+> # 			list( APPEND liblapacktest_all_cpp ${f} )
+> # 		endif()
+> # 	endforeach()
+> # 	add_library( lapacktest ${liblapacktest_all_cpp} )
+> # endif()
+456c467
+< 
+---
+> set_target_properties(magma_sparse PROPERTIES POSITION_INDEPENDENT_CODE ON)
+462,464c473,475
+< set( CMAKE_RUNTIME_OUTPUT_DIRECTORY testing )
+< set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY testing_lib )
+< set( CMAKE_LIBRARY_OUTPUT_DIRECTORY testing_lib )
+---
+> # set( CMAKE_RUNTIME_OUTPUT_DIRECTORY testing )
+> # set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY testing_lib )
+> # set( CMAKE_LIBRARY_OUTPUT_DIRECTORY testing_lib )
+467,478c478,489
+< foreach( f ${testing_all} )
+< 	if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+< 		list( APPEND testing_all_cpp ${f} )
+< 	endif()
+< endforeach()
+< foreach( TEST ${testing_all_cpp} )
+< 	string( REGEX REPLACE "\\.(cpp|f90|F90)" "" EXE ${TEST} )
+< 	string( REGEX REPLACE "testing/" "" EXE ${EXE} )
+< 	#message( "${TEST} --> ${EXE}" )
+< 	add_executable( ${EXE} ${TEST} )
+< 	target_link_libraries( ${EXE} ${LIBS} )
+< endforeach()
+---
+> # foreach( f ${testing_all} )
+> # 	if ( ${f} MATCHES "\\.(c|cu|cpp)$" )
+> # 		list( APPEND testing_all_cpp ${f} )
+> # 	endif()
+> # endforeach()
+> # foreach( TEST ${testing_all_cpp} )
+> # 	string( REGEX REPLACE "\\.(cpp|f90|F90)" "" EXE ${TEST} )
+> # 	string( REGEX REPLACE "testing/" "" EXE ${EXE} )
+> # 	#message( "${TEST} --> ${EXE}" )
+> # 	add_executable( ${EXE} ${TEST} )
+> # 	target_link_libraries( ${EXE} ${LIBS} )
+> # endforeach()
+483,490c494,501
+< set( CMAKE_RUNTIME_OUTPUT_DIRECTORY sparse/testing )
+< foreach( TEST ${sparse_testing_all} )
+< 	string( REGEX REPLACE "\\.(cpp|f90|F90)"     "" EXE ${TEST} )
+< 	string( REGEX REPLACE "sparse/testing/" "" EXE ${EXE} )
+< 	#message( "${TEST} --> ${EXE}" )
+< 	add_executable( ${EXE} ${TEST} )
+< 	target_link_libraries( ${EXE} ${LIBS_SPARSE} )
+< endforeach()
+---
+> # set( CMAKE_RUNTIME_OUTPUT_DIRECTORY sparse/testing )
+> # foreach( TEST ${sparse_testing_all} )
+> # 	string( REGEX REPLACE "\\.(cpp|f90|F90)"     "" EXE ${TEST} )
+> # 	string( REGEX REPLACE "sparse/testing/" "" EXE ${EXE} )
+> # 	#message( "${TEST} --> ${EXE}" )
+> # 	add_executable( ${EXE} ${TEST} )
+> # 	target_link_libraries( ${EXE} ${LIBS_SPARSE} )
+> # endforeach()