Improvements of Full Pfaffian Computation

.gitmodules

@@ -1,7 +1,3 @@
-[submodule "src/pfaffine"]
-	path = src/pfaffine
-	url = https://github.com/xrq-phys/Pfaffine
-	branch = blis
 [submodule "src/blis"]
 	path = src/blis
 	url = https://github.com/xrq-phys/blis

CMakeLists.txt

@@ -1,8 +1,9 @@
-cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
 project(mVMC NONE)
 
 option(USE_SCALAPACK "Use Scalapack" OFF)
 option(PFAFFIAN_BLOCKED "Use blocked-update Pfaffian to speed up." OFF)
+option(USE_GEMMT "Use GEMMT. Recommended regardless blocked-Pfaffian-update." ON)
 
 add_definitions(-D_mVMC)
 if(CONFIG)
@@ -17,18 +18,18 @@ message(STATUS "Build type: " ${CMAKE_BUILD_TYPE})
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 
 option(GIT_SUBMODULE_UPDATE "execute `git submodule update` automatically" ON)
-enable_language(C Fortran)
-if(PFAFFIAN_BLOCKED)
-  enable_language(CXX)
-  set(CMAKE_CXX_STANDARD 11)
-endif(PFAFFIAN_BLOCKED)
+
+# First, enables C language only.
+# External packages only use their C API.
+enable_language(C)
 
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_SKIP_BUILD_RPATH FALSE)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) 
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 set(CMAKE_MACOSX_RPATH 1)
 
+# TODO: Is this really needed?
 if("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_DEBUG}")
 else()
@@ -37,16 +38,19 @@ else()
 endif()
 
 if(CMAKE_C_COMPILER_ID STREQUAL "Intel")
+  # TODO: Really needs separation?
   if("${CMAKE_C_COMPILER_VERSION}" VERSION_LESS "15.0.0.20140528")
      set(OMP_FLAG_Intel "-openmp")
   else()
     set(OMP_FLAG_Intel "-qopenmp")
   endif()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OMP_FLAG_Intel}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMP_FLAG_Intel}")
 else()
   find_package(OpenMP)
   if(OPENMP_FOUND)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_C_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
   endif(OPENMP_FOUND)	
 endif()
 
@@ -57,13 +61,44 @@ if(MPI_C_FOUND)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_C_LINK_FLAGS}")
 endif(MPI_C_FOUND)
 
+# BLAS/LAPACK/Pfapack77 needs Fortran interface.
+enable_language(Fortran)
+
 find_package(LAPACK)
 if(USE_SCALAPACK MATCHES OFF)
   if(LAPACK_FOUND)
     add_definitions(-D_lapack)
   endif(LAPACK_FOUND)
 endif()
 
+if(PFAFFIAN_BLOCKED
+    OR USE_GEMMT)
+  set(Require_BLIS ON)
+endif()
+
+if(BLA_VENDOR MATCHES "Intel10*"
+    OR BLAS_LIBRARIES MATCHES "/*mkl*")
+  # Don't require BLIS when MKL is used.
+  add_definitions(-DMKL)
+  add_definitions(-DBLAS_EXTERNAL)
+  add_definitions(-DF77_COMPLEX_RET_INTEL)
+  set(Require_BLIS OFF)
+elseif(BLA_VENDOR MATCHES "FLA*"
+    OR BLAS_LIBRARIES MATCHES "/*blis*")
+  # Skip extra BLIS if it's already the BLAS vendor.
+  list(GET BLAS_LIBRARIES 0 BLIS_FIRST_LIB)
+  get_filename_component(BLIS_LIB_DIR ${BLIS_FIRST_LIB} DIRECTORY)
+  include_directories(${BLIS_LIB_DIR}/../include)
+  include_directories(${BLIS_LIB_DIR}/../include/blis)
+  set(Require_BLIS OFF)
+else()
+  # BLAS vendor preference:
+  # External > BLIS > Reference
+  if(DEFINED BLA_VENDOR)
+    add_definitions(-DBLAS_EXTERNAL)
+  endif()
+endif()
+
 # Build and enable tests
 # testing setup
 # enable_testing() must be called in the top-level CMakeLists.txt before any add_subdirectory() is called.
@@ -97,12 +132,23 @@ if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
 endif()
 add_subdirectory("${STDFACE_DIR}")
 
+# C++ support for pfupdates & ltl2inv.
+enable_language(CXX)
+set(CMAKE_CXX_STANDARD 11)
+
+if(Require_BLIS)
+  include("download_blis_artifact.cmake")
+else(Require_BLIS)
+  # Use bundled blis.h
+  include_directories(src/common/deps)
+endif(Require_BLIS)
+
 if(PFAFFIAN_BLOCKED)
   add_definitions(-D_pf_block_update)
-  include("download_blis_artifact.cmake")
-  # Must set BLIS artifact BEFORE adding pfupdates target.
   add_subdirectory(src/pfupdates)
-  add_dependencies(pfupdates blis_include)
+  if(Require_BLIS)
+    add_dependencies(pfupdates blis_include)
+  endif(Require_BLIS)
 endif(PFAFFIAN_BLOCKED)
 
 if (Document)
@@ -111,5 +157,9 @@ endif(Document)
 
 add_subdirectory(src/ComplexUHF)
 add_subdirectory(src/pfapack/fortran)
+add_subdirectory(src/ltl2inv)
+  if(Require_BLIS)
+    add_dependencies(ltl2inv blis_include)
+  endif(Require_BLIS)
 add_subdirectory(src/mVMC)
 add_subdirectory(tool)

config/aocc.cmake

@@ -9,6 +9,9 @@ set(CMAKE_C_FLAGS_RELEASE "-Wno-unknown-pragmas -O3 -DNDEBUG -DHAVE_SSE2" CACHE
 set(CMAKE_Fortran_COMPILER "flang" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
 
+# OpenMP, libatomic
+set(CMAKE_EXE_LINKER_FLAGS "-fopenmp -latomic") 
+
 # for AOCL
 set(BLA_VENDOR "FLAME" CACHE STRING "" FORCE)
 

config/apple.cmake

@@ -6,6 +6,7 @@ set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-O3 -Wno-unknown-pragmas -Wno-logical-not-parentheses")
 set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
+add_definitions(-DF77_COMPLEX_RET_INTEL)
 
 # OpenMP with libomp
 set(CMAKE_EXE_LINKER_FLAGS "-L/usr/local/lib -lomp") 

mVMCconfig.sh

@@ -40,7 +40,7 @@ F90 = frt
 FFLAGS = -DNDEBUG -DFUJITSU -Kfast
 CFLAGS = -DNDEBUG -Ofast -fopenmp
 CXXFLAGS = -DNDEBUG -Ofast -fopenmp
-CFLAGS += -D_lapack -D_pf_block_update # -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS += -DBLAS_EXTERNAL
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -74,7 +74,7 @@ F90 = ifort
 FFLAGS = -O3 -implicitnone
 CFLAGS = -O3 -qopenmp -Wno-unknown-pragmas
 CXXFLAGS = -O3 -std=gnu++14 -fpic -qopenmp
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS += -DMKL -DBLAS_EXTERNAL -DF77_COMPLEX_RET_INTEL
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -100,7 +100,7 @@ F90 = ifort
 FFLAGS = -O3 -implicitnone
 CFLAGS = -O3 -qopenmp -Wno-unknown-pragmas
 CXXFLAGS = -O3 -std=gnu++14 -fpic -qopenmp
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS += -DMKL -DBLAS_EXTERNAL -DF77_COMPLEX_RET_INTEL
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -124,7 +124,7 @@ F90 = flang
 FFLAGS = -O3
 CFLAGS = -O3 -fopenmp
 CXXFLAGS = -O3 -fPIC
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS +=
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -148,7 +148,7 @@ F90 = gfortran
 FFLAGS = -O3 -fimplicit-none
 CFLAGS = -O3 -fopenmp
 CXXFLAGS = -O3 -fPIC
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS +=
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -172,7 +172,7 @@ F90 = gfortran
 FFLAGS = -O3 -fimplicit-none
 CFLAGS = -O3 -fopenmp
 CXXFLAGS = -O3
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS += -DMKL -DBLAS_EXTERNAL -DF77_COMPLEX_RET_INTEL
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -196,7 +196,7 @@ F90 = gfortran
 FFLAGS = -O3 -fimplicit-none
 CFLAGS = -O3 -fopenmp
 CXXFLAGS = -O3
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS +=
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -220,7 +220,7 @@ F90 = gfortran
 FFLAGS = -O3 -fimplicit-none
 CFLAGS = -O3 -fopenmp
 CXXFLAGS = -O3
-CFLAGS += -D_lapack -D_pf_block_update -D_pfaffine
+CFLAGS += -D_lapack -D_pf_block_update
 CXXFLAGS += # -DBLAS_EXTERNAL # -DF77_COMPLEX_RET_INTEL # For Apple.
 
 BLIS_ROOT = ${PWD}/src/blis-install
@@ -242,7 +242,6 @@ EOF
     cat src/make.sys
     cat src/make-ext.sys
     ln -s $PWD/src/make.sys src/StdFace/make.sys;
-    ln -s $PWD/src/make.sys src/pfaffine/make.inc;
     ln -s $PWD/src/make.sys src/pfapack/fortran/make.inc;
 
     cat > makefile <<EOF