Skip to content

Commit

Permalink
Integrate waLBerla library
Browse files Browse the repository at this point in the history
Replace the lattice-Boltzmann and electrokinetics features by a
new implementation based on waLBerla, using PyStencils and LbmPy.

Co-authored-by: Rudolf Weeber <[email protected]>
Co-authored-by: Jean-Noël Grad <[email protected]>
Co-authored-by: Alexander Reinauer <[email protected]>
Co-authored-by: Ingo Tischler <[email protected]>
Co-authored-by: Sebastian Bindgen <[email protected]>
Co-authored-by: Christoph Lohrmann <[email protected]>
Co-authored-by: Patrick Kreissl <[email protected]>
Co-authored-by: RiccardoFrenner <[email protected]>
Co-authored-by: stekajack <[email protected]>
Co-authored-by: Kai Szuttor <[email protected]>
Co-authored-by: capomav <[email protected]>
  • Loading branch information
11 people committed May 12, 2023
1 parent b3efb66 commit 3fd1709
Show file tree
Hide file tree
Showing 434 changed files with 59,218 additions and 19,157 deletions.
3 changes: 3 additions & 0 deletions .codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,6 @@ ignore:
- "doc/tutorials"
- "samples"
- "maintainer"
- "src/walberla_bridge/**/generated_kernels/*"
- "src/walberla_bridge/myintrin.h"
- "src/walberla_bridge/philox_rand.h"
22 changes: 19 additions & 3 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ style_doxygen:
- mkdir build
- cd build
- cp ../maintainer/configs/maxset.hpp myconfig.hpp
- cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON
- cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_FFT=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON
- sh ../maintainer/CI/dox_warnings.sh
tags:
- espresso
Expand All @@ -87,13 +87,15 @@ default:
myconfig: 'default'
with_coverage: 'true'
with_scafacos: 'true'
with_walberla: 'true'
with_stokesian_dynamics: 'true'
check_skip_long: 'true'
script:
- bash maintainer/CI/build_cmake.sh
tags:
- espresso
- no-cuda
- numa

maxset:
<<: *global_job_definition
Expand All @@ -107,6 +109,7 @@ maxset:
myconfig: 'maxset'
with_coverage: 'true'
with_scafacos: 'true'
with_walberla: 'true'
with_stokesian_dynamics: 'true'
check_skip_long: 'true'
cmake_params: '-D ESPRESSO_TEST_NP=8'
Expand All @@ -116,6 +119,7 @@ maxset:
- espresso
- no-cuda
- numa
- avx2

no_rotation:
<<: *global_job_definition
Expand Down Expand Up @@ -169,6 +173,7 @@ clang-sanitizer:
with_asan: 'true'
with_ubsan: 'true'
with_scafacos: 'true'
with_walberla: 'true'
with_stokesian_dynamics: 'true'
script:
- bash maintainer/CI/build_cmake.sh
Expand Down Expand Up @@ -210,6 +215,7 @@ cuda11-coverage:
with_coverage: 'true'
check_skip_long: 'true'
with_scafacos: 'true'
with_walberla: 'true'
with_stokesian_dynamics: 'true'
script:
- bash maintainer/CI/build_cmake.sh
Expand All @@ -232,6 +238,8 @@ cuda11-maxset:
test_timeout: '900'
srcdir: '${CI_PROJECT_DIR}'
with_scafacos: 'true'
with_walberla: 'true'
with_walberla_avx: 'true'
with_stokesian_dynamics: 'true'
script:
- bash maintainer/CI/build_cmake.sh
Expand All @@ -256,6 +264,8 @@ tutorials-samples-maxset:
with_coverage: 'false'
with_coverage_python: 'true'
with_scafacos: 'true'
with_walberla: 'true'
with_walberla_avx: 'true'
with_stokesian_dynamics: 'true'
make_check_unit_tests: 'false'
make_check_python: 'false'
Expand All @@ -281,6 +291,8 @@ tutorials-samples-default:
with_cuda: 'true'
with_coverage: 'false'
with_scafacos: 'true'
with_walberla: 'true'
with_walberla_avx: 'true'
make_check_unit_tests: 'false'
make_check_python: 'false'
make_check_tutorials: 'true'
Expand All @@ -306,14 +318,14 @@ tutorials-samples-empty:
myconfig: 'empty'
with_cuda: 'true'
with_coverage: 'false'
with_scafacos: 'true'
with_scafacos: 'false'
with_walberla: 'false'
make_check_unit_tests: 'false'
make_check_python: 'false'
make_check_tutorials: 'true'
make_check_samples: 'true'
make_check_benchmarks: 'true'
test_timeout: '1200'
with_scafacos: 'false'
script:
- bash maintainer/CI/build_cmake.sh
tags:
Expand All @@ -334,6 +346,8 @@ tutorials-samples-no-gpu:
with_cuda: 'true'
with_coverage: 'false'
with_scafacos: 'true'
with_walberla: 'true'
with_walberla_avx: 'true'
make_check_unit_tests: 'false'
make_check_python: 'false'
make_check_tutorials: 'true'
Expand Down Expand Up @@ -362,6 +376,7 @@ installation:
make_check_unit_tests: 'false'
make_check_python: 'false'
with_scafacos: 'true'
with_walberla: 'true'
with_stokesian_dynamics: 'true'
srcdir: '${CI_PROJECT_DIR}'
build_type: 'Release'
Expand Down Expand Up @@ -396,6 +411,7 @@ empty:
with_cuda: 'true'
with_static_analysis: 'true'
with_scafacos: 'false'
with_walberla: 'false'
with_stokesian_dynamics: 'false'
with_coverage: 'false'
with_coverage_python: 'true'
Expand Down
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ repos:
entry: sh maintainer/format/clang-format.sh
language: system
always_run: false
files: '.*\.(cpp|hpp|cu|cuh)'
exclude: '^libs/'
files: '.*\.(cpp|hpp|h|cu|cuh)'
exclude: '^libs/|^src/walberla_bridge/src/.*/generated_kernels/.*\.(cpp|cu)|^maintainer/walberla_kernels/templates/.*\.tmpl\.(cpp|hpp|h|cu|cuh)'
args: ["-i", "-style=file"]

- id: autopep8
Expand Down
84 changes: 84 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ include(FeatureSummary)
project(ESPResSo)
include(GNUInstallDirs)
include(espresso_option_enum)
include(espresso_enable_avx2_support)
if(POLICY CMP0074)
# make find_package() use <PackageName>_ROOT variables
cmake_policy(SET CMP0074 NEW)
Expand Down Expand Up @@ -92,6 +93,11 @@ option(ESPRESSO_BUILD_TESTS "Enable tests" ON)
option(ESPRESSO_BUILD_WITH_SCAFACOS "Build with ScaFaCoS support" OFF)
option(ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS "Build with Stokesian Dynamics"
OFF)
option(ESPRESSO_BUILD_WITH_WALBERLA
"Build with waLBerla lattice-Boltzmann support" OFF)
option(ESPRESSO_BUILD_WITH_WALBERLA_AVX
"Build waLBerla lattice-Boltzmann with AVX vectorization" OFF)
option(ESPRESSO_BUILD_WITH_WALBERLA_FFT "Build waLBerla with FFT support" OFF)
option(ESPRESSO_BUILD_BENCHMARKS "Enable benchmarks" OFF)
option(ESPRESSO_BUILD_WITH_VALGRIND_MARKERS
"Build with valgrind instrumentation markers" OFF)
Expand Down Expand Up @@ -171,6 +177,15 @@ foreach(func_name __PRETTY_FUNCTION__ __FUNCSIG__ __FUNCTION__)
endif(result${func_name})
endforeach()

#
# AVX2 support
#

include(CheckCXXCompilerFlag)

add_library(espresso_avx_flags INTERFACE)
add_library(espresso::avx_flags ALIAS espresso_avx_flags)

#
# Interface libraries
#
Expand Down Expand Up @@ -573,6 +588,75 @@ if(ESPRESSO_BUILD_BENCHMARKS)
add_subdirectory(maintainer/benchmarks)
endif()

#
# waLBerla
#

if(ESPRESSO_BUILD_WITH_WALBERLA)
# cmake-format: off
include(FetchContent)
FetchContent_Declare(
walberla
GIT_REPOSITORY https://i10git.cs.fau.de/walberla/walberla.git
GIT_TAG 065ce5f311850371a97ac4766f47dbb5ca8424ba
)
# workaround for https://gitlab.kitware.com/cmake/cmake/-/issues/21146
if(NOT DEFINED walberla_SOURCE_DIR OR NOT EXISTS "${walberla_SOURCE_DIR}")
FetchContent_Populate(walberla)
endif()
# cmake-format: on
string(REGEX REPLACE "([/\\]walberla)-src$" "\\1-build" walberla_BINARY_DIR
"${walberla_SOURCE_DIR}")
set(WALBERLA_BUILD_TESTS off CACHE BOOL "")
set(WALBERLA_BUILD_BENCHMARKS off CACHE BOOL "")
set(WALBERLA_BUILD_TOOLS off CACHE BOOL "")
set(WALBERLA_BUILD_TUTORIALS off CACHE BOOL "")
set(WALBERLA_BUILD_SHOWCASES off CACHE BOOL "")
set(WALBERLA_BUILD_DOC off CACHE BOOL "")
set(WALBERLA_LOGLEVEL "WARNING" CACHE STRING "")
set(CMAKE_POSITION_INDEPENDENT_CODE on CACHE BOOL "")
if(ESPRESSO_BUILD_WITH_CUDA)
set(WALBERLA_BUILD_WITH_CUDA "on" CACHE BOOL "")
if(CMAKE_VERSION VERSION_LESS 3.25 OR NOT ESPRESSO_CUDA_COMPILER STREQUAL
"clang")
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES 75)
endif()
endif()
endif()
if(ESPRESSO_BUILD_WITH_WALBERLA_FFT)
set(ESPRESSO_USE_WALBERLA_FFT 1)
set(WALBERLA_BUILD_WITH_FFTW on CACHE BOOL "")
else()
set(WALBERLA_BUILD_WITH_FFTW off CACHE BOOL "")
endif()
set(WALBERLA_BUILD_WITH_FASTMATH off CACHE BOOL "")
add_subdirectory("${walberla_SOURCE_DIR}" "${walberla_BINARY_DIR}")
set(WALBERLA_LIBS
walberla::core walberla::domain_decomposition walberla::blockforest
walberla::boundary walberla::field walberla::lbm walberla::timeloop
walberla::vtk)
if(ESPRESSO_BUILD_WITH_WALBERLA_FFT)
set(WALBERLA_LIBS ${WALBERLA_LIBS} walberla::fft)
endif()
if(ESPRESSO_BUILD_WITH_CUDA AND WALBERLA_BUILD_WITH_CUDA)
set(WALBERLA_LIBS ${WALBERLA_LIBS} walberla::cuda)
endif()
# workaround for https://gitlab.kitware.com/cmake/cmake/-/issues/21283
foreach(target_w_namespace IN LISTS WALBERLA_LIBS)
string(REPLACE "walberla::" "" target_wo_namespace ${target_w_namespace})
add_library(${target_w_namespace} ALIAS ${target_wo_namespace})
endforeach()
if(ESPRESSO_BUILD_WITH_WALBERLA_AVX)
function(espresso_avx_flags_callback COMPILER_AVX2_FLAG)
target_compile_options(
espresso_avx_flags INTERFACE "${COMPILER_AVX2_FLAG}"
"-DESPRESSO_BUILD_WITH_AVX_KERNELS")
endfunction()
espresso_enable_avx2_support(espresso_avx_flags_callback)
endif()
endif()

#
# Subdirectories
#
Expand Down
9 changes: 8 additions & 1 deletion cmake/FindFFTW3.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,21 @@ endif(FFTW3_INCLUDE_DIR)

find_path(FFTW3_INCLUDE_DIR fftw3.h)
find_library(FFTW3_LIBRARIES NAMES fftw3)
find_path(FFTW3_MPI_INCLUDE_DIR fftw3-mpi.h)
find_library(FFTW3_MPI_LIBRARIES NAMES fftw3_mpi)

# handle the QUIETLY and REQUIRED arguments and set FFTW3_FOUND to TRUE if all
# listed variables are TRUE
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(FFTW3 DEFAULT_MSG FFTW3_LIBRARIES
FFTW3_INCLUDE_DIR)
set(FPHSA_NAME_MISMATCHED 1)
find_package_handle_standard_args(FFTW3_MPI DEFAULT_MSG FFTW3_MPI_LIBRARIES
FFTW3_MPI_INCLUDE_DIR)
unset(FPHSA_NAME_MISMATCHED)

mark_as_advanced(FFTW3_LIBRARIES FFTW3_INCLUDE_DIR FFTW3_MPI_LIBRARIES FFTW3_MPI_INCLUDE_DIR)

mark_as_advanced(FFTW3_LIBRARIES FFTW3_INCLUDE_DIR)

if(FFTW3_FOUND AND NOT TARGET FFTW3::FFTW3)
add_library(FFTW3::FFTW3 INTERFACE IMPORTED)
Expand Down
4 changes: 4 additions & 0 deletions cmake/espresso_cmake_config.cmakein
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@

#cmakedefine ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS

#cmakedefine ESPRESSO_BUILD_WITH_WALBERLA

#cmakedefine ESPRESSO_BUILD_WITH_WALBERLA_FFT

#cmakedefine ESPRESSO_BUILD_WITH_VALGRIND_MARKERS

#define PACKAGE_NAME "${PROJECT_NAME}"
Expand Down
67 changes: 67 additions & 0 deletions cmake/espresso_enable_avx2_support.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#
# Copyright (C) 2022-2023 The ESPResSo project
#
# This file is part of ESPResSo.
#
# ESPResSo is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ESPResSo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

function(espresso_enable_avx2_support callback)
set(COMPILER_AVX2_FLAG "")
foreach(FLAG_NAME "-mavx2" "/arch:AVX2")
string(REGEX REPLACE "[^0-9A-Za-z_]" "_" FLAG_VARIABLE "${FLAG_NAME}")
check_cxx_compiler_flag("${flag_name}"
COMPILER_HAS_${FLAG_VARIABLE}_FLAG_RESULT)
if(COMPILER_HAS_${FLAG_VARIABLE}_FLAG_RESULT)
set(COMPILER_AVX2_FLAG "${FLAG_NAME}")
cmake_language(CALL ${callback} "${COMPILER_AVX2_FLAG}")
break()
endif()
endforeach()
if(COMPILER_AVX2_FLAG STREQUAL "")
message(
FATAL_ERROR
"${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} doesn't support AVX2-specific compiler flags."
)
endif()
if(NOT COMPILER_AVX2_FLAG STREQUAL "/arch:AVX2")
execute_process(
COMMAND ${CMAKE_CXX_COMPILER} -march=native -E -v - INPUT_FILE /dev/null
OUTPUT_VARIABLE MARCH_NATIVE_OUTPUT_STRING
ERROR_VARIABLE MARCH_NATIVE_OUTPUT_STRING)
if(NOT "${MARCH_NATIVE_OUTPUT_STRING}" MATCHES "[ \n](\\+avx2|-mavx2|-D__AVX2__)[ \n]")
message(
FATAL_ERROR
"AVX2 not supported on this CPU architecture according to ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}. While ESPResSo will still compile, you will trigger SIGILL when calling AVX functions."
)
endif()
endif()
set(CMAKE_REQUIRED_FLAGS_BACKUP "${CMAKE_REQUIRED_FLAGS}")
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${COMPILER_AVX2_FLAG}")
check_cxx_source_compiles(
"#include <immintrin.h>
__m256i xi_i = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
__m256 xi_s = _mm256_set_ps(0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f);
__m256d xi_d = _mm256_set_pd(0.0, 1.0, 2.0, 3.0);
int main() {}
" COMPILER_HAS_AVX2_SUPPORT)
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS_BACKUP}")
if(NOT COMPILER_HAS_AVX2_SUPPORT)
message(
FATAL_ERROR
"Cannot execute a simple AVX2 program compiled by ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}."
)
endif()
endfunction()

Loading

0 comments on commit 3fd1709

Please sign in to comment.