Skip to content

Commit

Permalink
Merge remote-tracking branch 'remotes/HPSCTerrSys/UseMaskNc' into CUD…
Browse files Browse the repository at this point in the history 
…A11_mgsemi_jacobian_fix_UseMaskNc

UPDATE oas3 interface to use clmgrid.nc for masking recv fields
The currently implemented oas3 interface used by TSMP only
  masks rcv fields from CLM35 by inactive or active parflow
  cells. This only works if ParFlow and CLM share the same
  land-water mask (active and inactive cells). To work around
  this, it is now implemented that ParFlow additionally masks
  rcv fields using the mask that is shipped with clmgrid.nc to
  ensure that all inactive pixels from CLM are also masked out in
  ParFlow. (by niklaswr)
  • Loading branch information
@AGonzalezNicolas
AGonzalezNicolas committed Mar 13, 2024
2 parents 049f768 + b31fe58 commit da53878
Show file tree
Hide file tree
Showing 69 changed files with 3,310 additions and 1,010 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ jobs:
if: matrix.config.backend == 'kokkos'
run: |
if [[ "$CACHE_HIT" != 'true' ]]; then
URL=https://github.com/kokkos/kokkos/archive/refs/tags/3.3.01.tar.gz
URL=https://github.com/kokkos/kokkos/archive/refs/tags/4.2.01.tar.gz
mkdir kokkos
cd kokkos
curl -L $URL | tar --strip-components=1 -xz
Expand Down
12 changes: 12 additions & 0 deletions CONTRIBUTORS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,15 @@ Kitware, Inc
**Jaro Hokkanen**
IBG-3, Forschungszentrum Juelich,
Juelich, Germany

**George Artavanis**
Princeton Unversity
[email protected]

**Chen Yang**
Princeton University
[email protected]

**Benjamin West**
University of Arizona
[email protected]
4 changes: 2 additions & 2 deletions README-Spack.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ compile the GNU compiler suite under Spack and avoid using Clang.
Building GCC will take considerable amount of time.

```shell
spack install gcc@10.3.0
spack install gcc@12.2.0
```

Add the compiler to the set of compilers Spack can use:

```shell
spack load gcc@10.3.0
spack load gcc@12.2.0
spack compiler find
spack compilers
```
Expand Down
2 changes: 1 addition & 1 deletion docs/user_manual/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Welcome to the ParFlow User's Manual

**Reed M. Maxwell** [1]_, **Stefan J. Kollet** [2]_, **Laura E. Condon** [3]_,
**Steven G. Smith** [4]_, **Carol S. Woodward** [5]_, **Robert D. Falgout** [6]_,
**Ian M. Ferguson** [7]_, **Nicholas Engdahl** [8]_, **Jaro Hokkanen** [9]_, **Basile Hector** [10]_, **James Gilbert** [11]_, **Lindsay Bearup** [7]_,
**Ian M. Ferguson** [7]_, **Nicholas Engdahl** [8]_, **Jaro Hokkanen** [9]_, **George Artavanis** [1]_, **Benjamin West** [3]_, **Chen Yang** [1]_, **Basile Hector** [10]_, **James Gilbert** [11]_, **Lindsay Bearup** [7]_,
**Jennifer Jefferson** [12]_, **Chuck Baldwin**, **William J. Bosl** [13]_,
**Richard Hornung** [4]_, **Steven Ashby** [14]_, **Ketan B. Kulkarni** [15]_

Expand Down
19 changes: 5 additions & 14 deletions pfsimulator/amps/mpi1/amps_gpupacking.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,6 @@ void kokkosMemCpyDeviceToDevice(char *dest, char *src, size_t size){
Kokkos::deep_copy(dest_view, src_view);
}


/**
* @brief Device-host memcopy
*
Expand All @@ -240,7 +239,6 @@ void kokkosMemCpyDeviceToHost(char *dest, char *src, size_t size){
#endif
Kokkos::deep_copy(dest_view, src_view);
}

/**
* @brief Host-device memcopy
*
Expand All @@ -258,7 +256,6 @@ void kokkosMemCpyHostToDevice(char *dest, char *src, size_t size){
#endif
Kokkos::deep_copy(dest_view, src_view);
}

/**
* @brief Host-host memcopy
*
Expand Down Expand Up @@ -350,18 +347,14 @@ void amps_gpu_free_bufs(){
void amps_gpu_finalize(){
amps_gpu_free_bufs();
amps_gpu_destroy_streams();
if(Kokkos::is_initialized) Kokkos::finalize();
if(Kokkos::is_initialized() && !Kokkos::is_finalized()) Kokkos::finalize();
}

/**
* @brief Initialize Kokkos if not initialized
*/
void amps_kokkos_initialization(){
if(!Kokkos::is_initialized()){
Kokkos::InitArguments args;
args.ndevices = 1;
Kokkos::initialize(args);
}
if(!Kokkos::is_initialized()) Kokkos::initialize();
}

/**
Expand Down Expand Up @@ -625,11 +618,10 @@ int amps_gpupacking(int action, amps_Invoice inv, int inv_num, char **buffer_out
return __LINE__;
}
#endif

/* Run packing or unpacking kernel */
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >;
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{len_x, len_y, len_z}});

if(action == AMPS_PACK){
Kokkos::parallel_for(mdpolicy_3d, KOKKOS_LAMBDA(int i, int j, int k)
{
Expand Down Expand Up @@ -662,11 +654,10 @@ int amps_gpupacking(int action, amps_Invoice inv, int inv_num, char **buffer_out
});
inv->flags &= ~AMPS_PACKED;
}

pos += size;
ptr = ptr->next;
}

/* Check that the size is calculated right */
// if(pos != amps_sizeof_invoice(amps_CommWorld, inv)){
// printf("ERROR at %s:%d: The size does not match the invoice size\n", __FILE__, __LINE__);
Expand Down
23 changes: 20 additions & 3 deletions pfsimulator/amps/oas3/oas_pfl_define.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
!
INTEGER :: part_id ! ID returned by prism_def_partition_proto
INTEGER :: ii, jj, nn ! Local Variables
INTEGER, POINTER :: mask_land(:,:) ! Mask land

REAL(KIND=8), ALLOCATABLE :: lglon(:,:), &!
lglat(:,:) ! Global Grid Centres
Expand All @@ -84,7 +83,8 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
REAL(KIND=8), ALLOCATABLE :: clmlon(:,:), &!
clmlat(:,:) ! Global Grid Centres
INTEGER :: status, pflncid, &!
pflvarid(3) ! CPS increased to 3,Debug netcdf output
pflvarid(3), &! CPS increased to 3,Debug netcdf output
ib, npes
!------------------------------------------------------------------------------
!- End of header
!------------------------------------------------------------------------------
Expand All @@ -93,7 +93,24 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
!- Begin Subroutine oas_pfl_define
!------------------------------------------------------------------------------
!
!++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
! Read in land mask for each sub-domain to mask recv values from CLM
ALLOCATE( mask_land_sub(nx,ny), stat = ierror )
IF (ierror >0) CALL prism_abort_proto(comp_id, 'oas_pfl_define', 'Failure in allocating mask_land_sub')
CALL MPI_Comm_size(localComm, npes, ierror)
DO ib = 0,npes-1
IF (rank == ib ) THEN
status = nf90_open("clmgrid.nc", NF90_NOWRITE, pflncid)
status = nf90_inq_varid(pflncid, "LANDMASK" , pflvarid(3))
status = nf90_get_var(pflncid, pflvarid(3), mask_land_sub, &
start = (/ix+1, iy+1/), &
count = (/nx, ny/) )
status = nf90_close(pflncid)
mask_land_sub = mask_land_sub
ENDIF
ENDDO


!++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
!Global grid definition for OASIS3, written by master process for
!the component, i.e rank = 0
!+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Expand Down
2 changes: 2 additions & 0 deletions pfsimulator/amps/oas3/oas_pfl_vardef.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ MODULE oas_pfl_vardef
INTEGER, PUBLIC :: OASIS_Rcv = 1 ! return code if received field
INTEGER, PUBLIC :: OASIS_idle = 0 ! return code if nothing done by oasis

INTEGER, DIMENSION(:,:), ALLOCATABLE :: mask_land ! Mask land
INTEGER, DIMENSION(:,:), ALLOCATABLE :: mask_land_sub ! Mask land
!
REAL(KIND=8), DIMENSION(:,:), ALLOCATABLE :: bufz ! Temp buffer for field transfer
REAL(KIND=8) :: pfl_timestep, &! parflow time step in hrs
Expand Down
4 changes: 1 addition & 3 deletions pfsimulator/amps/oas3/receive_fld2_clm.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ SUBROUTINE receive_fld2_clm(evap_trans,topo,ix,iy,nx,ny,nz,nx_f,ny_f,pstep)

! All vecotrs from parflow on grid w/ ghost nodes for current proc
!Local Variables

INTEGER :: i, j, k, l
INTEGER :: isecs ! Parflow model time in seconds
INTEGER :: j_incr, k_incr ! convert 1D vector to 3D i,j,k array
Expand Down Expand Up @@ -96,7 +95,6 @@ SUBROUTINE receive_fld2_clm(evap_trans,topo,ix,iy,nx,ny,nz,nx_f,ny_f,pstep)
ENDDO
ENDDO

!
DO k = 1, nlevsoil
IF( trcv(k)%laction ) CALL oas_pfl_rcv( k, isecs, frcv(:,:,k),nx, ny, info )
ENDDO
Expand All @@ -105,7 +103,7 @@ SUBROUTINE receive_fld2_clm(evap_trans,topo,ix,iy,nx,ny,nz,nx_f,ny_f,pstep)
DO i = 1, nx
DO j = 1, ny
DO k = 1, nlevsoil
IF (topo_mask(i,j) .gt. 0) THEN !CPS mask bug fix
IF ((topo_mask(i,j) .gt. 0) .and. (mask_land_sub(i,j) .gt. 0)) THEN !CPS mask bug fix
l = 1+i + j_incr*(j) + k_incr*(topo_mask(i,j)-(k-1)) !
evap_trans(l) = frcv(i,j,k)
END IF
Expand Down
6 changes: 2 additions & 4 deletions pfsimulator/parflow_lib/pf_kokkos.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,14 @@ void kokkosMemSet(char *ptr, size_t size){
* @brief Initialize Kokkos
*/
void kokkosInit(){
Kokkos::InitArguments args;
args.ndevices = 1;
Kokkos::initialize(args);
if(!Kokkos::is_initialized()) Kokkos::initialize();
}

/**
* @brief Finalize Kokkos
*/
void kokkosFinalize(){
if(Kokkos::is_initialized) Kokkos::finalize();
if(Kokkos::is_initialized() && !Kokkos::is_finalized()) Kokkos::finalize();
}

}
26 changes: 13 additions & 13 deletions pfsimulator/parflow_lib/pf_kokkosloops.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ static const int FDIR_TABLE[6][3] = {
loop_body; \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
\
Expand Down Expand Up @@ -260,7 +260,7 @@ static const int FDIR_TABLE[6][3] = {
loop_body; \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
\
Expand Down Expand Up @@ -306,7 +306,7 @@ static const int FDIR_TABLE[6][3] = {
loop_body; \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
\
Expand Down Expand Up @@ -349,7 +349,7 @@ static const int FDIR_TABLE[6][3] = {
lambda_body(i, j, k, lsum); \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
typedef function_traits<decltype(lambda_body)> traits; \
if(std::is_same<traits::result_type, struct ReduceSumType<double>>::value) \
Expand Down Expand Up @@ -411,7 +411,7 @@ static const int FDIR_TABLE[6][3] = {
lambda_body(i, j, k, lsum); \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
typedef function_traits<decltype(lambda_body)> traits; \
if(std::is_same<traits::result_type, struct ReduceSumType<double>>::value) \
Expand Down Expand Up @@ -490,7 +490,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_nx, PV_ny, PV_nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -524,7 +524,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -596,7 +596,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_nx, PV_ny, PV_nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -637,7 +637,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -697,7 +697,7 @@ static const int FDIR_TABLE[6][3] = {
}; \
n_prev += nz * ny * nx; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -809,7 +809,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -867,7 +867,7 @@ static const int FDIR_TABLE[6][3] = {
loop_body; \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_diff_x, PV_diff_y, PV_diff_z}});\
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down Expand Up @@ -923,7 +923,7 @@ static const int FDIR_TABLE[6][3] = {
} \
}; \
\
using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >; \
MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}}); \
Kokkos::parallel_for(mdpolicy_3d, lambda_body); \
Kokkos::fence(); \
Expand Down
Loading

0 comments on commit da53878

Please sign in to comment.