diff --git a/scripts_future_API/Project.toml b/scripts_future_API/Project.toml index bd7e78ad..836ea93d 100644 --- a/scripts_future_API/Project.toml +++ b/scripts_future_API/Project.toml @@ -1,4 +1,5 @@ [deps] +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FastIce = "e0de9f13-a007-490e-b696-b07d031015ca" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" diff --git a/scripts_future_API/run_stokes3D.sh b/scripts_future_API/run_stokes3D.sh new file mode 100755 index 00000000..996cbb95 --- /dev/null +++ b/scripts_future_API/run_stokes3D.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +module load LUMI/22.08 +module load partition/G +module load rocm/5.3.3 + +export MPICH_GPU_SUPPORT_ENABLED=1 + +export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so + +julia --project -O3 tm_stokes_mpi_wip.jl diff --git a/scripts_future_API/submit_stokes3D.sh b/scripts_future_API/submit_stokes3D.sh new file mode 100755 index 00000000..b02d34dc --- /dev/null +++ b/scripts_future_API/submit_stokes3D.sh @@ -0,0 +1,12 @@ +#!/bin/bash -l +#SBATCH --job-name="FastIce3D" +#SBATCH --output=FastIce3D.%j.o +#SBATCH --error=FastIce3D.%j.e +#SBATCH --time=00:30:00 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=8 +#SBATCH --gpus-per-node=8 +#SBATCH --partition=standard-g +#SBATCH --account project_465000557 + +srun --cpu-bind=map_cpu:49,57,17,25,1,9,33,41 ./run_stokes3D.sh diff --git a/scripts_future_API/tm_stokes_mpi_wip.jl b/scripts_future_API/tm_stokes_mpi_wip.jl index a4b7e27a..c44098bd 100644 --- a/scripts_future_API/tm_stokes_mpi_wip.jl +++ b/scripts_future_API/tm_stokes_mpi_wip.jl @@ -13,16 +13,23 @@ const TBC = BoundaryCondition{Traction} const SBC = BoundaryCondition{Slip} using KernelAbstractions +using AMDGPU -using GLMakie +using CairoMakie using FastIce.Distributed using MPI +println("import done") + +@views avx(A) = 0.5 .* (A[1:end-1, :, :] .+ A[2:end, :, :]) +@views avy(A) = 0.5 .* (A[:, 1:end-1, :] .+ A[:, 2:end, :]) +@views avz(A) = 0.5 .* (A[:, :, 1:end-1] .+ A[:, :, 2:end]) + MPI.Init() -backend = CPU() -dims = (2, 0, 2) +backend = ROCBackend() +dims = (0, 0, 0) arch = Architecture(backend, dims, MPI.COMM_WORLD) # physics @@ -30,7 +37,7 @@ ebg = 1.0 topo = details(arch) -size_l = (64, 64, 64) +size_l = (128, 128, 128) size_g = global_grid_size(topo, size_l) if global_rank(topo) == 0 @@ -38,28 +45,28 @@ if global_rank(topo) == 0 @show size_g end -grid_g = CartesianGrid(; origin=(-1.0, -0.5, 0.0), - extent=(2.0, 1.0, 2.0), +grid_g = CartesianGrid(; origin=(-1.0, -1.0, 0.0), + extent=(2.0, 2.0, 2.0), size=size_g) grid_l = local_grid(grid_g, topo) -psh_x(x, _, _) = -x * ebg -psh_y(_, y, _) = y * ebg +psh_x(x, _, _, ebg) = -x * ebg +psh_y(_, y, _, ebg) = y * ebg -x_bc = BoundaryFunction(psh_x; reduce_dims=false) -y_bc = BoundaryFunction(psh_y; reduce_dims=false) +x_bc = BoundaryFunction(psh_x; reduce_dims=false, parameters=(ebg,)) +y_bc = BoundaryFunction(psh_y; reduce_dims=false, parameters=(ebg,)) boundary_conditions = (x = (VBC(x_bc, y_bc, 0.0), VBC(x_bc, y_bc, 0.0)), y = (VBC(x_bc, y_bc, 0.0), VBC(x_bc, y_bc, 0.0)), z = (SBC(0.0, 0.0, 0.0), TBC(0.0, 0.0, 0.0))) # numerics -nt = 1000 -nviz = 10 +nt = 50maximum(size(grid_g)) +ncheck = 1maximum(size(grid_g)) r = 0.7 -re_mech = 10π +re_mech = 5π lτ_re_m = minimum(extent(grid_g)) / re_mech vdτ = minimum(spacing(grid_g)) / sqrt(ndims(grid_g) * 3.1) θ_dτ = lτ_re_m * (r + 4 / 3) / vdτ @@ -83,6 +90,8 @@ model = IsothermalFullStokesModel(; iter_params, other_fields) +println("model created") + if global_rank(topo) == 0 fig = Figure(; resolution=(1200, 1000), fontsize=32) axs = (Pr=Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"), @@ -95,11 +104,6 @@ if global_rank(topo) == 0 Vy_g = zeros(size(grid_g)) Vz_g = zeros(size(grid_g)) - Pr_v = zeros(size(grid_l)) - Vx_v = zeros(size(grid_l)) - Vy_v = zeros(size(grid_l)) - Vz_v = zeros(size(grid_l)) - plt = (Pr=heatmap!(axs.Pr, xcenters(grid_g), zcenters(grid_g), @view Pr_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo), Vx=heatmap!(axs.Vx, xcenters(grid_g), zcenters(grid_g), @view Vx_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo), Vy=heatmap!(axs.Vy, xcenters(grid_g), zcenters(grid_g), @view Vy_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo), @@ -115,6 +119,11 @@ else Vz_g = nothing end +Pr_v = zeros(size(grid_l)) +Vx_v = zeros(size(grid_l)) +Vy_v = zeros(size(grid_l)) +Vz_v = zeros(size(grid_l)) + # set!(model.fields.Pr, 0.0) # foreach(x -> set!(x, 0.0), model.fields.τ) @@ -124,8 +133,8 @@ foreach(x -> fill!(parent(x), 0.0), model.fields.V) KernelLaunch.apply_all_boundary_conditions!(arch, grid_l, model.boundary_conditions.stress) -set!(model.fields.V.x, grid_l, psh_x) -set!(model.fields.V.y, grid_l, psh_y) +set!(model.fields.V.x, grid_l, psh_x; parameters=(ebg,)) +set!(model.fields.V.y, grid_l, psh_y; parameters=(ebg,)) set!(model.fields.V.z, 0.0) KernelLaunch.apply_all_boundary_conditions!(arch, grid_l, model.boundary_conditions.velocity) @@ -137,29 +146,36 @@ if global_rank(topo) == 0 display(fig) end +@info "action" + for it in 1:nt advance_iteration!(model, 0.0, 1.0; async=false) - if it % nviz == 0 - @views avx(A) = 0.5.*(A[1:end-1,:,:] .+ A[2:end,:,:]) - @views avy(A) = 0.5.*(A[:,1:end-1,:] .+ A[:,2:end,:]) - @views avz(A) = 0.5.*(A[:,:,1:end-1] .+ A[:,:,2:end]) - - comm = cartesian_communicator(topo) - - gather!(Pr_g, interior(model.fields.Pr), comm) - gather!(Vx_g, avx(interior(model.fields.V.x)), comm) - gather!(Vy_g, avy(interior(model.fields.V.y)), comm) - gather!(Vz_g, avz(interior(model.fields.V.z)), comm) - if global_rank(topo) == 0 - plt.Pr[3][] = @view Pr_g[:, size(grid_g, 2)÷2, :] - plt.Vx[3][] = @view Vx_g[:, size(grid_g, 2)÷2, :] - plt.Vy[3][] = @view Vy_g[:, size(grid_g, 2)÷2, :] - plt.Vz[3][] = @view Vz_g[:, size(grid_g, 2)÷2, :] - yield() - end + if (it % ncheck == 0) && (global_rank(topo) == 0) + println("iter/nx = $(it/maximum(size(grid_g)))") end end -sleep(120) +comm = cartesian_communicator(topo) + +copyto!(Pr_v, interior(model.fields.Pr)) +copyto!(Vx_v, avx(interior(model.fields.V.x))) +copyto!(Vy_v, avy(interior(model.fields.V.y))) +copyto!(Vz_v, avz(interior(model.fields.V.z))) + +KernelAbstractions.synchronize(backend) + +gather!(Pr_g, Pr_v, comm) +gather!(Vx_g, Vx_v, comm) +gather!(Vy_g, Vy_v, comm) +gather!(Vz_g, Vz_v, comm) + +if global_rank(topo) == 0 + plt.Pr[3][] = @view Pr_g[:, size(grid_g, 2)÷2, :] + plt.Vx[3][] = @view Vx_g[:, size(grid_g, 2)÷2, :] + plt.Vy[3][] = @view Vy_g[:, size(grid_g, 2)÷2, :] + plt.Vz[3][] = @view Vz_g[:, size(grid_g, 2)÷2, :] + + save("stokes.png", fig) +end MPI.Finalize()