Update scripts

scripts_future_API/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 FastIce = "e0de9f13-a007-490e-b696-b07d031015ca"
 GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"

scripts_future_API/run_stokes3D.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+module load LUMI/22.08
+module load partition/G
+module load rocm/5.3.3
+
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so
+
+julia --project -O3 tm_stokes_mpi_wip.jl

scripts_future_API/submit_stokes3D.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -l
+#SBATCH --job-name="FastIce3D"
+#SBATCH --output=FastIce3D.%j.o
+#SBATCH --error=FastIce3D.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=8
+#SBATCH --ntasks-per-node=8
+#SBATCH --gpus-per-node=8
+#SBATCH --partition=standard-g
+#SBATCH --account project_465000557
+
+srun --cpu-bind=map_cpu:49,57,17,25,1,9,33,41 ./run_stokes3D.sh

scripts_future_API/tm_stokes_mpi_wip.jl

@@ -13,53 +13,60 @@ const TBC = BoundaryCondition{Traction}
 const SBC = BoundaryCondition{Slip}
 
 using KernelAbstractions
+using AMDGPU
 
-using GLMakie
+using CairoMakie
 
 using FastIce.Distributed
 using MPI
 
+println("import done")
+
+@views avx(A) = 0.5 .* (A[1:end-1, :, :] .+ A[2:end, :, :])
+@views avy(A) = 0.5 .* (A[:, 1:end-1, :] .+ A[:, 2:end, :])
+@views avz(A) = 0.5 .* (A[:, :, 1:end-1] .+ A[:, :, 2:end])
+
 MPI.Init()
 
-backend = CPU()
-dims    = (2, 0, 2)
+backend = ROCBackend()
+dims    = (0, 0, 0)
 arch    = Architecture(backend, dims, MPI.COMM_WORLD)
 
 # physics
 ebg = 1.0
 
 topo = details(arch)
 
-size_l = (64, 64, 64)
+size_l = (128, 128, 128)
 size_g = global_grid_size(topo, size_l)
 
 if global_rank(topo) == 0
     @show dimensions(topo)
     @show size_g
 end
 
-grid_g = CartesianGrid(; origin=(-1.0, -0.5, 0.0),
-                       extent=(2.0, 1.0, 2.0),
+grid_g = CartesianGrid(; origin=(-1.0, -1.0, 0.0),
+                       extent=(2.0, 2.0, 2.0),
                        size=size_g)
 
 grid_l = local_grid(grid_g, topo)
 
-psh_x(x, _, _) = -x * ebg
-psh_y(_, y, _) = y * ebg
+psh_x(x, _, _, ebg) = -x * ebg
+psh_y(_, y, _, ebg) = y * ebg
 
-x_bc = BoundaryFunction(psh_x; reduce_dims=false)
-y_bc = BoundaryFunction(psh_y; reduce_dims=false)
+x_bc = BoundaryFunction(psh_x; reduce_dims=false, parameters=(ebg,))
+y_bc = BoundaryFunction(psh_y; reduce_dims=false, parameters=(ebg,))
 
 boundary_conditions = (x = (VBC(x_bc, y_bc, 0.0), VBC(x_bc, y_bc, 0.0)),
                        y = (VBC(x_bc, y_bc, 0.0), VBC(x_bc, y_bc, 0.0)),
                        z = (SBC(0.0, 0.0, 0.0), TBC(0.0, 0.0, 0.0)))
 
 # numerics
-nt   = 1000
-nviz = 10
+nt     = 50maximum(size(grid_g))
+ncheck = 1maximum(size(grid_g))
 
 r       = 0.7
-re_mech = 10π
+re_mech = 5π
 lτ_re_m = minimum(extent(grid_g)) / re_mech
 vdτ     = minimum(spacing(grid_g)) / sqrt(ndims(grid_g) * 3.1)
 θ_dτ    = lτ_re_m * (r + 4 / 3) / vdτ
@@ -83,6 +90,8 @@ model = IsothermalFullStokesModel(;
                                   iter_params,
                                   other_fields)
 
+println("model created")
+
 if global_rank(topo) == 0
     fig = Figure(; resolution=(1200, 1000), fontsize=32)
     axs = (Pr=Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"),
@@ -95,11 +104,6 @@ if global_rank(topo) == 0
     Vy_g = zeros(size(grid_g))
     Vz_g = zeros(size(grid_g))
 
-    Pr_v = zeros(size(grid_l))
-    Vx_v = zeros(size(grid_l))
-    Vy_v = zeros(size(grid_l))
-    Vz_v = zeros(size(grid_l))
-
     plt = (Pr=heatmap!(axs.Pr, xcenters(grid_g), zcenters(grid_g), @view Pr_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo),
            Vx=heatmap!(axs.Vx, xcenters(grid_g), zcenters(grid_g), @view Vx_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo),
            Vy=heatmap!(axs.Vy, xcenters(grid_g), zcenters(grid_g), @view Vy_g[:, size(grid_g, 2)÷2, :]; colormap=:turbo),
@@ -115,6 +119,11 @@ else
     Vz_g = nothing
 end
 
+Pr_v = zeros(size(grid_l))
+Vx_v = zeros(size(grid_l))
+Vy_v = zeros(size(grid_l))
+Vz_v = zeros(size(grid_l))
+
 # set!(model.fields.Pr, 0.0)
 # foreach(x -> set!(x, 0.0), model.fields.τ)
 
@@ -124,8 +133,8 @@ foreach(x -> fill!(parent(x), 0.0), model.fields.V)
 
 KernelLaunch.apply_all_boundary_conditions!(arch, grid_l, model.boundary_conditions.stress)
 
-set!(model.fields.V.x, grid_l, psh_x)
-set!(model.fields.V.y, grid_l, psh_y)
+set!(model.fields.V.x, grid_l, psh_x; parameters=(ebg,))
+set!(model.fields.V.y, grid_l, psh_y; parameters=(ebg,))
 set!(model.fields.V.z, 0.0)
 
 KernelLaunch.apply_all_boundary_conditions!(arch, grid_l, model.boundary_conditions.velocity)
@@ -137,29 +146,36 @@ if global_rank(topo) == 0
     display(fig)
 end
 
+@info "action"
+
 for it in 1:nt
     advance_iteration!(model, 0.0, 1.0; async=false)
-    if it % nviz == 0
-        @views avx(A) = 0.5.*(A[1:end-1,:,:] .+ A[2:end,:,:])
-        @views avy(A) = 0.5.*(A[:,1:end-1,:] .+ A[:,2:end,:])
-        @views avz(A) = 0.5.*(A[:,:,1:end-1] .+ A[:,:,2:end])
-
-        comm = cartesian_communicator(topo)
-
-        gather!(Pr_g, interior(model.fields.Pr), comm)
-        gather!(Vx_g, avx(interior(model.fields.V.x)), comm)
-        gather!(Vy_g, avy(interior(model.fields.V.y)), comm)
-        gather!(Vz_g, avz(interior(model.fields.V.z)), comm)
-        if global_rank(topo) == 0
-            plt.Pr[3][] = @view Pr_g[:, size(grid_g, 2)÷2, :]
-            plt.Vx[3][] = @view Vx_g[:, size(grid_g, 2)÷2, :]
-            plt.Vy[3][] = @view Vy_g[:, size(grid_g, 2)÷2, :]
-            plt.Vz[3][] = @view Vz_g[:, size(grid_g, 2)÷2, :]
-            yield()
-        end
+    if (it % ncheck == 0) && (global_rank(topo) == 0)
+        println("iter/nx = $(it/maximum(size(grid_g)))")
     end
 end
 
-sleep(120)
+comm = cartesian_communicator(topo)
+
+copyto!(Pr_v, interior(model.fields.Pr))
+copyto!(Vx_v, avx(interior(model.fields.V.x)))
+copyto!(Vy_v, avy(interior(model.fields.V.y)))
+copyto!(Vz_v, avz(interior(model.fields.V.z)))
+
+KernelAbstractions.synchronize(backend)
+
+gather!(Pr_g, Pr_v, comm)
+gather!(Vx_g, Vx_v, comm)
+gather!(Vy_g, Vy_v, comm)
+gather!(Vz_g, Vz_v, comm)
+
+if global_rank(topo) == 0
+    plt.Pr[3][] = @view Pr_g[:, size(grid_g, 2)÷2, :]
+    plt.Vx[3][] = @view Vx_g[:, size(grid_g, 2)÷2, :]
+    plt.Vy[3][] = @view Vy_g[:, size(grid_g, 2)÷2, :]
+    plt.Vz[3][] = @view Vz_g[:, size(grid_g, 2)÷2, :]
+
+    save("stokes.png", fig)
+end
 
 MPI.Finalize()