From 8feee5fe1a8dd37eabca280af734e5db87b55318 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 24 Mar 2023 22:41:23 +0100 Subject: [PATCH 01/31] setup distributed driver --- Project.toml | 2 + test/_DistributedTests.jl | 203 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 test/_DistributedTests.jl diff --git a/Project.toml b/Project.toml index e0af04f..996afe3 100644 --- a/Project.toml +++ b/Project.toml @@ -8,9 +8,11 @@ AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" +GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MiniQhull = "978d7f02-9e05-4691-894f-ae31a51d76ca" +PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl new file mode 100644 index 0000000..6b5aabf --- /dev/null +++ b/test/_DistributedTests.jl @@ -0,0 +1,203 @@ +module DistributedPoissonTests + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapDistributed: DistributedDiscreteModel +using GridapDistributed: DistributedGridapType +using GridapDistributed: DistributedTriangulation +using GridapDistributed: DistributedFESpace +using GridapDistributed: generate_gids + +import Gridap.Geometry: Triangulation +import Gridap.Geometry: get_background_model +import GridapEmbedded.Interfaces: cut +import GridapEmbedded.Interfaces: Cutter +import GridapEmbedded.Interfaces: EmbeddedBoundary +import GridapEmbedded.AgFEM: aggregate +import GridapEmbedded.AgFEM: AgFEMSpace +import GridapDistributed: local_views + +struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType + discretizations::A + model::B + function DistributedEmbeddedDiscretization( + d::AbstractPData{<:EmbeddedDiscretization{Dp,T}}, + model::DistributedDiscreteModel) where {Dp,T} + A = typeof(d) + B = typeof(model) + new{Dp,T,A,B}(d,model) + end +end + +local_views(a::DistributedEmbeddedDiscretization) = a.discretizations + +get_background_model(a::DistributedEmbeddedDiscretization) = a.model + + +# TODO: do not compute ghost cells +# map_parts(remove_ghost_cells,local_views(Ω),local_views(gids)) + +function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + discretizations = map_parts(local_views(bgmodel)) do lmodel + cut(cutter,lmodel,args...) + end + DistributedEmbeddedDiscretization(discretizations,bgmodel) +end + +function cut(bgmodel::DistributedDiscreteModel,args...) + cutter = LevelSetCutter() + cut(cutter,bgmodel,args...) +end + +function Triangulation(cutgeo::DistributedEmbeddedDiscretization,args...) + trians = map_parts(local_views(cutgeo)) do lcutgeo + Triangulation(lcutgeo,args...) + end + bgmodel = get_background_model(cutgeo) + DistributedTriangulation(trians,bgmodel) +end + +function EmbeddedBoundary(cutgeo::DistributedEmbeddedDiscretization,args...) + trians = map_parts(local_views(cutgeo)) do lcutgeo + EmbeddedBoundary(lcutgeo,args...) + end + bgmodel = get_background_model(cutgeo) + DistributedTriangulation(trians,bgmodel) +end + +function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) + map_parts(local_views(cutgeo)) do lcutgeo + aggregate(stragegy,lcutgeo) + end + # TODO: parallel aggregation here +end + +function AgFEMSpace( + bgmodel::DistributedDiscreteModel, + f::DistributedFESpace, + bgcell_to_bgcellin::SequentialData{<:AbstractVector}, + g::DistributedFESpace=f) + + fagg = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) + gids = generate_gids(bgmodel,fagg) + vector_type = get_vector_type(f) + DistributedSingleFieldFESpace(fagg,gids,vector_type) +end + +function global_aggregates(bgmodel::DistributedDiscreteModel,aggregates) + gids = map_parts(get_lid_to_gid,local_views(get_cell_gids(bgmodel))) + map_parts(gids,aggregates) do gid,agg + map( i ->i == 0 ? 0 : gid[i], agg) + end +end + + +#gids = map_parts(get_lid_to_gid,local_views(get_cell_gids(bgmodel))) +#aggregates_gid = map_parts(aggregates,gids) do agg,gid +# map(Reindex(gid),agg) +#end +#_aggregates = PVector(aggregates_gid,get_cell_gids(bgmodel)) +#exchange(aggregates_gid,...) + +partition = (2,2) + +parts = get_part_ids(SequentialBackend(),partition) + +u(x) = x[1] - x[2] +f(x) = -Δ(u)(x) +ud(x) = u(x) + +R = 0.5 +L = 0.8*(2*R) +p0 = Point(0.0,0.0) + +geo = disk(R,x0=p0) + +t = 1.1 +pmin = p0-t*R +pmax = p0+t*R + +n = 10 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(parts,pmin,pmax,mesh_partition) +bgtrian = Triangulation(bgmodel) +writevtk(bgtrian,"bgtrian") + +dp = pmax - pmin +h = dp[1]/n + +cutgeo = cut(bgmodel,geo) + +strategy = AggregateAllCutCells() +aggregates = aggregate(strategy,cutgeo) + +Ω_bg = Triangulation(bgmodel) +Ω_act = Triangulation(cutgeo,ACTIVE) +Ω = Triangulation(cutgeo,PHYSICAL) +Γ = EmbeddedBoundary(cutgeo) + +writevtk(Ω,"trian") +writevtk(Ω_act,"trian_act") +n_Γ = get_normal_vector(Γ) + +order = 1 +degree = 2*order +dΩ = Measure(Ω,degree) +dΓ = Measure(Γ,degree) + +reffe = ReferenceFE(lagrangian,Float64,order) + +Vstd = FESpace(Ω_act,reffe) + +#V = AgFEMSpace(bgmodel,Vstd,aggregates) +V = Vstd +U = TrialFESpace(V) + + +const γd = 10.0 + +a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + +l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + +op = AffineFEOperator(a,l,U,V) +uh = solve(op) + +e = u - uh + +l2(u) = sqrt(sum( ∫( u*u )*dΩ )) +h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + +el2 = l2(e) +eh1 = h1(e) +ul2 = l2(uh) +uh1 = h1(uh) + +# TODO: include ghost/owner information into aggregates and colors +# +colors = map_parts(color_aggregates,aggregates,local_views(bgmodel)) +gids = get_cell_gids(bgmodel) +ohids = map_parts(get_lid_to_ohid,local_views(gids)) +oids = map_parts(i->findall(>(0),i),ohids) +_aggregates = map_parts(aggregates,oids) do agg,oid + map(Reindex(agg),oid) +end +_colors = map_parts(colors,oids) do col,oid + map(Reindex(col),oid) +end + +writevtk(Ω_bg,"trian",celldata=["aggregate"=>_aggregates,"color"=>_colors],cellfields=["uh"=>uh]) +writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) +writevtk(Γ,"trian_G") +@test el2/ul2 < 1.e-8 +@test eh1/uh1 < 1.e-7 + +end # module From 120981e1a993e51b3930bdf9f95978360b2118bd Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Mon, 27 Mar 2023 19:45:06 +0200 Subject: [PATCH 02/31] remove ghost cells from cut triangulations workaround for test passing --- test/_DistributedTests.jl | 63 +++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index 6b5aabf..274ac2f 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -20,6 +20,10 @@ import GridapEmbedded.Interfaces: EmbeddedBoundary import GridapEmbedded.AgFEM: aggregate import GridapEmbedded.AgFEM: AgFEMSpace import GridapDistributed: local_views +import GridapDistributed: remove_ghost_cells +using GridapEmbedded.Interfaces: SubFacetTriangulation +using Gridap.Geometry: AppendedTriangulation +using Gridap.Arrays struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType discretizations::A @@ -37,10 +41,6 @@ local_views(a::DistributedEmbeddedDiscretization) = a.discretizations get_background_model(a::DistributedEmbeddedDiscretization) = a.model - -# TODO: do not compute ghost cells -# map_parts(remove_ghost_cells,local_views(Ω),local_views(gids)) - function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) discretizations = map_parts(local_views(bgmodel)) do lmodel cut(cutter,lmodel,args...) @@ -95,13 +95,27 @@ function global_aggregates(bgmodel::DistributedDiscreteModel,aggregates) end end +function remove_ghost_cells(trian::DistributedTriangulation) + model = get_background_model(trian) + gids = get_cell_gids(model) + trians = map_parts(local_views(trian),local_views(gids)) do trian,gids + remove_ghost_cells(trian,gids) + end + DistributedTriangulation(trians,model) +end -#gids = map_parts(get_lid_to_gid,local_views(get_cell_gids(bgmodel))) -#aggregates_gid = map_parts(aggregates,gids) do agg,gid -# map(Reindex(gid),agg) -#end -#_aggregates = PVector(aggregates_gid,get_cell_gids(bgmodel)) -#exchange(aggregates_gid,...) +function remove_ghost_cells(trian::AppendedTriangulation,gids) + a = remove_ghost_cells(trian.a,gids) + b = remove_ghost_cells(trian.b,gids) + lazy_append(a,b) +end + +function remove_ghost_cells(trian::SubFacetTriangulation,gids) + model = get_background_model(trian) + D = num_cell_dims(model) + glue = get_glue(trian,Val{D}()) + remove_ghost_cells(glue,trian,gids) +end partition = (2,2) @@ -140,6 +154,12 @@ aggregates = aggregate(strategy,cutgeo) Ω = Triangulation(cutgeo,PHYSICAL) Γ = EmbeddedBoundary(cutgeo) +Γ = remove_ghost_cells(Γ) +Ω = remove_ghost_cells(Ω) + +t1 = get_part(local_views(Ω),1) +writevtk(t1,"t1") + writevtk(Ω,"trian") writevtk(Ω_act,"trian_act") n_Γ = get_normal_vector(Γ) @@ -181,20 +201,33 @@ eh1 = h1(e) ul2 = l2(uh) uh1 = h1(uh) -# TODO: include ghost/owner information into aggregates and colors # colors = map_parts(color_aggregates,aggregates,local_views(bgmodel)) -gids = get_cell_gids(bgmodel) -ohids = map_parts(get_lid_to_ohid,local_views(gids)) +cell_gids = get_cell_gids(bgmodel) +ohids = map_parts(get_lid_to_ohid,local_views(cell_gids)) +gids = map_parts(get_lid_to_gid,local_views(cell_gids)) oids = map_parts(i->findall(>(0),i),ohids) -_aggregates = map_parts(aggregates,oids) do agg,oid + +_aggregates = map_parts(aggregates,gids) do agg,gid + map(i-> i==0 ? 0 : gid[i],agg) +end +_aggregates = map_parts(_aggregates,oids) do agg,oid map(Reindex(agg),oid) end _colors = map_parts(colors,oids) do col,oid map(Reindex(col),oid) end +_gids = map_parts(gids,oids) do gid,oid + map(Reindex(gid),oid) +end + +writevtk(Ω_bg,"trian", + celldata=[ + "aggregate"=>_aggregates, + "color"=>_colors, + "gid"=>_gids], + cellfields=["uh"=>uh]) -writevtk(Ω_bg,"trian",celldata=["aggregate"=>_aggregates,"color"=>_colors],cellfields=["uh"=>uh]) writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) writevtk(Γ,"trian_G") @test el2/ul2 < 1.e-8 From 722a69816e548e2e1d90069429e5ab30bb87349e Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 31 Mar 2023 18:05:40 +0200 Subject: [PATCH 03/31] setup consistent dof owner --- src/AgFEM/AgFEMSpaces.jl | 13 ++++-- test/_DistributedTests.jl | 83 +++++++++++++++++++++++++++------------ 2 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/AgFEM/AgFEMSpaces.jl b/src/AgFEM/AgFEMSpaces.jl index b54cfeb..60bb74a 100644 --- a/src/AgFEM/AgFEMSpaces.jl +++ b/src/AgFEM/AgFEMSpaces.jl @@ -51,7 +51,8 @@ function _setup_agfem_constraints( acell_to_acellin, acell_to_dof_ids, acell_to_coeffs, - acell_to_proj) + acell_to_proj)#, +# acell_to_gcell) n_acells = length(acell_to_acellin) fdof_to_isagg = fill(true,n_fdofs) @@ -62,12 +63,16 @@ function _setup_agfem_constraints( acellin = acell_to_acellin[acell] iscut = acell != acellin dofs = getindex!(cache,acell_to_dof_ids,acell) + # gcell = acell_to_gcell[acell] for (ldof,dof) in enumerate(dofs) if dof > 0 fdof = dof - fdof_to_isagg[fdof] = iscut && fdof_to_isagg[fdof] - fdof_to_acell[fdof] = acell - fdof_to_ldof[fdof] = ldof + # gcell_dof = acell_to_gcell[fdof_to_acell[fdof]] + # if gcell > gcell_dof + fdof_to_acell[fdof] = acell + fdof_to_isagg[fdof] = iscut && fdof_to_isagg[fdof] + fdof_to_ldof[fdof] = ldof + # end end end end diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index 274ac2f..791ac93 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -1,4 +1,4 @@ -module DistributedPoissonTests +#module DistributedPoissonTests using Gridap using GridapEmbedded @@ -11,6 +11,7 @@ using GridapDistributed: DistributedGridapType using GridapDistributed: DistributedTriangulation using GridapDistributed: DistributedFESpace using GridapDistributed: generate_gids +using GridapDistributed: generate_cell_gids import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model @@ -21,8 +22,14 @@ import GridapEmbedded.AgFEM: aggregate import GridapEmbedded.AgFEM: AgFEMSpace import GridapDistributed: local_views import GridapDistributed: remove_ghost_cells +import GridapDistributed: dof_wise_to_cell_wise! +import GridapDistributed: dof_wise_to_cell_wise +import GridapDistributed: generate_gids +import GridapDistributed: add_ghost_cells using GridapEmbedded.Interfaces: SubFacetTriangulation using Gridap.Geometry: AppendedTriangulation +using Gridap.Geometry: FaceToFaceGlue +using Gridap.FESpaces: SingleFieldFESpace using Gridap.Arrays struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType @@ -42,6 +49,10 @@ local_views(a::DistributedEmbeddedDiscretization) = a.discretizations get_background_model(a::DistributedEmbeddedDiscretization) = a.model function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + # TODO: do not compute ghost cells + # - cut own_bgmodels + # - set (local) bgmodels + # - reindex bgcell/facet data discretizations = map_parts(local_views(bgmodel)) do lmodel cut(cutter,lmodel,args...) end @@ -76,23 +87,35 @@ function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) # TODO: parallel aggregation here end +#function AgFEMSpace( +# bgmodel::DistributedDiscreteModel, +# f::DistributedFESpace, +# bgcell_to_bgcellin::SequentialData{<:AbstractVector}, +# g::DistributedFESpace=f) +# +# fagg = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) +# gids = generate_gids(bgmodel,fagg) +# vector_type = get_vector_type(f) +# DistributedSingleFieldFESpace(fagg,gids,vector_type) +#end + + function AgFEMSpace( bgmodel::DistributedDiscreteModel, f::DistributedFESpace, bgcell_to_bgcellin::SequentialData{<:AbstractVector}, g::DistributedFESpace=f) - fagg = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) - gids = generate_gids(bgmodel,fagg) - vector_type = get_vector_type(f) - DistributedSingleFieldFESpace(fagg,gids,vector_type) -end - -function global_aggregates(bgmodel::DistributedDiscreteModel,aggregates) - gids = map_parts(get_lid_to_gid,local_views(get_cell_gids(bgmodel))) - map_parts(gids,aggregates) do gid,agg - map( i ->i == 0 ? 0 : gid[i], agg) - end + spaces = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) + trians = map_parts(get_triangulation,local_views(f)) + trian = DistributedTriangulation(trians,bgmodel) + trian = add_ghost_cells(trian) + trian_gids = generate_cell_gids(trian) + cell_to_ldofs = map_parts(get_cell_dof_ids,spaces) + nldofs = map_parts(num_free_dofs,spaces) + gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) + vector_type = _find_vector_type(spaces,gids) + DistributedSingleFieldFESpace(spaces,gids,vector_type) end function remove_ghost_cells(trian::DistributedTriangulation) @@ -117,6 +140,9 @@ function remove_ghost_cells(trian::SubFacetTriangulation,gids) remove_ghost_cells(glue,trian,gids) end + +# Driver + partition = (2,2) parts = get_part_ids(SequentialBackend(),partition) @@ -125,15 +151,20 @@ u(x) = x[1] - x[2] f(x) = -Δ(u)(x) ud(x) = u(x) -R = 0.5 -L = 0.8*(2*R) +L = 1 p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + +R = 0.35 geo = disk(R,x0=p0) -t = 1.1 -pmin = p0-t*R -pmax = p0+t*R +R = 0.15 +d = L/4 +geo1 = disk(R,x0=p0-d) +geo2 = disk(R,x0=p0+d) +#geo = !union(geo1,geo2) n = 10 mesh_partition = (n,n) @@ -147,6 +178,7 @@ h = dp[1]/n cutgeo = cut(bgmodel,geo) strategy = AggregateAllCutCells() +strategy = AggregateCutCellsByThreshold(0) aggregates = aggregate(strategy,cutgeo) Ω_bg = Triangulation(bgmodel) @@ -154,14 +186,15 @@ aggregates = aggregate(strategy,cutgeo) Ω = Triangulation(cutgeo,PHYSICAL) Γ = EmbeddedBoundary(cutgeo) + +writevtk(Ω_bg,"trian") + +writevtk(Ω,"trian_O") +writevtk(Γ,"trian_G") + Γ = remove_ghost_cells(Γ) Ω = remove_ghost_cells(Ω) -t1 = get_part(local_views(Ω),1) -writevtk(t1,"t1") - -writevtk(Ω,"trian") -writevtk(Ω_act,"trian_act") n_Γ = get_normal_vector(Γ) order = 1 @@ -225,12 +258,12 @@ writevtk(Ω_bg,"trian", celldata=[ "aggregate"=>_aggregates, "color"=>_colors, - "gid"=>_gids], - cellfields=["uh"=>uh]) + "gid"=>_gids])#, +# cellfields=["uh"=>uh]) writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) writevtk(Γ,"trian_G") @test el2/ul2 < 1.e-8 @test eh1/uh1 < 1.e-7 -end # module +# end # module From c4fd428cff4375c3b61c35648ddce477a5decbba Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Tue, 4 Apr 2023 12:41:50 +0200 Subject: [PATCH 04/31] setup distributed agfem spaces --- src/AgFEM/AgFEMSpaces.jl | 28 ++++++++++++++++--------- test/_DistributedTests.jl | 44 ++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/AgFEM/AgFEMSpaces.jl b/src/AgFEM/AgFEMSpaces.jl index 60bb74a..4c0f0c4 100644 --- a/src/AgFEM/AgFEMSpaces.jl +++ b/src/AgFEM/AgFEMSpaces.jl @@ -1,7 +1,12 @@ -function AgFEMSpace(f::SingleFieldFESpace,bgcell_to_bgcellin::AbstractVector,g::SingleFieldFESpace=f) +function AgFEMSpace( + f::SingleFieldFESpace, + bgcell_to_bgcellin::AbstractVector, + g::SingleFieldFESpace=f, + args...) + @assert get_triangulation(f) === get_triangulation(g) - AgFEMSpace(f,bgcell_to_bgcellin,get_fe_basis(g),get_fe_dof_basis(g)) + AgFEMSpace(f,bgcell_to_bgcellin,get_fe_basis(g),get_fe_dof_basis(g),args...) end # Note: cell is in fact bgcell in this function since f will usually be an ExtendedFESpace @@ -9,7 +14,8 @@ function AgFEMSpace( f::SingleFieldFESpace, bgcell_to_bgcellin::AbstractVector, shfns_g::CellField, - dofs_g::CellDof) + dofs_g::CellDof, + bgcell_to_gcell::AbstractVector=1:length(bgcell_to_bgcellin)) # Triangulation made of active cells trian_a = get_triangulation(f) @@ -21,6 +27,7 @@ function AgFEMSpace( bgcell_to_acell = glue.mface_to_tface acell_to_bgcellin = lazy_map(Reindex(bgcell_to_bgcellin),acell_to_bgcell) acell_to_acellin = collect(lazy_map(Reindex(bgcell_to_acell),acell_to_bgcellin)) + acell_to_gcell = lazy_map(Reindex(bgcell_to_gcell),acell_to_bgcell) # Build shape funs of g by replacing local funs in cut cells by the ones at the root # This needs to be done with shape functions in the physical domain @@ -41,7 +48,8 @@ function AgFEMSpace( acell_to_acellin, acell_to_dof_ids, acell_to_coeffs, - acell_to_proj) + acell_to_proj, + acell_to_gcell) FESpaceWithLinearConstraints(aggdof_to_fdof,aggdof_to_dofs,aggdof_to_coeffs,f) end @@ -51,8 +59,8 @@ function _setup_agfem_constraints( acell_to_acellin, acell_to_dof_ids, acell_to_coeffs, - acell_to_proj)#, -# acell_to_gcell) + acell_to_proj, + acell_to_gcell) n_acells = length(acell_to_acellin) fdof_to_isagg = fill(true,n_fdofs) @@ -63,16 +71,16 @@ function _setup_agfem_constraints( acellin = acell_to_acellin[acell] iscut = acell != acellin dofs = getindex!(cache,acell_to_dof_ids,acell) - # gcell = acell_to_gcell[acell] + gcell = acell_to_gcell[acell] for (ldof,dof) in enumerate(dofs) if dof > 0 fdof = dof - # gcell_dof = acell_to_gcell[fdof_to_acell[fdof]] - # if gcell > gcell_dof + acell_dof = fdof_to_acell[fdof] + if acell_dof == 0 || gcell > acell_to_gcell[acell_dof] fdof_to_acell[fdof] = acell fdof_to_isagg[fdof] = iscut && fdof_to_isagg[fdof] fdof_to_ldof[fdof] = ldof - # end + end end end end diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index 791ac93..f8d0f23 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -1,4 +1,4 @@ -#module DistributedPoissonTests +# module DistributedPoissonTests using Gridap using GridapEmbedded @@ -10,8 +10,10 @@ using GridapDistributed: DistributedDiscreteModel using GridapDistributed: DistributedGridapType using GridapDistributed: DistributedTriangulation using GridapDistributed: DistributedFESpace +using GridapDistributed: DistributedSingleFieldFESpace using GridapDistributed: generate_gids using GridapDistributed: generate_cell_gids +using GridapDistributed: _find_vector_type import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model @@ -87,31 +89,26 @@ function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) # TODO: parallel aggregation here end -#function AgFEMSpace( -# bgmodel::DistributedDiscreteModel, -# f::DistributedFESpace, -# bgcell_to_bgcellin::SequentialData{<:AbstractVector}, -# g::DistributedFESpace=f) -# -# fagg = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) -# gids = generate_gids(bgmodel,fagg) -# vector_type = get_vector_type(f) -# DistributedSingleFieldFESpace(fagg,gids,vector_type) -#end - - function AgFEMSpace( - bgmodel::DistributedDiscreteModel, - f::DistributedFESpace, - bgcell_to_bgcellin::SequentialData{<:AbstractVector}, - g::DistributedFESpace=f) - - spaces = map_parts(AgFEMSpace,local_views(f),bgcell_to_bgcellin,local_views(g)) + bgmodel::DistributedDiscreteModel, + f::DistributedFESpace, + bgcell_to_bgcellin::AbstractPData{<:AbstractVector}, + g::DistributedFESpace=f) + + bgmodel_gids = get_cell_gids(bgmodel) + spaces = map_parts( + local_views(f), + bgcell_to_bgcellin, + local_views(g), + local_views(bgmodel_gids)) do f,bgcell_to_bgcellin,g,gids + AgFEMSpace(f,bgcell_to_bgcellin,g,get_lid_to_gid(gids)) + end trians = map_parts(get_triangulation,local_views(f)) trian = DistributedTriangulation(trians,bgmodel) trian = add_ghost_cells(trian) trian_gids = generate_cell_gids(trian) cell_to_ldofs = map_parts(get_cell_dof_ids,spaces) + cell_to_ldofs = map_parts(i->map(sort,i),cell_to_ldofs) nldofs = map_parts(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) vector_type = _find_vector_type(spaces,gids) @@ -166,7 +163,7 @@ geo1 = disk(R,x0=p0-d) geo2 = disk(R,x0=p0+d) #geo = !union(geo1,geo2) -n = 10 +n = 8 mesh_partition = (n,n) bgmodel = CartesianDiscreteModel(parts,pmin,pmax,mesh_partition) bgtrian = Triangulation(bgmodel) @@ -178,7 +175,7 @@ h = dp[1]/n cutgeo = cut(bgmodel,geo) strategy = AggregateAllCutCells() -strategy = AggregateCutCellsByThreshold(0) +strategy = AggregateCutCellsByThreshold(0.5) aggregates = aggregate(strategy,cutgeo) Ω_bg = Triangulation(bgmodel) @@ -206,8 +203,7 @@ reffe = ReferenceFE(lagrangian,Float64,order) Vstd = FESpace(Ω_act,reffe) -#V = AgFEMSpace(bgmodel,Vstd,aggregates) -V = Vstd +V = AgFEMSpace(bgmodel,Vstd,aggregates) U = TrialFESpace(V) From fbc92af09ecfc07587e54a1bfd5f4b798bc5ec4e Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 13 Apr 2023 18:58:02 +0200 Subject: [PATCH 05/31] Add ghosts to touched_ gids --- test/_DistributedTests.jl | 65 ++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index f8d0f23..44e5ae8 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -1,4 +1,4 @@ -# module DistributedPoissonTests +module DistributedPoissonTests using Gridap using GridapEmbedded @@ -14,6 +14,13 @@ using GridapDistributed: DistributedSingleFieldFESpace using GridapDistributed: generate_gids using GridapDistributed: generate_cell_gids using GridapDistributed: _find_vector_type +using GridapEmbedded.Interfaces: SubFacetTriangulation +using Gridap.Geometry: AppendedTriangulation +using Gridap.Geometry: FaceToFaceGlue +using Gridap.FESpaces: SingleFieldFESpace +using Gridap.Arrays +using Gridap.Helpers +using PartitionedArrays: LinearGidToPart import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model @@ -28,11 +35,7 @@ import GridapDistributed: dof_wise_to_cell_wise! import GridapDistributed: dof_wise_to_cell_wise import GridapDistributed: generate_gids import GridapDistributed: add_ghost_cells -using GridapEmbedded.Interfaces: SubFacetTriangulation -using Gridap.Geometry: AppendedTriangulation -using Gridap.Geometry: FaceToFaceGlue -using Gridap.FESpaces: SingleFieldFESpace -using Gridap.Arrays +import PartitionedArrays: _part_to_firstgid struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType discretizations::A @@ -111,6 +114,7 @@ function AgFEMSpace( cell_to_ldofs = map_parts(i->map(sort,i),cell_to_ldofs) nldofs = map_parts(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) + gids = add_gid_to_part(gids) vector_type = _find_vector_type(spaces,gids) DistributedSingleFieldFESpace(spaces,gids,vector_type) end @@ -137,6 +141,53 @@ function remove_ghost_cells(trian::SubFacetTriangulation,gids) remove_ghost_cells(glue,trian,gids) end +## PartitionedArrays.jl + +function _part_to_firstgid(ngids::Vector,np::Integer) + @check length(ngids) == np + s = zeros(Int32,np) + s[2:np] = cumsum(ngids)[1:np-1] + s .+= 1 +end + +function _gid_to_part(gids::PRange) + if gids.gid_to_part !== nothing + gids.gid_to_part + else + np = num_parts(gids) + noids = map_parts(num_oids,local_views(gids)) + ngids = gather_all(noids) + gid_to_part = map_parts(ngids) do ngids + part_to_firstgid = _part_to_firstgid(ngids,np) + LinearGidToPart(sum(ngids),part_to_firstgid) + end + map_parts(_test_gid_to_part,local_views(gids),gid_to_part) + gid_to_part + end +end + +function _test_gid_to_part(i::IndexSet,gid_to_part::AbstractVector) + for (gid,part) in zip(i.lid_to_gid,i.lid_to_part) + @check gid_to_part[gid] == part + end +end + +function add_gid_to_part(gids::PRange,gid_to_part=_gid_to_part(gids)) + PRange( + gids.ngids, + gids.partition, + gids.exchanger, + gid_to_part, + gids.ghost) +end + +function PartitionedArrays.touched_hids( + a::PRange, + gids::AbstractPData{<:AbstractVector{<:Integer}}) + + add_gids!(a,gids) + map_parts(touched_hids,a.partition,gids) +end # Driver @@ -262,4 +313,4 @@ writevtk(Γ,"trian_G") @test el2/ul2 < 1.e-8 @test eh1/uh1 < 1.e-7 -# end # module +end # module From a4297928de65702a279825337fc8b3b461d51649 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 14 Apr 2023 19:23:56 +0200 Subject: [PATCH 06/31] [Distributed] compute cuts only in own cells - pausing developments --- test/_DistributedTests.jl | 197 +++++++++++++++++++++++++++++++++++++- 1 file changed, 195 insertions(+), 2 deletions(-) diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index 44e5ae8..2eabf4e 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -15,12 +15,17 @@ using GridapDistributed: generate_gids using GridapDistributed: generate_cell_gids using GridapDistributed: _find_vector_type using GridapEmbedded.Interfaces: SubFacetTriangulation +using Gridap.Geometry using Gridap.Geometry: AppendedTriangulation using Gridap.Geometry: FaceToFaceGlue using Gridap.FESpaces: SingleFieldFESpace using Gridap.Arrays using Gridap.Helpers using PartitionedArrays: LinearGidToPart +using PartitionedArrays: get_part_ids +using GridapEmbedded.Interfaces: SubCellData +using GridapEmbedded.Interfaces: SubFacetData +using GridapEmbedded.CSG import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model @@ -36,6 +41,7 @@ import GridapDistributed: dof_wise_to_cell_wise import GridapDistributed: generate_gids import GridapDistributed: add_ghost_cells import PartitionedArrays: _part_to_firstgid +import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType discretizations::A @@ -185,10 +191,162 @@ function PartitionedArrays.touched_hids( a::PRange, gids::AbstractPData{<:AbstractVector{<:Integer}}) - add_gids!(a,gids) + if a.gid_to_part !== nothing + add_gids!(a,gids) + end map_parts(touched_hids,a.partition,gids) end +function remove_ghost_cells(model::DiscreteModel,gids::IndexSet) + DiscreteModelPortion(model,gids.oid_to_lid) +end + +function _exchange!( + i_to_a::AbstractPData{<:Vector{<:Vector}}, + exchanger::Exchanger) + + n = length(get_part(i_to_a)) + for i in 1:n + a = map_parts(i_to_a) do i_to_a + i_to_a[i] + end + exchange!(a,exchanger) + end +end + +function _cut(bgmodel::DistributedDiscreteModel,args...) + _cut(LevelSetCutter(),bgmodel,args...) +end + +function _cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + gids = get_cell_gids(bgmodel) + cuts = map_parts(local_views(bgmodel),local_views(gids)) do bgmodel,gids + ownmodel = remove_ghost_cells(bgmodel,gids) + cutgeo = cut(cutter,ownmodel,args...) + change_bgmodel(cutgeo,bgmodel,gids.oid_to_lid) + end + ls_to_bgcell_to_inoutcut = map_parts(c->c.ls_to_bgcell_to_inoutcut,cuts) + _exchange!(ls_to_bgcell_to_inoutcut,gids.exchanger) + DistributedEmbeddedDiscretization(cuts,bgmodel) +end + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + in_or_out::Integer=IN) + + geo = get_part(map_parts(c->c.geo,local_views(cutgeo))) + _aggregate(stragegy,cutgeo,geo,in_or_out) +end + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out::Integer=IN) + + bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + _aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) +end + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out::Integer, + bgf_to_ioc::AbstractPData{<:AbstractVector}) + + map_parts(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc + aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) + end + # TODO: parallel aggregation here +end + +function change_bgmodel( + cut::EmbeddedDiscretization, + newmodel::DiscreteModel, + cell_to_newcell) + + ls_to_bgc_to_ioc = map(cut.ls_to_bgcell_to_inoutcut) do bgc_to_ioc + new_bgc_to_ioc = Vector{Int8}(undef,num_cells(newmodel)) + new_bgc_to_ioc[cell_to_newcell] = bgc_to_ioc + new_bgc_to_ioc + end + subcells = change_bgmodel(cut.subcells,cell_to_newcell) + subfacets = change_bgmodel(cut.subfacets,cell_to_newcell) + EmbeddedDiscretization( + newmodel, + ls_to_bgc_to_ioc, + subcells, + cut.ls_to_subcell_to_inout, + subfacets, + cut.ls_to_subfacet_to_inout, + cut.oid_to_ls, + cut.geo) +end + +function change_bgmodel(cells::SubCellData,cell_to_newcell) + cell_to_bgcell = lazy_map(Reindex(cell_to_newcell),cells.cell_to_bgcell) + SubCellData( + cells.cell_to_points, + collect(Int32,cell_to_bgcell), + cells.point_to_coords, + cells.point_to_rcoords) +end + +function change_bgmodel(facets::SubFacetData,cell_to_newcell) + facet_to_bgcell = lazy_map(Reindex(cell_to_newcell),facets.facet_to_bgcell) + SubFacetData( + facets.facet_to_points, + facets.facet_to_normal, + collect(Int32,facet_to_bgcell), + facets.point_to_coords, + facets.point_to_rcoords) +end + +function compute_bgfacet_to_inoutcut( + bgmodel::DistributedDiscreteModel, + bgf_to_ioc::AbstractPData{<:AbstractVector}) + + D = num_dims(get_part(local_views(bgmodel))) + gids = get_cell_gids(bgmodel) + bgf_to_ioc = map_parts( + local_views(bgmodel), + local_views(gids), + bgf_to_ioc) do bgmodel,gids,bgf_to_ioc + + ownmodel = remove_ghost_cells(bgmodel,gids) + f_to_pf = Gridap.Geometry.get_face_to_parent_face(ownmodel,D-1) + _bgf_to_ioc = Vector{eltype(bgf_to_ioc)}(undef,num_faces(bgmodel,D-1)) + _bgf_to_ioc[f_to_pf] .= bgf_to_ioc + _bgf_to_ioc + end + facet_gids = get_face_gids(bgmodel,D-1) + exchange!(bgf_to_ioc,facet_gids.exchanger) + bgf_to_ioc +end + + +function compute_bgfacet_to_inoutcut( + cutter::Cutter, + bgmodel::DistributedDiscreteModel, + geo) + + gids = get_cell_gids(bgmodel) + bgf_to_ioc = map_parts(local_views(bgmodel),local_views(gids)) do model,gids + ownmodel = remove_ghost_cells(model,gids) + compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) + end + compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc) +end + + +function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) + cutter = LevelSetCutter() + compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) +end + + # Driver partition = (2,2) @@ -236,7 +394,7 @@ aggregates = aggregate(strategy,cutgeo) writevtk(Ω_bg,"trian") - +writevtk(Ω_act,"trian_act") writevtk(Ω,"trian_O") writevtk(Γ,"trian_G") @@ -313,4 +471,39 @@ writevtk(Γ,"trian_G") @test el2/ul2 < 1.e-8 @test eh1/uh1 < 1.e-7 + +## + +partition = (2,2) + +parts = get_part_ids(SequentialBackend(),partition) + +L = 1 +p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + + +R = 0.35 +geo = disk(R,x0=p0) + +n = 8 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(parts,pmin,pmax,mesh_partition) + +cutgeo = _cut(bgmodel,geo) + +bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + +# BUG: aggregates do not have information of ghost cut cells +# TODO: exchange measures and aggregates +_aggregate(strategy,cutgeo) + + +# TODO: parallel aggregation (parallel aggfem article) +# 0. exchange measures and aggregates through gid +# 1. root in ghost layer +# 2. root in neighbors +# 3. root in neighbors of neighbors + end # module From 066eee8e85c67cd48740959eea056dcc1c9d7b15 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 22 Mar 2024 16:40:51 +0100 Subject: [PATCH 07/31] [distributed] upgrade GridapDistributed --- Project.toml | 2 + test/_DistributedTests.jl | 247 ++++++++++++++++++++------------------ 2 files changed, 133 insertions(+), 116 deletions(-) diff --git a/Project.toml b/Project.toml index e09cc45..32969f7 100644 --- a/Project.toml +++ b/Project.toml @@ -29,6 +29,8 @@ Gridap = "0.17.13" LightGraphs = "1.3.3" MiniQhull = "0.1.0, 0.2, 0.3, 0.4" julia = "1.3" +GridapDistributed = "0.3.6" +PartitionedArrays = "0.3.4" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/_DistributedTests.jl b/test/_DistributedTests.jl index 2eabf4e..d705ba9 100644 --- a/test/_DistributedTests.jl +++ b/test/_DistributedTests.jl @@ -7,7 +7,6 @@ using PartitionedArrays using Test using GridapDistributed: DistributedDiscreteModel -using GridapDistributed: DistributedGridapType using GridapDistributed: DistributedTriangulation using GridapDistributed: DistributedFESpace using GridapDistributed: DistributedSingleFieldFESpace @@ -21,8 +20,6 @@ using Gridap.Geometry: FaceToFaceGlue using Gridap.FESpaces: SingleFieldFESpace using Gridap.Arrays using Gridap.Helpers -using PartitionedArrays: LinearGidToPart -using PartitionedArrays: get_part_ids using GridapEmbedded.Interfaces: SubCellData using GridapEmbedded.Interfaces: SubFacetData using GridapEmbedded.CSG @@ -32,6 +29,7 @@ import Gridap.Geometry: get_background_model import GridapEmbedded.Interfaces: cut import GridapEmbedded.Interfaces: Cutter import GridapEmbedded.Interfaces: EmbeddedBoundary +import GridapEmbedded.Interfaces: ActiveInOrOut import GridapEmbedded.AgFEM: aggregate import GridapEmbedded.AgFEM: AgFEMSpace import GridapDistributed: local_views @@ -40,14 +38,14 @@ import GridapDistributed: dof_wise_to_cell_wise! import GridapDistributed: dof_wise_to_cell_wise import GridapDistributed: generate_gids import GridapDistributed: add_ghost_cells -import PartitionedArrays: _part_to_firstgid +# import PartitionedArrays: _part_to_firstgid import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut -struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: DistributedGridapType +struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: GridapType discretizations::A model::B function DistributedEmbeddedDiscretization( - d::AbstractPData{<:EmbeddedDiscretization{Dp,T}}, + d::AbstractArray{<:EmbeddedDiscretization{Dp,T}}, model::DistributedDiscreteModel) where {Dp,T} A = typeof(d) B = typeof(model) @@ -64,7 +62,7 @@ function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) # - cut own_bgmodels # - set (local) bgmodels # - reindex bgcell/facet data - discretizations = map_parts(local_views(bgmodel)) do lmodel + discretizations = map(local_views(bgmodel)) do lmodel cut(cutter,lmodel,args...) end DistributedEmbeddedDiscretization(discretizations,bgmodel) @@ -75,24 +73,39 @@ function cut(bgmodel::DistributedDiscreteModel,args...) cut(cutter,bgmodel,args...) end +function Triangulation( + cutgeo::DistributedEmbeddedDiscretization, + in_or_out::ActiveInOrOut, + args...) + + distributed_embedded_triangulation(Triangulation,cutgeo,in_or_out,args...) +end + function Triangulation(cutgeo::DistributedEmbeddedDiscretization,args...) - trians = map_parts(local_views(cutgeo)) do lcutgeo - Triangulation(lcutgeo,args...) - end - bgmodel = get_background_model(cutgeo) - DistributedTriangulation(trians,bgmodel) + trian = distributed_embedded_triangulation(Triangulation,cutgeo,args...) + remove_ghost_cells(trian) end function EmbeddedBoundary(cutgeo::DistributedEmbeddedDiscretization,args...) - trians = map_parts(local_views(cutgeo)) do lcutgeo - EmbeddedBoundary(lcutgeo,args...) + trian = distributed_embedded_triangulation(EmbeddedBoundary,cutgeo,args...) + remove_ghost_cells(trian) +end + +function distributed_embedded_triangulation( + T, + cutgeo::DistributedEmbeddedDiscretization, + args...) + + trians = map(local_views(cutgeo)) do lcutgeo + T(lcutgeo,args...) end bgmodel = get_background_model(cutgeo) DistributedTriangulation(trians,bgmodel) end + function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) - map_parts(local_views(cutgeo)) do lcutgeo + map(local_views(cutgeo)) do lcutgeo aggregate(stragegy,lcutgeo) end # TODO: parallel aggregation here @@ -101,26 +114,26 @@ end function AgFEMSpace( bgmodel::DistributedDiscreteModel, f::DistributedFESpace, - bgcell_to_bgcellin::AbstractPData{<:AbstractVector}, + bgcell_to_bgcellin::AbstractArray{<:AbstractVector}, g::DistributedFESpace=f) bgmodel_gids = get_cell_gids(bgmodel) - spaces = map_parts( + spaces = map( local_views(f), bgcell_to_bgcellin, local_views(g), local_views(bgmodel_gids)) do f,bgcell_to_bgcellin,g,gids - AgFEMSpace(f,bgcell_to_bgcellin,g,get_lid_to_gid(gids)) + AgFEMSpace(f,bgcell_to_bgcellin,g,local_to_global(gids)) end - trians = map_parts(get_triangulation,local_views(f)) + trians = map(get_triangulation,local_views(f)) trian = DistributedTriangulation(trians,bgmodel) trian = add_ghost_cells(trian) trian_gids = generate_cell_gids(trian) - cell_to_ldofs = map_parts(get_cell_dof_ids,spaces) - cell_to_ldofs = map_parts(i->map(sort,i),cell_to_ldofs) - nldofs = map_parts(num_free_dofs,spaces) + cell_to_ldofs = map(get_cell_dof_ids,spaces) + cell_to_ldofs = map(i->map(sort,i),cell_to_ldofs) + nldofs = map(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) - gids = add_gid_to_part(gids) + # gids = add_gid_to_part(gids) vector_type = _find_vector_type(spaces,gids) DistributedSingleFieldFESpace(spaces,gids,vector_type) end @@ -128,7 +141,7 @@ end function remove_ghost_cells(trian::DistributedTriangulation) model = get_background_model(trian) gids = get_cell_gids(model) - trians = map_parts(local_views(trian),local_views(gids)) do trian,gids + trians = map(local_views(trian),local_views(gids)) do trian,gids remove_ghost_cells(trian,gids) end DistributedTriangulation(trians,model) @@ -149,69 +162,66 @@ end ## PartitionedArrays.jl -function _part_to_firstgid(ngids::Vector,np::Integer) - @check length(ngids) == np - s = zeros(Int32,np) - s[2:np] = cumsum(ngids)[1:np-1] - s .+= 1 -end - -function _gid_to_part(gids::PRange) - if gids.gid_to_part !== nothing - gids.gid_to_part - else - np = num_parts(gids) - noids = map_parts(num_oids,local_views(gids)) - ngids = gather_all(noids) - gid_to_part = map_parts(ngids) do ngids - part_to_firstgid = _part_to_firstgid(ngids,np) - LinearGidToPart(sum(ngids),part_to_firstgid) - end - map_parts(_test_gid_to_part,local_views(gids),gid_to_part) - gid_to_part - end -end - -function _test_gid_to_part(i::IndexSet,gid_to_part::AbstractVector) - for (gid,part) in zip(i.lid_to_gid,i.lid_to_part) - @check gid_to_part[gid] == part - end -end - -function add_gid_to_part(gids::PRange,gid_to_part=_gid_to_part(gids)) - PRange( - gids.ngids, - gids.partition, - gids.exchanger, - gid_to_part, - gids.ghost) -end - -function PartitionedArrays.touched_hids( - a::PRange, - gids::AbstractPData{<:AbstractVector{<:Integer}}) - - if a.gid_to_part !== nothing - add_gids!(a,gids) - end - map_parts(touched_hids,a.partition,gids) -end - -function remove_ghost_cells(model::DiscreteModel,gids::IndexSet) - DiscreteModelPortion(model,gids.oid_to_lid) +# function _part_to_firstgid(ngids::Vector,np::Integer) +# @check length(ngids) == np +# s = zeros(Int32,np) +# s[2:np] = cumsum(ngids)[1:np-1] +# s .+= 1 +# end + +# function _gid_to_part(gids::PRange) +# if gids.gid_to_part !== nothing +# gids.gid_to_part +# else +# np = num_parts(gids) +# noids = map(num_oids,local_views(gids)) +# ngids = gather_all(noids) +# gid_to_part = map(ngids) do ngids +# part_to_firstgid = _part_to_firstgid(ngids,np) +# LinearGidToPart(sum(ngids),part_to_firstgid) +# end +# map(_test_gid_to_part,local_views(gids),gid_to_part) +# gid_to_part +# end +# end + +# function _test_gid_to_part(i::AbstractLocalIndices,gid_to_part::AbstractVector) +# for (gid,part) in zip(i.lid_to_gid,i.lid_to_part) +# @check gid_to_part[gid] == part +# end +# end + +# function add_gid_to_part(gids::PRange,gid_to_part=_gid_to_part(gids)) +# PRange( +# gids.ngids, +# gids.partition, +# gids.exchanger, +# gid_to_part, +# gids.ghost) +# end + +# function PartitionedArrays.touched_hids( +# a::PRange, +# gids::AbstractArray{<:AbstractVector{<:Integer}}) + +# if a.gid_to_part !== nothing +# add_gids!(a,gids) +# end +# map(touched_hids,a.partition,gids) +# end + +function remove_ghost_cells(model::DiscreteModel,gids::AbstractLocalIndices) + DiscreteModelPortion(model,own_to_local(gids)) end function _exchange!( - i_to_a::AbstractPData{<:Vector{<:Vector}}, - exchanger::Exchanger) - - n = length(get_part(i_to_a)) - for i in 1:n - a = map_parts(i_to_a) do i_to_a - i_to_a[i] - end - exchange!(a,exchanger) - end + p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, + graph::ExchangeGraph) + + tables = map(Table,p_to_i_to_a) + data = map(t->t.data,tables) + exchange!(data,data,graph) + map(copyto!,p_to_i_to_a,tables) end function _cut(bgmodel::DistributedDiscreteModel,args...) @@ -220,13 +230,14 @@ end function _cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) gids = get_cell_gids(bgmodel) - cuts = map_parts(local_views(bgmodel),local_views(gids)) do bgmodel,gids + cuts = map(local_views(bgmodel),local_views(gids)) do bgmodel,gids ownmodel = remove_ghost_cells(bgmodel,gids) cutgeo = cut(cutter,ownmodel,args...) - change_bgmodel(cutgeo,bgmodel,gids.oid_to_lid) + change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) end - ls_to_bgcell_to_inoutcut = map_parts(c->c.ls_to_bgcell_to_inoutcut,cuts) - _exchange!(ls_to_bgcell_to_inoutcut,gids.exchanger) + ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) + graph = assembly_graph(partition(gids)) + _exchange!(ls_to_bgcell_to_inoutcut,graph) DistributedEmbeddedDiscretization(cuts,bgmodel) end @@ -235,8 +246,9 @@ function _aggregate( cutgeo::DistributedEmbeddedDiscretization, in_or_out::Integer=IN) - geo = get_part(map_parts(c->c.geo,local_views(cutgeo))) - _aggregate(stragegy,cutgeo,geo,in_or_out) + map(local_views(cutgeo)) do cutgeo + aggregate(stragegy,cutgeo,cutgeo.geo,in_or_out) + end end function _aggregate( @@ -254,9 +266,9 @@ function _aggregate( cutgeo::DistributedEmbeddedDiscretization, geo::CSG.Geometry, in_or_out::Integer, - bgf_to_ioc::AbstractPData{<:AbstractVector}) + bgf_to_ioc::AbstractArray{<:AbstractVector}) - map_parts(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc + map(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) end # TODO: parallel aggregation here @@ -306,11 +318,11 @@ end function compute_bgfacet_to_inoutcut( bgmodel::DistributedDiscreteModel, - bgf_to_ioc::AbstractPData{<:AbstractVector}) + bgf_to_ioc::AbstractArray{<:AbstractVector}) - D = num_dims(get_part(local_views(bgmodel))) + D = num_dims(eltype(local_views(bgmodel))) gids = get_cell_gids(bgmodel) - bgf_to_ioc = map_parts( + bgf_to_ioc = map( local_views(bgmodel), local_views(gids), bgf_to_ioc) do bgmodel,gids,bgf_to_ioc @@ -322,7 +334,8 @@ function compute_bgfacet_to_inoutcut( _bgf_to_ioc end facet_gids = get_face_gids(bgmodel,D-1) - exchange!(bgf_to_ioc,facet_gids.exchanger) + graph = assembly_graph(partition(facet_gids)) + exchange!(bgf_to_ioc,bgf_to_ioc,graph) bgf_to_ioc end @@ -333,7 +346,7 @@ function compute_bgfacet_to_inoutcut( geo) gids = get_cell_gids(bgmodel) - bgf_to_ioc = map_parts(local_views(bgmodel),local_views(gids)) do model,gids + bgf_to_ioc = map(local_views(bgmodel),local_views(gids)) do model,gids ownmodel = remove_ghost_cells(model,gids) compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) end @@ -349,9 +362,11 @@ end # Driver -partition = (2,2) +distribute = identity + +np = (2,2) -parts = get_part_ids(SequentialBackend(),partition) +ranks = distribute(LinearIndices((prod(np),))) u(x) = x[1] - x[2] f(x) = -Δ(u)(x) @@ -374,7 +389,7 @@ geo2 = disk(R,x0=p0+d) n = 8 mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(parts,pmin,pmax,mesh_partition) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) bgtrian = Triangulation(bgmodel) writevtk(bgtrian,"bgtrian") @@ -398,9 +413,6 @@ writevtk(Ω_act,"trian_act") writevtk(Ω,"trian_O") writevtk(Γ,"trian_G") -Γ = remove_ghost_cells(Γ) -Ω = remove_ghost_cells(Ω) - n_Γ = get_normal_vector(Γ) order = 1 @@ -412,7 +424,9 @@ reffe = ReferenceFE(lagrangian,Float64,order) Vstd = FESpace(Ω_act,reffe) + V = AgFEMSpace(bgmodel,Vstd,aggregates) +V = Vstd U = TrialFESpace(V) @@ -440,22 +454,22 @@ ul2 = l2(uh) uh1 = h1(uh) # -colors = map_parts(color_aggregates,aggregates,local_views(bgmodel)) +colors = map(color_aggregates,aggregates,local_views(bgmodel)) cell_gids = get_cell_gids(bgmodel) -ohids = map_parts(get_lid_to_ohid,local_views(cell_gids)) -gids = map_parts(get_lid_to_gid,local_views(cell_gids)) -oids = map_parts(i->findall(>(0),i),ohids) +ohids = map(local_to_own,local_views(cell_gids)) +gids = map(local_to_global,local_views(cell_gids)) +oids = map(i->findall(>(0),i),ohids) -_aggregates = map_parts(aggregates,gids) do agg,gid +_aggregates = map(aggregates,gids) do agg,gid map(i-> i==0 ? 0 : gid[i],agg) end -_aggregates = map_parts(_aggregates,oids) do agg,oid +_aggregates = map(_aggregates,oids) do agg,oid map(Reindex(agg),oid) end -_colors = map_parts(colors,oids) do col,oid +_colors = map(colors,oids) do col,oid map(Reindex(col),oid) end -_gids = map_parts(gids,oids) do gid,oid +_gids = map(gids,oids) do gid,oid map(Reindex(gid),oid) end @@ -471,12 +485,13 @@ writevtk(Γ,"trian_G") @test el2/ul2 < 1.e-8 @test eh1/uh1 < 1.e-7 - ## -partition = (2,2) +distribute = identity + +np = (2,2) -parts = get_part_ids(SequentialBackend(),partition) +ranks = distribute(LinearIndices((prod(np),))) L = 1 p0 = Point(0.0,0.0) @@ -489,7 +504,7 @@ geo = disk(R,x0=p0) n = 8 mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(parts,pmin,pmax,mesh_partition) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) cutgeo = _cut(bgmodel,geo) @@ -497,7 +512,7 @@ bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) # BUG: aggregates do not have information of ghost cut cells # TODO: exchange measures and aggregates -_aggregate(strategy,cutgeo) +# _aggregate(strategy,cutgeo) # TODO: parallel aggregation (parallel aggfem article) From 27db09694bfc6536fa912581fb00307e91104aa0 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Wed, 3 Apr 2024 15:24:52 +0200 Subject: [PATCH 08/31] distributed devs to src --- .../Distributed/Distributed.jl | 421 ++++-------------- src/GridapEmbedded.jl | 2 + test/DistributedTests/DistributedDevs.jl | 216 +++++++++ test/DistributedTests/PoissonTests.jl | 138 ++++++ test/DistributedTests/runtests.jl | 5 + .../sequential/PoissonTests.jl | 7 + test/DistributedTests/sequential/runtests.jl | 7 + test/runtests.jl | 2 + 8 files changed, 463 insertions(+), 335 deletions(-) rename test/_DistributedTests.jl => src/Distributed/Distributed.jl (53%) create mode 100644 test/DistributedTests/DistributedDevs.jl create mode 100644 test/DistributedTests/PoissonTests.jl create mode 100644 test/DistributedTests/runtests.jl create mode 100644 test/DistributedTests/sequential/PoissonTests.jl create mode 100644 test/DistributedTests/sequential/runtests.jl diff --git a/test/_DistributedTests.jl b/src/Distributed/Distributed.jl similarity index 53% rename from test/_DistributedTests.jl rename to src/Distributed/Distributed.jl index d705ba9..a53dfac 100644 --- a/test/_DistributedTests.jl +++ b/src/Distributed/Distributed.jl @@ -1,45 +1,41 @@ -module DistributedPoissonTests +module Distributed using Gridap -using GridapEmbedded using GridapDistributed using PartitionedArrays -using Test +using Gridap.Arrays +using Gridap.Geometry +using Gridap.Helpers + +using GridapEmbedded.CSG +using GridapEmbedded.LevelSetCutters +using GridapEmbedded.Interfaces +using GridapEmbedded.Interfaces: Cutter +using GridapEmbedded.Interfaces: ActiveInOrOut +using GridapEmbedded.Interfaces: SubFacetTriangulation +using GridapEmbedded.Interfaces: SubCellData +using GridapEmbedded.Interfaces: SubFacetData +using Gridap.Geometry: AppendedTriangulation using GridapDistributed: DistributedDiscreteModel using GridapDistributed: DistributedTriangulation using GridapDistributed: DistributedFESpace using GridapDistributed: DistributedSingleFieldFESpace +using GridapDistributed: add_ghost_cells using GridapDistributed: generate_gids using GridapDistributed: generate_cell_gids using GridapDistributed: _find_vector_type -using GridapEmbedded.Interfaces: SubFacetTriangulation -using Gridap.Geometry -using Gridap.Geometry: AppendedTriangulation -using Gridap.Geometry: FaceToFaceGlue -using Gridap.FESpaces: SingleFieldFESpace -using Gridap.Arrays -using Gridap.Helpers -using GridapEmbedded.Interfaces: SubCellData -using GridapEmbedded.Interfaces: SubFacetData -using GridapEmbedded.CSG -import Gridap.Geometry: Triangulation -import Gridap.Geometry: get_background_model -import GridapEmbedded.Interfaces: cut -import GridapEmbedded.Interfaces: Cutter -import GridapEmbedded.Interfaces: EmbeddedBoundary -import GridapEmbedded.Interfaces: ActiveInOrOut import GridapEmbedded.AgFEM: aggregate import GridapEmbedded.AgFEM: AgFEMSpace +import GridapEmbedded.Interfaces: cut +import GridapEmbedded.Interfaces: EmbeddedBoundary +import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut +import Gridap.Geometry: Triangulation +import Gridap.Geometry: get_background_model import GridapDistributed: local_views import GridapDistributed: remove_ghost_cells -import GridapDistributed: dof_wise_to_cell_wise! -import GridapDistributed: dof_wise_to_cell_wise -import GridapDistributed: generate_gids -import GridapDistributed: add_ghost_cells -# import PartitionedArrays: _part_to_firstgid -import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut + struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: GridapType discretizations::A @@ -57,20 +53,20 @@ local_views(a::DistributedEmbeddedDiscretization) = a.discretizations get_background_model(a::DistributedEmbeddedDiscretization) = a.model -function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) - # TODO: do not compute ghost cells - # - cut own_bgmodels - # - set (local) bgmodels - # - reindex bgcell/facet data - discretizations = map(local_views(bgmodel)) do lmodel - cut(cutter,lmodel,args...) - end - DistributedEmbeddedDiscretization(discretizations,bgmodel) +function cut(bgmodel::DistributedDiscreteModel,args...) + cut(LevelSetCutter(),bgmodel,args...) end -function cut(bgmodel::DistributedDiscreteModel,args...) - cutter = LevelSetCutter() - cut(cutter,bgmodel,args...) +function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + gids = get_cell_gids(bgmodel) + cuts = map(local_views(bgmodel),local_views(gids)) do bgmodel,gids + ownmodel = remove_ghost_cells(bgmodel,gids) + cutgeo = cut(cutter,ownmodel,args...) + change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) + end + ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) + _consistent!(ls_to_bgcell_to_inoutcut,gids) + DistributedEmbeddedDiscretization(cuts,bgmodel) end function Triangulation( @@ -105,10 +101,26 @@ end function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) - map(local_views(cutgeo)) do lcutgeo + aggregates = map(local_views(cutgeo)) do lcutgeo aggregate(stragegy,lcutgeo) end - # TODO: parallel aggregation here + bgmodel = get_background_model(cutgeo) + consistent_aggregates(aggregates,bgmodel) +end + +function consistent_aggregates(aggregates,bgmodel::DistributedDiscreteModel) + consistent_aggregates(aggregates,get_cell_gids(bgmodel)) +end + +function consistent_aggregates(aggregates,gids::PRange) + global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid + map(i-> iszero(i) ? i : gid[i],agg) + end + paggregates = PVector(global_aggregates,partition(gids)) + consistent!(paggregates) |> wait + map(local_values(paggregates),global_to_local(gids)) do agg,lgid + map(i-> iszero(i) ? i : lgid[i],agg) + end end function AgFEMSpace( @@ -133,7 +145,6 @@ function AgFEMSpace( cell_to_ldofs = map(i->map(sort,i),cell_to_ldofs) nldofs = map(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) - # gids = add_gid_to_part(gids) vector_type = _find_vector_type(spaces,gids) DistributedSingleFieldFESpace(spaces,gids,vector_type) end @@ -160,118 +171,63 @@ function remove_ghost_cells(trian::SubFacetTriangulation,gids) remove_ghost_cells(glue,trian,gids) end -## PartitionedArrays.jl - -# function _part_to_firstgid(ngids::Vector,np::Integer) -# @check length(ngids) == np -# s = zeros(Int32,np) -# s[2:np] = cumsum(ngids)[1:np-1] -# s .+= 1 -# end - -# function _gid_to_part(gids::PRange) -# if gids.gid_to_part !== nothing -# gids.gid_to_part -# else -# np = num_parts(gids) -# noids = map(num_oids,local_views(gids)) -# ngids = gather_all(noids) -# gid_to_part = map(ngids) do ngids -# part_to_firstgid = _part_to_firstgid(ngids,np) -# LinearGidToPart(sum(ngids),part_to_firstgid) -# end -# map(_test_gid_to_part,local_views(gids),gid_to_part) -# gid_to_part -# end -# end - -# function _test_gid_to_part(i::AbstractLocalIndices,gid_to_part::AbstractVector) -# for (gid,part) in zip(i.lid_to_gid,i.lid_to_part) -# @check gid_to_part[gid] == part -# end -# end - -# function add_gid_to_part(gids::PRange,gid_to_part=_gid_to_part(gids)) -# PRange( -# gids.ngids, -# gids.partition, -# gids.exchanger, -# gid_to_part, -# gids.ghost) -# end - -# function PartitionedArrays.touched_hids( -# a::PRange, -# gids::AbstractArray{<:AbstractVector{<:Integer}}) - -# if a.gid_to_part !== nothing -# add_gids!(a,gids) -# end -# map(touched_hids,a.partition,gids) -# end - function remove_ghost_cells(model::DiscreteModel,gids::AbstractLocalIndices) DiscreteModelPortion(model,own_to_local(gids)) end -function _exchange!( +function _consistent!( p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, - graph::ExchangeGraph) + prange::PRange) tables = map(Table,p_to_i_to_a) data = map(t->t.data,tables) - exchange!(data,data,graph) + pdata = PVector(data,partition(prange)) + consistent!(pdata) |> wait map(copyto!,p_to_i_to_a,tables) end -function _cut(bgmodel::DistributedDiscreteModel,args...) - _cut(LevelSetCutter(),bgmodel,args...) -end -function _cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) +function compute_bgfacet_to_inoutcut( + bgmodel::DistributedDiscreteModel, + bgf_to_ioc::AbstractArray{<:AbstractVector}) + + D = num_dims(eltype(local_views(bgmodel))) gids = get_cell_gids(bgmodel) - cuts = map(local_views(bgmodel),local_views(gids)) do bgmodel,gids + bgf_to_ioc = map( + local_views(bgmodel), + local_views(gids), + bgf_to_ioc) do bgmodel,gids,bgf_to_ioc + ownmodel = remove_ghost_cells(bgmodel,gids) - cutgeo = cut(cutter,ownmodel,args...) - change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) + f_to_pf = Gridap.Geometry.get_face_to_parent_face(ownmodel,D-1) + _bgf_to_ioc = Vector{eltype(bgf_to_ioc)}(undef,num_faces(bgmodel,D-1)) + _bgf_to_ioc[f_to_pf] .= bgf_to_ioc + _bgf_to_ioc end - ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) - graph = assembly_graph(partition(gids)) - _exchange!(ls_to_bgcell_to_inoutcut,graph) - DistributedEmbeddedDiscretization(cuts,bgmodel) + facet_gids = get_face_gids(bgmodel,D-1) + pbgf_to_ioc = PVector(bgf_to_ioc,partition(facet_gids)) + consistent!(pbgf_to_ioc) |> wait + local_values(pbgf_to_ioc) end -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - in_or_out::Integer=IN) - - map(local_views(cutgeo)) do cutgeo - aggregate(stragegy,cutgeo,cutgeo.geo,in_or_out) - end -end -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out::Integer=IN) +function compute_bgfacet_to_inoutcut( + cutter::Cutter, + bgmodel::DistributedDiscreteModel, + geo) - bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) - _aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) + gids = get_cell_gids(bgmodel) + bgf_to_ioc = map(local_views(bgmodel),local_views(gids)) do model,gids + ownmodel = remove_ghost_cells(model,gids) + compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) + end + compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc) end -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out::Integer, - bgf_to_ioc::AbstractArray{<:AbstractVector}) - map(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc - aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) - end - # TODO: parallel aggregation here +function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) + cutter = LevelSetCutter() + compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) end function change_bgmodel( @@ -316,209 +272,4 @@ function change_bgmodel(facets::SubFacetData,cell_to_newcell) facets.point_to_rcoords) end -function compute_bgfacet_to_inoutcut( - bgmodel::DistributedDiscreteModel, - bgf_to_ioc::AbstractArray{<:AbstractVector}) - - D = num_dims(eltype(local_views(bgmodel))) - gids = get_cell_gids(bgmodel) - bgf_to_ioc = map( - local_views(bgmodel), - local_views(gids), - bgf_to_ioc) do bgmodel,gids,bgf_to_ioc - - ownmodel = remove_ghost_cells(bgmodel,gids) - f_to_pf = Gridap.Geometry.get_face_to_parent_face(ownmodel,D-1) - _bgf_to_ioc = Vector{eltype(bgf_to_ioc)}(undef,num_faces(bgmodel,D-1)) - _bgf_to_ioc[f_to_pf] .= bgf_to_ioc - _bgf_to_ioc - end - facet_gids = get_face_gids(bgmodel,D-1) - graph = assembly_graph(partition(facet_gids)) - exchange!(bgf_to_ioc,bgf_to_ioc,graph) - bgf_to_ioc -end - - -function compute_bgfacet_to_inoutcut( - cutter::Cutter, - bgmodel::DistributedDiscreteModel, - geo) - - gids = get_cell_gids(bgmodel) - bgf_to_ioc = map(local_views(bgmodel),local_views(gids)) do model,gids - ownmodel = remove_ghost_cells(model,gids) - compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) - end - compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc) -end - - -function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) - cutter = LevelSetCutter() - compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) -end - - -# Driver - -distribute = identity - -np = (2,2) - -ranks = distribute(LinearIndices((prod(np),))) - -u(x) = x[1] - x[2] -f(x) = -Δ(u)(x) -ud(x) = u(x) - -L = 1 -p0 = Point(0.0,0.0) -pmin = p0-L/2 -pmax = p0+L/2 - - -R = 0.35 -geo = disk(R,x0=p0) - -R = 0.15 -d = L/4 -geo1 = disk(R,x0=p0-d) -geo2 = disk(R,x0=p0+d) -#geo = !union(geo1,geo2) - -n = 8 -mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) -bgtrian = Triangulation(bgmodel) -writevtk(bgtrian,"bgtrian") - -dp = pmax - pmin -h = dp[1]/n - -cutgeo = cut(bgmodel,geo) - -strategy = AggregateAllCutCells() -strategy = AggregateCutCellsByThreshold(0.5) -aggregates = aggregate(strategy,cutgeo) - -Ω_bg = Triangulation(bgmodel) -Ω_act = Triangulation(cutgeo,ACTIVE) -Ω = Triangulation(cutgeo,PHYSICAL) -Γ = EmbeddedBoundary(cutgeo) - - -writevtk(Ω_bg,"trian") -writevtk(Ω_act,"trian_act") -writevtk(Ω,"trian_O") -writevtk(Γ,"trian_G") - -n_Γ = get_normal_vector(Γ) - -order = 1 -degree = 2*order -dΩ = Measure(Ω,degree) -dΓ = Measure(Γ,degree) - -reffe = ReferenceFE(lagrangian,Float64,order) - -Vstd = FESpace(Ω_act,reffe) - - -V = AgFEMSpace(bgmodel,Vstd,aggregates) -V = Vstd -U = TrialFESpace(V) - - -const γd = 10.0 - -a(u,v) = - ∫( ∇(v)⋅∇(u) ) * dΩ + - ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ - -l(v) = - ∫( v*f ) * dΩ + - ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ - -op = AffineFEOperator(a,l,U,V) -uh = solve(op) - -e = u - uh - -l2(u) = sqrt(sum( ∫( u*u )*dΩ )) -h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) - -el2 = l2(e) -eh1 = h1(e) -ul2 = l2(uh) -uh1 = h1(uh) - -# -colors = map(color_aggregates,aggregates,local_views(bgmodel)) -cell_gids = get_cell_gids(bgmodel) -ohids = map(local_to_own,local_views(cell_gids)) -gids = map(local_to_global,local_views(cell_gids)) -oids = map(i->findall(>(0),i),ohids) - -_aggregates = map(aggregates,gids) do agg,gid - map(i-> i==0 ? 0 : gid[i],agg) -end -_aggregates = map(_aggregates,oids) do agg,oid - map(Reindex(agg),oid) -end -_colors = map(colors,oids) do col,oid - map(Reindex(col),oid) -end -_gids = map(gids,oids) do gid,oid - map(Reindex(gid),oid) -end - -writevtk(Ω_bg,"trian", - celldata=[ - "aggregate"=>_aggregates, - "color"=>_colors, - "gid"=>_gids])#, -# cellfields=["uh"=>uh]) - -writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) -writevtk(Γ,"trian_G") -@test el2/ul2 < 1.e-8 -@test eh1/uh1 < 1.e-7 - -## - -distribute = identity - -np = (2,2) - -ranks = distribute(LinearIndices((prod(np),))) - -L = 1 -p0 = Point(0.0,0.0) -pmin = p0-L/2 -pmax = p0+L/2 - - -R = 0.35 -geo = disk(R,x0=p0) - -n = 8 -mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) - -cutgeo = _cut(bgmodel,geo) - -bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) - -# BUG: aggregates do not have information of ghost cut cells -# TODO: exchange measures and aggregates -# _aggregate(strategy,cutgeo) - - -# TODO: parallel aggregation (parallel aggfem article) -# 0. exchange measures and aggregates through gid -# 1. root in ghost layer -# 2. root in neighbors -# 3. root in neighbors of neighbors - end # module diff --git a/src/GridapEmbedded.jl b/src/GridapEmbedded.jl index 175cee0..545ae5f 100644 --- a/src/GridapEmbedded.jl +++ b/src/GridapEmbedded.jl @@ -12,6 +12,8 @@ include("MomentFittedQuadratures/MomentFittedQuadratures.jl") include("AlgoimUtils/AlgoimUtils.jl") +include("Distributed/Distributed.jl") + include("Exports.jl") end # module diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl new file mode 100644 index 0000000..fdbb83c --- /dev/null +++ b/test/DistributedTests/DistributedDevs.jl @@ -0,0 +1,216 @@ +module DistributedDevs + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.CSG + +## Parallel aggregation + +using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + in_or_out::Integer=IN) + + map(local_views(cutgeo)) do cutgeo + aggregate(stragegy,cutgeo,cutgeo.geo,in_or_out) + end +end + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out::Integer=IN) + + bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + _aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) +end + +function _aggregate( + stragegy, + cutgeo::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out::Integer, + bgf_to_ioc::AbstractArray{<:AbstractVector}) + + map(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc + #input cell meas or IN cells + aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) + end +end + +# Driver + +distribute = identity + +np = (2,2) + +ranks = distribute(LinearIndices((prod(np),))) + +u(x) = x[1] - x[2] +f(x) = -Δ(u)(x) +ud(x) = u(x) + +L = 1 +p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + + +R = 0.35 +geo = disk(R,x0=p0) + +R = 0.15 +d = L/4 +geo1 = disk(R,x0=p0-d) +geo2 = disk(R,x0=p0+d) +#geo = !union(geo1,geo2) + +n = 8 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) +bgtrian = Triangulation(bgmodel) +writevtk(bgtrian,"bgtrian") + +dp = pmax - pmin +h = dp[1]/n + +cutgeo = cut(bgmodel,geo) + +strategy = AggregateAllCutCells() +strategy = AggregateCutCellsByThreshold(0.5) +aggregates = aggregate(strategy,cutgeo) + + +Ω_bg = Triangulation(bgmodel) +Ω_act = Triangulation(cutgeo,ACTIVE) +Ω = Triangulation(cutgeo,PHYSICAL) +Γ = EmbeddedBoundary(cutgeo) + +using GridapEmbedded.Interfaces: CUT +Ω_in = Triangulation(cutgeo,IN) +Ω_out = Triangulation(cutgeo,OUT) +Ω_cut = Triangulation(cutgeo,CUT) + +writevtk(Ω_in,"trian_in") +writevtk(Ω_out,"trian_out") +writevtk(Ω_cut,"trian_cut") + + + +writevtk(Ω_bg,"trian") +writevtk(Ω_act,"trian_act") +writevtk(Ω,"trian_O") +writevtk(Γ,"trian_G") + +n_Γ = get_normal_vector(Γ) + +order = 1 +degree = 2*order +dΩ = Measure(Ω,degree) +dΓ = Measure(Γ,degree) + +reffe = ReferenceFE(lagrangian,Float64,order) + +Vstd = FESpace(Ω_act,reffe) + + +V = AgFEMSpace(bgmodel,Vstd,aggregates) +# V = Vstd +U = TrialFESpace(V) + + +const γd = 10.0 + +a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + +l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + +op = AffineFEOperator(a,l,U,V) +uh = solve(op) + +e = u - uh + +l2(u) = sqrt(sum( ∫( u*u )*dΩ )) +h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + +el2 = l2(e) +eh1 = h1(e) +ul2 = l2(uh) +uh1 = h1(uh) + +# +colors = map(color_aggregates,aggregates,local_views(bgmodel)) +gids = get_cell_gids(bgmodel) + + +global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid + map(i-> i==0 ? 0 : gid[i],agg) +end +own_aggregates = map(global_aggregates,own_to_local(gids)) do agg,oid + map(Reindex(agg),oid) +end +own_colors = map(colors,own_to_local(gids)) do col,oid + map(Reindex(col),oid) +end + + +writevtk(Ω_bg,"trian", + celldata=[ + "aggregate"=>own_aggregates, + "color"=>own_colors, + "gid"=>own_to_global(gids)])#, +# cellfields=["uh"=>uh]) + +writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) +writevtk(Γ,"trian_G") +@test el2/ul2 < 1.e-8 +@test eh1/uh1 < 1.e-7 + +## + +distribute = identity + +np = (2,2) + +ranks = distribute(LinearIndices((prod(np),))) + +L = 1 +p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + + +R = 0.35 +geo = disk(R,x0=p0) + +n = 8 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) + +cutgeo = cut(bgmodel,geo) + +bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + +# BUG: aggregates do not have information of ghost cut cells +# TODO: exchange measures and aggregates +# _aggregate(strategy,cutgeo) + + +# TODO: parallel aggregation (parallel aggfem article) +# 0. exchange measures and aggregates through gid +# 1. root in ghost layer +# 2. root in neighbors +# 3. root in neighbors of neighbors + +end # module diff --git a/test/DistributedTests/PoissonTests.jl b/test/DistributedTests/PoissonTests.jl new file mode 100644 index 0000000..0047d92 --- /dev/null +++ b/test/DistributedTests/PoissonTests.jl @@ -0,0 +1,138 @@ +module PoissonTests + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.CSG + +function main(distribute,parts; + n=8, + cells=(n,n)) + + ranks = distribute(LinearIndices((prod(parts),))) + + u(x) = x[1] - x[2] + f(x) = -Δ(u)(x) + ud(x) = u(x) + + L = 1 + p0 = Point(0.0,0.0) + pmin = p0-L/2 + pmax = p0+L/2 + + + R = 0.35 + geo = disk(R,x0=p0) + + R = 0.15 + d = L/4 + geo1 = disk(R,x0=p0-d) + geo2 = disk(R,x0=p0+d) + #geo = !union(geo1,geo2) + + bgmodel = CartesianDiscreteModel(ranks,parts,pmin,pmax,cells) + # bgtrian = Triangulation(bgmodel) + # writevtk(bgtrian,"bgtrian") + + dp = pmax - pmin + h = dp[1]/n + + cutgeo = cut(bgmodel,geo) + + strategy = AggregateAllCutCells() + strategy = AggregateCutCellsByThreshold(0.5) + aggregates = aggregate(strategy,cutgeo) + + + Ω_bg = Triangulation(bgmodel) + Ω_act = Triangulation(cutgeo,ACTIVE) + Ω = Triangulation(cutgeo,PHYSICAL) + Γ = EmbeddedBoundary(cutgeo) + + # using GridapEmbedded.Interfaces: CUT + # Ω_in = Triangulation(cutgeo,IN) + # Ω_out = Triangulation(cutgeo,OUT) + # Ω_cut = Triangulation(cutgeo,CUT) + + # writevtk(Ω_in,"trian_in") + # writevtk(Ω_out,"trian_out") + # writevtk(Ω_cut,"trian_cut") + + + writevtk(Ω_bg,"trian") + writevtk(Ω_act,"trian_act") + writevtk(Ω,"trian_O") + writevtk(Γ,"trian_G") + + n_Γ = get_normal_vector(Γ) + + order = 1 + degree = 2*order + dΩ = Measure(Ω,degree) + dΓ = Measure(Γ,degree) + + reffe = ReferenceFE(lagrangian,Float64,order) + + Vstd = FESpace(Ω_act,reffe) + + V = AgFEMSpace(bgmodel,Vstd,aggregates) + U = TrialFESpace(V) + + + γd = 10.0 + + a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + + l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + + op = AffineFEOperator(a,l,U,V) + uh = solve(op) + + e = u - uh + + l2(u) = sqrt(sum( ∫( u*u )*dΩ )) + h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + + el2 = l2(e) + eh1 = h1(e) + ul2 = l2(uh) + uh1 = h1(uh) + + # + colors = map(color_aggregates,aggregates,local_views(bgmodel)) + gids = get_cell_gids(bgmodel) + + + global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid + map(i-> i==0 ? 0 : gid[i],agg) + end + own_aggregates = map(global_aggregates,own_to_local(gids)) do agg,oid + map(Reindex(agg),oid) + end + own_colors = map(colors,own_to_local(gids)) do col,oid + map(Reindex(col),oid) + end + + + writevtk(Ω_bg,"trian", + celldata=[ + "aggregate"=>own_aggregates, + "color"=>own_colors, + "gid"=>own_to_global(gids)])#, + # cellfields=["uh"=>uh]) + + writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) + writevtk(Γ,"trian_G") + @test el2/ul2 < 1.e-8 + @test eh1/uh1 < 1.e-7 + +end + +end # module diff --git a/test/DistributedTests/runtests.jl b/test/DistributedTests/runtests.jl new file mode 100644 index 0000000..e4b3489 --- /dev/null +++ b/test/DistributedTests/runtests.jl @@ -0,0 +1,5 @@ +module DistributedTests + +include("sequential/runtests.jl") + +end # module diff --git a/test/DistributedTests/sequential/PoissonTests.jl b/test/DistributedTests/sequential/PoissonTests.jl new file mode 100644 index 0000000..8436c65 --- /dev/null +++ b/test/DistributedTests/sequential/PoissonTests.jl @@ -0,0 +1,7 @@ +module PoissonTestsSeq +using PartitionedArrays +include("../PoissonTests.jl") +with_debug() do distribute + PoissonTests.main(distribute,(2,2)) +end +end diff --git a/test/DistributedTests/sequential/runtests.jl b/test/DistributedTests/sequential/runtests.jl new file mode 100644 index 0000000..e68831c --- /dev/null +++ b/test/DistributedTests/sequential/runtests.jl @@ -0,0 +1,7 @@ +module SequentialTests + +using Test + +@time @testset "PoissonSeq" begin include("PoissonTests.jl") end + +end diff --git a/test/runtests.jl b/test/runtests.jl index 8b1363f..31fc717 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,6 +21,8 @@ if QHULL_LOADED @time @testset "MomentFittedQuadratures" begin include("MomentFittedQuadraturesTests/runtests.jl") end + @time @testset "Distributed" begin include("DistributedTests/runtests.jl") end + include(joinpath(@__DIR__,"..","examples","runexamples.jl")) else From 507230afdbfe8c2f38674f277debcadbe0a2f908 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 4 Apr 2024 14:27:29 +0200 Subject: [PATCH 09/31] [distributed] deveolping distributed aggregation --- src/Distributed/Distributed.jl | 7 + test/DistributedTests/DistributedDevs.jl | 297 +++++++++++++++-------- 2 files changed, 203 insertions(+), 101 deletions(-) diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index a53dfac..e8d6a1d 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -31,6 +31,7 @@ import GridapEmbedded.AgFEM: AgFEMSpace import GridapEmbedded.Interfaces: cut import GridapEmbedded.Interfaces: EmbeddedBoundary import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut +import GridapEmbedded.Interfaces: compute_bgcell_to_inoutcut import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model import GridapDistributed: local_views @@ -230,6 +231,12 @@ function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) end +function compute_bgcell_to_inoutcut(cutgeo::DistributedEmbeddedDiscretization,args...) + map(local_views(cutgeo)) do cutgeo + compute_bgcell_to_inoutcut(cutgeo,args...) + end +end + function change_bgmodel( cut::EmbeddedDiscretization, newmodel::DiscreteModel, diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl index fdbb83c..6fcbdd2 100644 --- a/test/DistributedTests/DistributedDevs.jl +++ b/test/DistributedTests/DistributedDevs.jl @@ -7,6 +7,87 @@ using PartitionedArrays using Test using GridapEmbedded.CSG +using GridapEmbedded.Interfaces: CUT +using Gridap.Geometry +using Gridap.ReferenceFEs + +using GridapEmbedded.AgFEM: _touch_aggregated_cells! + +function _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + all_aggregated = true + for cell in own_cells + if ! cell_to_touched[cell] && cell_to_inoutcut[cell] == CUT + neigh_cell = _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + if neigh_cell > 0 + cellin = cell_to_cellin[neigh_cell] + centroid = cell_to_root_centroid[neigh_cell] + part = cell_to_root_part[neigh_cell] + + cell_to_neig[cell] = neigh_cell + cell_to_cellin[cell] = cellin + cell_to_root_centroid[cell] = centroid + cell_to_root_part[cell] = part + else + all_aggregated = false + end + end + end + _touch_aggregated_cells!(cell_to_touched,cell_to_cellin) + all_aggregated +end + +function _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + + faces = getindex!(c1,cell_to_faces,cell) + dmin = Inf + T = eltype(eltype(face_to_cells)) + best_neigh_cell = zero(T) + for face in faces + inoutcut = facet_to_inoutcut[face] + if inoutcut != CUT && inoutcut != loc + continue + end + neigh_cells = getindex!(c2,face_to_cells,face) + for neigh_cell in neigh_cells + if neigh_cell != cell && cell_to_touched[neigh_cell] + p = cell_to_root_centroid[neigh_cell] + q = cell_to_root_centroid[cell] + d = norm(p-q) + if (1.0+1.0e-9)*d < dmin + dmin = d + best_neigh_cell = neigh_cell + end + end + end + end + best_neigh_cell +end ## Parallel aggregation @@ -53,10 +134,6 @@ np = (2,2) ranks = distribute(LinearIndices((prod(np),))) -u(x) = x[1] - x[2] -f(x) = -Δ(u)(x) -ud(x) = u(x) - L = 1 p0 = Point(0.0,0.0) pmin = p0-L/2 @@ -66,141 +143,159 @@ pmax = p0+L/2 R = 0.35 geo = disk(R,x0=p0) -R = 0.15 -d = L/4 -geo1 = disk(R,x0=p0-d) -geo2 = disk(R,x0=p0+d) -#geo = !union(geo1,geo2) - n = 8 mesh_partition = (n,n) bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) -bgtrian = Triangulation(bgmodel) -writevtk(bgtrian,"bgtrian") - -dp = pmax - pmin -h = dp[1]/n cutgeo = cut(bgmodel,geo) -strategy = AggregateAllCutCells() -strategy = AggregateCutCellsByThreshold(0.5) -aggregates = aggregate(strategy,cutgeo) +bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) +Ω = Triangulation(cutgeo) -Ω_bg = Triangulation(bgmodel) -Ω_act = Triangulation(cutgeo,ACTIVE) -Ω = Triangulation(cutgeo,PHYSICAL) -Γ = EmbeddedBoundary(cutgeo) -using GridapEmbedded.Interfaces: CUT -Ω_in = Triangulation(cutgeo,IN) -Ω_out = Triangulation(cutgeo,OUT) -Ω_cut = Triangulation(cutgeo,CUT) -writevtk(Ω_in,"trian_in") -writevtk(Ω_out,"trian_out") -writevtk(Ω_cut,"trian_cut") +writevtk(Ω,"trian") -writevtk(Ω_bg,"trian") -writevtk(Ω_act,"trian_act") -writevtk(Ω,"trian_O") -writevtk(Γ,"trian_G") -n_Γ = get_normal_vector(Γ) -order = 1 -degree = 2*order -dΩ = Measure(Ω,degree) -dΓ = Measure(Γ,degree) +threshold = 0.8 +loc = IN +facet_to_inoutcut = compute_bgfacet_to_inoutcut(model,geo) -reffe = ReferenceFE(lagrangian,Float64,order) -Vstd = FESpace(Ω_act,reffe) +cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) +trian = Triangulation(cutgeo,cutinorout,geo) -V = AgFEMSpace(bgmodel,Vstd,aggregates) -# V = Vstd -U = TrialFESpace(V) +model = get_background_model(cutgeo) +bgtrian = get_triangulation(model) +cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) +cell_to_meas = map(get_cell_measure,local_views(bgtrian)) +cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m + lazy_map(/,c_to_cm,c_to_m) +end -const γd = 10.0 -a(u,v) = - ∫( ∇(v)⋅∇(u) ) * dΩ + - ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ +cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) -l(v) = - ∫( v*f ) * dΩ + - ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ -op = AffineFEOperator(a,l,U,V) -uh = solve(op) -e = u - uh +cell_to_coords = map(get_cell_coordinates,local_views(model)) +topo = get_grid_topology(model) +D = num_cell_dims(model) +cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) +face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) -l2(u) = sqrt(sum( ∫( u*u )*dΩ )) -h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) +ocell_to_threshold = map(cell_to_unit_cut_meas) do c_to_m + n_cells = length(c_to_m) + c_to_t = falses(n_cells) + c = array_cache(c_to_m) + for cell in 1:n_cells + if getindex!(c,c_to_m,cell) ≥ threshold + c_to_t[cell] = true + end + end + c_to_t +end -el2 = l2(e) -eh1 = h1(e) -ul2 = l2(uh) -uh1 = h1(uh) +cell_to_root_centroid = map(cell_to_coords) do cell_to_coords + map(i->sum(i)/length(i),cell_to_coords) +end +PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait -# -colors = map(color_aggregates,aggregates,local_views(bgmodel)) -gids = get_cell_gids(bgmodel) +gids = get_cell_gids(model) -global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid - map(i-> i==0 ? 0 : gid[i],agg) +cell_to_threshold = map(ocell_to_threshold,local_to_own(gids)) do o_to_t,l_to_o + T = eltype(o_to_t) + map(o-> iszero(o) ? zero(T) : o_to_t[o],l_to_o) end -own_aggregates = map(global_aggregates,own_to_local(gids)) do agg,oid - map(Reindex(agg),oid) +PVector(cell_to_threshold,partition(gids)) |> consistent! |> wait +cell_to_threshold + +n_cells = map(length,cell_to_threshold) +cell_to_cellin = map(n->zeros(Int32,n),n_cells) +cell_to_touched = map(falses,n_cells) +cells_threshold = map(findall,cell_to_threshold) +cell_to_cellin = map(cells_threshold,cell_to_cellin) do cells,c_to_ci + c_to_ci[cells] = cells # TODO: gid here + c_to_ci end -own_colors = map(colors,own_to_local(gids)) do col,oid - map(Reindex(col),oid) -end - - -writevtk(Ω_bg,"trian", - celldata=[ - "aggregate"=>own_aggregates, - "color"=>own_colors, - "gid"=>own_to_global(gids)])#, -# cellfields=["uh"=>uh]) - -writevtk(Ω,"trian_O",cellfields=["uh"=>uh]) -writevtk(Γ,"trian_G") -@test el2/ul2 < 1.e-8 -@test eh1/uh1 < 1.e-7 +cell_to_neig = map(copy,cell_to_cellin) +cell_to_touched = cell_to_threshold + + +c1 = map(array_cache,cell_to_faces) +c2 = map(array_cache,face_to_cells) + +max_iters = 1 +all_aggregated = false + +# for iter in 1:max_iters +own_cells = own_to_local(gids)[1] + +cell_to_root_part = map(collect,local_to_owner(gids)) +# communicate + +# end # for + + +# init variables + + +# for iter in 1:max_iters + +all_aggregated = map(c1,c2,own_to_local(gids), + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut) do c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut + + _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) -## - -distribute = identity - -np = (2,2) - -ranks = distribute(LinearIndices((prod(np),))) - -L = 1 -p0 = Point(0.0,0.0) -pmin = p0-L/2 -pmax = p0+L/2 +end -R = 0.35 -geo = disk(R,x0=p0) +# consistent! (communicate ghost) +(cell_to_touched, +cell_to_neig, # neig_lid (if lid not consistent, ignore ghost) +cell_to_cellin, # root_gid +cell_to_root_centroid, +cell_to_root_part) -n = 8 -mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) +# reduce all_aggregated and break if all(all_aggregated) -cutgeo = cut(bgmodel,geo) -bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) +# return cell_to_rood_gid, cell_to_root_part, cell_to_neig # BUG: aggregates do not have information of ghost cut cells # TODO: exchange measures and aggregates From 0930d942f5d98b2e04d08e68205176fd46f4e2fb Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 4 Apr 2024 17:33:14 +0200 Subject: [PATCH 10/31] [distributed] parallel aggregation - to be tested --- test/DistributedTests/DistributedDevs.jl | 346 ++++++++++++----------- 1 file changed, 178 insertions(+), 168 deletions(-) diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl index 6fcbdd2..ed3f123 100644 --- a/test/DistributedTests/DistributedDevs.jl +++ b/test/DistributedTests/DistributedDevs.jl @@ -5,6 +5,8 @@ using GridapEmbedded using GridapDistributed using PartitionedArrays using Test +using FillArrays + using GridapEmbedded.CSG using GridapEmbedded.Interfaces: CUT @@ -13,6 +15,160 @@ using Gridap.ReferenceFEs using GridapEmbedded.AgFEM: _touch_aggregated_cells! +using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization + +function _aggregate( + strategy::AggregateCutCellsByThreshold, + cut::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out=IN) + + bgmodel = get_background_model(cut) + facet_to_inoutcut = compute_bgfacet_to_inoutcut(bgmodel,geo) + _distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut) +end + + +function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_inoutcut) + @assert loc in (IN,OUT) + + cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) + trian = Triangulation(cutgeo,cutinorout,geo) + model = get_background_model(cutgeo) + bgtrian = get_triangulation(model) + cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) + cell_to_meas = map(get_cell_measure,local_views(bgtrian)) + cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m + lazy_map(/,c_to_cm,c_to_m) + end + + cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) + + cell_to_coords = map(get_cell_coordinates,local_views(model)) + topo = get_grid_topology(model) + D = num_cell_dims(model) + cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) + face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) + gids = get_cell_gids(model) + + _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) +end + + +function _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) + + + ocell_to_touched = map(cell_to_unit_cut_meas) do c_to_m + map(≥,c_to_m,Fill(threshold,length(c_to_m))) + end + cell_to_touched = _add_ghost_values(ocell_to_touched,gids) + + cell_to_root_centroid = map(cell_to_coords) do cell_to_coords + map(i->sum(i)/length(i),cell_to_coords) + end + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + + n_cells = map(length,cell_to_touched) + touched_cells = map(findall,cell_to_touched) + + cell_to_cellin = map(n->zeros(Int32,n),n_cells) + map(cell_to_cellin,touched_cells,local_to_global(gids)) do c_to_ci,cells,l_to_g + gcells = lazy_map(Reindex(l_to_g),cells) + c_to_ci[cells] = gcells + end + + cell_to_neig = map(n->zeros(Int32,n),n_cells) + + cell_to_root_part = map(collect,local_to_owner(gids)) + + c1 = map(array_cache,cell_to_faces) + c2 = map(array_cache,face_to_cells) + + max_iters = 1 + for iter in 1:max_iters + all_aggregated = _aggregate_one_step!(c1,c2,gids, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + PVector(cell_to_touched,partition(gids)) |> consistent! |> wait + PVector(cell_to_neig,partition(gids)) |> consistent! |> wait + PVector(cell_to_cellin,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait + + reduction!(&,all_aggregated,all_aggregated) + emit!(all_aggregated,all_aggregated) + + if PartitionedArrays.getany(all_aggregated) + break + end + end + + cell_to_cellin, cell_to_root_part, cell_to_neig +end + +function _aggregate_one_step!(c1,c2,gids::PRange, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + map(c1,c2,own_to_local(gids), + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + local_to_global(gids), + cell_to_faces, + face_to_cells, + facet_to_inoutcut) do c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut + + _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + end +end + function _aggregate_one_step!( c1,c2,own_cells, cell_to_inoutcut, @@ -21,6 +177,7 @@ function _aggregate_one_step!( cell_to_cellin, cell_to_root_centroid, cell_to_root_part, + cell_to_gcell, cell_to_faces, face_to_cells, facet_to_inoutcut, @@ -41,8 +198,9 @@ function _aggregate_one_step!( cellin = cell_to_cellin[neigh_cell] centroid = cell_to_root_centroid[neigh_cell] part = cell_to_root_part[neigh_cell] + neigh_gcell = cell_to_gcell[neigh_cell] - cell_to_neig[cell] = neigh_cell + cell_to_neig[cell] = neigh_gcell cell_to_cellin[cell] = cellin cell_to_root_centroid[cell] = centroid cell_to_root_part[cell] = part @@ -89,6 +247,17 @@ function _find_best_neighbor_from_centroid_distance( best_neigh_cell end +function _add_ghost_values(own_v,gids::PRange) + lens = map(length,local_views(gids)) + eltypes = map(eltype,own_v) + local_v = map(zeros,eltypes,lens) + map(local_v,own_v,own_to_local(gids)) do l,o,o_to_l + l[o_to_l] = o + end + PVector(local_v,partition(gids)) |> consistent! |> wait + local_v +end + ## Parallel aggregation using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization @@ -103,29 +272,6 @@ function _aggregate( end end -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out::Integer=IN) - - bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) - _aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) -end - -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out::Integer, - bgf_to_ioc::AbstractArray{<:AbstractVector}) - - map(local_views(cutgeo),bgf_to_ioc) do cutgeo,bgf_to_ioc - #input cell meas or IN cells - aggregate(stragegy,cutgeo,geo,in_or_out,bgf_to_ioc) - end -end - # Driver distribute = identity @@ -153,153 +299,17 @@ bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) Ω = Triangulation(cutgeo) - - writevtk(Ω,"trian") +strategy = AggregateCutCellsByThreshold(0.8) +aggregates,aggregate_owner,aggregate_neig = _aggregate(strategy,cutgeo,geo,IN) - - - -threshold = 0.8 -loc = IN -facet_to_inoutcut = compute_bgfacet_to_inoutcut(model,geo) - - -cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) - -trian = Triangulation(cutgeo,cutinorout,geo) - -model = get_background_model(cutgeo) - -bgtrian = get_triangulation(model) -cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) -cell_to_meas = map(get_cell_measure,local_views(bgtrian)) -cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m - lazy_map(/,c_to_cm,c_to_m) -end - - -cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) - - - -cell_to_coords = map(get_cell_coordinates,local_views(model)) -topo = get_grid_topology(model) -D = num_cell_dims(model) -cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) -face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) - -ocell_to_threshold = map(cell_to_unit_cut_meas) do c_to_m - n_cells = length(c_to_m) - c_to_t = falses(n_cells) - c = array_cache(c_to_m) - for cell in 1:n_cells - if getindex!(c,c_to_m,cell) ≥ threshold - c_to_t[cell] = true - end - end - c_to_t -end - -cell_to_root_centroid = map(cell_to_coords) do cell_to_coords - map(i->sum(i)/length(i),cell_to_coords) -end -PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait - - -gids = get_cell_gids(model) - -cell_to_threshold = map(ocell_to_threshold,local_to_own(gids)) do o_to_t,l_to_o - T = eltype(o_to_t) - map(o-> iszero(o) ? zero(T) : o_to_t[o],l_to_o) -end -PVector(cell_to_threshold,partition(gids)) |> consistent! |> wait -cell_to_threshold - -n_cells = map(length,cell_to_threshold) -cell_to_cellin = map(n->zeros(Int32,n),n_cells) -cell_to_touched = map(falses,n_cells) -cells_threshold = map(findall,cell_to_threshold) -cell_to_cellin = map(cells_threshold,cell_to_cellin) do cells,c_to_ci - c_to_ci[cells] = cells # TODO: gid here - c_to_ci -end -cell_to_neig = map(copy,cell_to_cellin) -cell_to_touched = cell_to_threshold - - -c1 = map(array_cache,cell_to_faces) -c2 = map(array_cache,face_to_cells) - -max_iters = 1 -all_aggregated = false - -# for iter in 1:max_iters -own_cells = own_to_local(gids)[1] - -cell_to_root_part = map(collect,local_to_owner(gids)) -# communicate - -# end # for - - -# init variables - - -# for iter in 1:max_iters - -all_aggregated = map(c1,c2,own_to_local(gids), - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut) do c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut - - _aggregate_one_step!( - c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - -end - - -# consistent! (communicate ghost) -(cell_to_touched, -cell_to_neig, # neig_lid (if lid not consistent, ignore ghost) -cell_to_cellin, # root_gid -cell_to_root_centroid, -cell_to_root_part) - -# reduce all_aggregated and break if all(all_aggregated) - - -# return cell_to_rood_gid, cell_to_root_part, cell_to_neig - -# BUG: aggregates do not have information of ghost cut cells -# TODO: exchange measures and aggregates -# _aggregate(strategy,cutgeo) +# TODO: +# - print aggregates +# - move to src +# - test aggregates with several geometries +# - reconstruct paths +# - add remote roots to model # TODO: parallel aggregation (parallel aggfem article) From be663166205305ca0859ae051b33f457d0a5a60c Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 5 Apr 2024 12:31:14 +0200 Subject: [PATCH 11/31] testing distributed aggregation --- src/Distributed/Distributed.jl | 247 +++++++++++++++++ test/DistributedTests/AggregationTests.jl | 97 +++++++ test/DistributedTests/DistributedDevs.jl | 321 ---------------------- 3 files changed, 344 insertions(+), 321 deletions(-) create mode 100644 test/DistributedTests/AggregationTests.jl delete mode 100644 test/DistributedTests/DistributedDevs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index e8d6a1d..2df7dbb 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -3,10 +3,12 @@ module Distributed using Gridap using GridapDistributed using PartitionedArrays +using FillArrays using Gridap.Arrays using Gridap.Geometry using Gridap.Helpers +using Gridap.ReferenceFEs using GridapEmbedded.CSG using GridapEmbedded.LevelSetCutters @@ -16,6 +18,8 @@ using GridapEmbedded.Interfaces: ActiveInOrOut using GridapEmbedded.Interfaces: SubFacetTriangulation using GridapEmbedded.Interfaces: SubCellData using GridapEmbedded.Interfaces: SubFacetData +using GridapEmbedded.AgFEM: _touch_aggregated_cells! +using GridapEmbedded.AgFEM: AggregateCutCellsByThreshold using Gridap.Geometry: AppendedTriangulation using GridapDistributed: DistributedDiscreteModel using GridapDistributed: DistributedTriangulation @@ -279,4 +283,247 @@ function change_bgmodel(facets::SubFacetData,cell_to_newcell) facets.point_to_rcoords) end +function distributed_aggregate( + strategy::AggregateCutCellsByThreshold, + cut::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out=IN) + + bgmodel = get_background_model(cut) + facet_to_inoutcut = compute_bgfacet_to_inoutcut(bgmodel,geo) + _distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut) +end + + +function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_inoutcut) + @assert loc in (IN,OUT) + + cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) + trian = Triangulation(cutgeo,cutinorout,geo) + model = get_background_model(cutgeo) + bgtrian = get_triangulation(model) + cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) + cell_to_meas = map(get_cell_measure,local_views(bgtrian)) + cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m + lazy_map(/,c_to_cm,c_to_m) + end + + cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) + + cell_to_coords = map(get_cell_coordinates,local_views(model)) + topo = get_grid_topology(model) + D = num_cell_dims(model) + cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) + face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) + gids = get_cell_gids(model) + + _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) +end + + +function _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) + + + ocell_to_touched = map(cell_to_unit_cut_meas) do c_to_m + map(≥,c_to_m,Fill(threshold,length(c_to_m))) + end + cell_to_touched = _add_ghost_values(ocell_to_touched,gids) + + cell_to_root_centroid = map(cell_to_coords) do cell_to_coords + map(i->sum(i)/length(i),cell_to_coords) + end + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + + n_cells = map(length,cell_to_touched) + touched_cells = map(findall,cell_to_touched) + + cell_to_cellin = map(n->zeros(Int32,n),n_cells) + map(cell_to_cellin,touched_cells,local_to_global(gids)) do c_to_ci,cells,l_to_g + gcells = lazy_map(Reindex(l_to_g),cells) + c_to_ci[cells] = gcells + end + + cell_to_neig = map(n->zeros(Int32,n),n_cells) + + cell_to_root_part = map(collect,local_to_owner(gids)) + + c1 = map(array_cache,cell_to_faces) + c2 = map(array_cache,face_to_cells) + + max_iters = 20 + for iter in 1:max_iters + all_aggregated = _aggregate_one_step!(c1,c2,gids, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + PVector(cell_to_touched,partition(gids)) |> consistent! |> wait + PVector(cell_to_neig,partition(gids)) |> consistent! |> wait + PVector(cell_to_cellin,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait + + reduction!(&,all_aggregated,all_aggregated) + emit!(all_aggregated,all_aggregated) + + if PartitionedArrays.getany(all_aggregated) + break + end + end + + cell_to_cellin, cell_to_root_part, cell_to_neig +end + +function _aggregate_one_step!(c1,c2,gids::PRange, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + map(c1,c2,own_to_local(gids), + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + local_to_global(gids), + cell_to_faces, + face_to_cells, + facet_to_inoutcut) do c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut + + _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + end +end + +function _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + all_aggregated = true + for cell in own_cells + if ! cell_to_touched[cell] && cell_to_inoutcut[cell] == CUT + neigh_cell = _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + if neigh_cell > 0 + cellin = cell_to_cellin[neigh_cell] + centroid = cell_to_root_centroid[neigh_cell] + part = cell_to_root_part[neigh_cell] + neigh_gcell = cell_to_gcell[neigh_cell] + + cell_to_neig[cell] = neigh_gcell + cell_to_cellin[cell] = cellin + cell_to_root_centroid[cell] = centroid + cell_to_root_part[cell] = part + else + all_aggregated = false + end + end + end + _touch_aggregated_cells!(cell_to_touched,cell_to_cellin) + all_aggregated +end + +function _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + + faces = getindex!(c1,cell_to_faces,cell) + dmin = Inf + T = eltype(eltype(face_to_cells)) + best_neigh_cell = zero(T) + for face in faces + inoutcut = facet_to_inoutcut[face] + if inoutcut != CUT && inoutcut != loc + continue + end + neigh_cells = getindex!(c2,face_to_cells,face) + for neigh_cell in neigh_cells + if neigh_cell != cell && cell_to_touched[neigh_cell] + p = cell_to_root_centroid[neigh_cell] + q = cell_to_root_centroid[cell] + d = norm(p-q) + if (1.0+1.0e-9)*d < dmin + dmin = d + best_neigh_cell = neigh_cell + end + end + end + end + best_neigh_cell +end + +function _add_ghost_values(own_v,gids::PRange) + lens = map(length,local_views(gids)) + eltypes = map(eltype,own_v) + local_v = map(zeros,eltypes,lens) + map(local_v,own_v,own_to_local(gids)) do l,o,o_to_l + l[o_to_l] = o + end + PVector(local_v,partition(gids)) |> consistent! |> wait + local_v +end + + + end # module diff --git a/test/DistributedTests/AggregationTests.jl b/test/DistributedTests/AggregationTests.jl new file mode 100644 index 0000000..38d3706 --- /dev/null +++ b/test/DistributedTests/AggregationTests.jl @@ -0,0 +1,97 @@ +module DistributedAggregationTests + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.Distributed: distributed_aggregate + +distribute = PartitionedArrays.DebugArray + +np = (4,1) + +ranks = distribute(LinearIndices((prod(np),))) + +L = 1 +p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + +R = 0.2 +x = VectorValue(1.0,0.0) + +d1 = VectorValue(0.8,0.0) +d2 = VectorValue(0.0,0.02) +geo1 = disk(R,x0=p0-x*(L/2-R)) +geo2 = quadrilateral(;x0=p0-d1/2-d2/2,d1=d1,d2=d2) +geo = union(geo1,geo2) + +n = 16 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) + +cutgeo = cut(bgmodel,geo) + +bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + +Ω = Triangulation(cutgeo) + +writevtk(Ω,"trian") + +strategy = AggregateCutCellsByThreshold(1.0) +aggregates,aggregate_owner,aggregate_neig = distributed_aggregate( + strategy,cutgeo,geo,IN) + +cut_in = map(aggregates) do agg + findall(!iszero,agg) +end + +cut_owner = map(aggregate_owner,cut_in) do owners,cut_in + owners[cut_in] +end + +map(ranks,cut_owner) do p,cut_owner + if p ∈ (3,4) + @test all(i->i==2,cut_owner) + end +end + +gids = get_cell_gids(bgmodel) + +oaggregates = map(aggregates,own_to_local(gids)) do agg,o_to_l + map(Reindex(agg),o_to_l) +end + +oaggregate_owner = map(aggregate_owner,own_to_local(gids)) do agg,o_to_l + map(Reindex(agg),o_to_l) +end + + +Ωbg = Triangulation(bgmodel) +Ω = Triangulation(cutgeo) +Ωin = Triangulation(cutgeo,IN) +Γ = EmbeddedBoundary(cutgeo) + +writevtk(Ωin,"trian_in") +writevtk(Γ,"bnd") +writevtk(Ωbg,"bgtrian",celldata= + ["aggregate"=>oaggregates, + "aggregate_owner"=>oaggregate_owner]) + + +# TODO: +# - print aggregates [x] +# - move to src +# - test aggregates with several geometries [x] +# - reconstruct paths +# - add remote roots to model + +# TODO: parallel aggregation (parallel aggfem article) +# 0. exchange measures and aggregates through gid +# 1. root in ghost layer +# 2. root in neighbors +# 3. root in neighbors of neighbors + +end # module diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl deleted file mode 100644 index ed3f123..0000000 --- a/test/DistributedTests/DistributedDevs.jl +++ /dev/null @@ -1,321 +0,0 @@ -module DistributedDevs - -using Gridap -using GridapEmbedded -using GridapDistributed -using PartitionedArrays -using Test -using FillArrays - - -using GridapEmbedded.CSG -using GridapEmbedded.Interfaces: CUT -using Gridap.Geometry -using Gridap.ReferenceFEs - -using GridapEmbedded.AgFEM: _touch_aggregated_cells! - -using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization - -function _aggregate( - strategy::AggregateCutCellsByThreshold, - cut::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out=IN) - - bgmodel = get_background_model(cut) - facet_to_inoutcut = compute_bgfacet_to_inoutcut(bgmodel,geo) - _distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut) -end - - -function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_inoutcut) - @assert loc in (IN,OUT) - - cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) - trian = Triangulation(cutgeo,cutinorout,geo) - model = get_background_model(cutgeo) - bgtrian = get_triangulation(model) - cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) - cell_to_meas = map(get_cell_measure,local_views(bgtrian)) - cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m - lazy_map(/,c_to_cm,c_to_m) - end - - cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) - - cell_to_coords = map(get_cell_coordinates,local_views(model)) - topo = get_grid_topology(model) - D = num_cell_dims(model) - cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) - face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) - gids = get_cell_gids(model) - - _distributed_aggregate_by_threshold_barrier( - threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, - loc,cell_to_coords,cell_to_faces,face_to_cells,gids) -end - - -function _distributed_aggregate_by_threshold_barrier( - threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, - loc,cell_to_coords,cell_to_faces,face_to_cells,gids) - - - ocell_to_touched = map(cell_to_unit_cut_meas) do c_to_m - map(≥,c_to_m,Fill(threshold,length(c_to_m))) - end - cell_to_touched = _add_ghost_values(ocell_to_touched,gids) - - cell_to_root_centroid = map(cell_to_coords) do cell_to_coords - map(i->sum(i)/length(i),cell_to_coords) - end - PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait - - n_cells = map(length,cell_to_touched) - touched_cells = map(findall,cell_to_touched) - - cell_to_cellin = map(n->zeros(Int32,n),n_cells) - map(cell_to_cellin,touched_cells,local_to_global(gids)) do c_to_ci,cells,l_to_g - gcells = lazy_map(Reindex(l_to_g),cells) - c_to_ci[cells] = gcells - end - - cell_to_neig = map(n->zeros(Int32,n),n_cells) - - cell_to_root_part = map(collect,local_to_owner(gids)) - - c1 = map(array_cache,cell_to_faces) - c2 = map(array_cache,face_to_cells) - - max_iters = 1 - for iter in 1:max_iters - all_aggregated = _aggregate_one_step!(c1,c2,gids, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - PVector(cell_to_touched,partition(gids)) |> consistent! |> wait - PVector(cell_to_neig,partition(gids)) |> consistent! |> wait - PVector(cell_to_cellin,partition(gids)) |> consistent! |> wait - PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait - PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait - - reduction!(&,all_aggregated,all_aggregated) - emit!(all_aggregated,all_aggregated) - - if PartitionedArrays.getany(all_aggregated) - break - end - end - - cell_to_cellin, cell_to_root_part, cell_to_neig -end - -function _aggregate_one_step!(c1,c2,gids::PRange, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - map(c1,c2,own_to_local(gids), - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - local_to_global(gids), - cell_to_faces, - face_to_cells, - facet_to_inoutcut) do c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut - - _aggregate_one_step!( - c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - end -end - -function _aggregate_one_step!( - c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - all_aggregated = true - for cell in own_cells - if ! cell_to_touched[cell] && cell_to_inoutcut[cell] == CUT - neigh_cell = _find_best_neighbor_from_centroid_distance( - c1,c2,cell, - cell_to_faces, - face_to_cells, - cell_to_touched, - cell_to_root_centroid, - facet_to_inoutcut, - loc) - if neigh_cell > 0 - cellin = cell_to_cellin[neigh_cell] - centroid = cell_to_root_centroid[neigh_cell] - part = cell_to_root_part[neigh_cell] - neigh_gcell = cell_to_gcell[neigh_cell] - - cell_to_neig[cell] = neigh_gcell - cell_to_cellin[cell] = cellin - cell_to_root_centroid[cell] = centroid - cell_to_root_part[cell] = part - else - all_aggregated = false - end - end - end - _touch_aggregated_cells!(cell_to_touched,cell_to_cellin) - all_aggregated -end - -function _find_best_neighbor_from_centroid_distance( - c1,c2,cell, - cell_to_faces, - face_to_cells, - cell_to_touched, - cell_to_root_centroid, - facet_to_inoutcut, - loc) - - faces = getindex!(c1,cell_to_faces,cell) - dmin = Inf - T = eltype(eltype(face_to_cells)) - best_neigh_cell = zero(T) - for face in faces - inoutcut = facet_to_inoutcut[face] - if inoutcut != CUT && inoutcut != loc - continue - end - neigh_cells = getindex!(c2,face_to_cells,face) - for neigh_cell in neigh_cells - if neigh_cell != cell && cell_to_touched[neigh_cell] - p = cell_to_root_centroid[neigh_cell] - q = cell_to_root_centroid[cell] - d = norm(p-q) - if (1.0+1.0e-9)*d < dmin - dmin = d - best_neigh_cell = neigh_cell - end - end - end - end - best_neigh_cell -end - -function _add_ghost_values(own_v,gids::PRange) - lens = map(length,local_views(gids)) - eltypes = map(eltype,own_v) - local_v = map(zeros,eltypes,lens) - map(local_v,own_v,own_to_local(gids)) do l,o,o_to_l - l[o_to_l] = o - end - PVector(local_v,partition(gids)) |> consistent! |> wait - local_v -end - -## Parallel aggregation - -using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization - -function _aggregate( - stragegy, - cutgeo::DistributedEmbeddedDiscretization, - in_or_out::Integer=IN) - - map(local_views(cutgeo)) do cutgeo - aggregate(stragegy,cutgeo,cutgeo.geo,in_or_out) - end -end - -# Driver - -distribute = identity - -np = (2,2) - -ranks = distribute(LinearIndices((prod(np),))) - -L = 1 -p0 = Point(0.0,0.0) -pmin = p0-L/2 -pmax = p0+L/2 - - -R = 0.35 -geo = disk(R,x0=p0) - -n = 8 -mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) - -cutgeo = cut(bgmodel,geo) - -bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) - -Ω = Triangulation(cutgeo) - -writevtk(Ω,"trian") - -strategy = AggregateCutCellsByThreshold(0.8) -aggregates,aggregate_owner,aggregate_neig = _aggregate(strategy,cutgeo,geo,IN) - -# TODO: -# - print aggregates -# - move to src -# - test aggregates with several geometries -# - reconstruct paths -# - add remote roots to model - - -# TODO: parallel aggregation (parallel aggfem article) -# 0. exchange measures and aggregates through gid -# 1. root in ghost layer -# 2. root in neighbors -# 3. root in neighbors of neighbors - -end # module From da5d3bf4aed522dfb99616974ec08d8f425dc890 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 5 Apr 2024 17:17:12 +0200 Subject: [PATCH 12/31] =?UTF-8?q?[distributed]=20testing=20root=20aggregat?= =?UTF-8?q?es=20in=20ghost=20layer=20[=E2=9C=93]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Distributed/Distributed.jl | 9 +- test/DistributedTests/AggregationTests.jl | 8 +- test/DistributedTests/DistributedDevs.jl | 181 ++++++++++++++++++++++ 3 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 test/DistributedTests/DistributedDevs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 2df7dbb..2c63c4a 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -302,7 +302,7 @@ function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_i trian = Triangulation(cutgeo,cutinorout,geo) model = get_background_model(cutgeo) bgtrian = get_triangulation(model) - cell_to_cut_meas = map(get_cell_measure,local_views(trian),local_views(bgtrian)) + cell_to_cut_meas = map(_get_cell_measure,local_views(trian),local_views(bgtrian)) cell_to_meas = map(get_cell_measure,local_views(bgtrian)) cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m lazy_map(/,c_to_cm,c_to_m) @@ -524,6 +524,13 @@ function _add_ghost_values(own_v,gids::PRange) local_v end +function _get_cell_measure(trian1::Triangulation,trian2::Triangulation) + if num_cells(trian1) == 0 + Fill(0.0,num_cells(trian2)) + else + get_cell_measure(trian1,trian2) + end +end end # module diff --git a/test/DistributedTests/AggregationTests.jl b/test/DistributedTests/AggregationTests.jl index 38d3706..0dff75c 100644 --- a/test/DistributedTests/AggregationTests.jl +++ b/test/DistributedTests/AggregationTests.jl @@ -83,7 +83,7 @@ writevtk(Ωbg,"bgtrian",celldata= # TODO: # - print aggregates [x] -# - move to src +# - move to src [x] # - test aggregates with several geometries [x] # - reconstruct paths # - add remote roots to model @@ -94,4 +94,10 @@ writevtk(Ωbg,"bgtrian",celldata= # 2. root in neighbors # 3. root in neighbors of neighbors + +# TODO: +# - test possion eq with root in ghost layer (simplify this geometry) +# - test graph reconstruction in 1D arrays +# - add remote roots to model + end # module diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl new file mode 100644 index 0000000..300435b --- /dev/null +++ b/test/DistributedTests/DistributedDevs.jl @@ -0,0 +1,181 @@ +module DistributedDevs + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.Distributed: distributed_aggregate + +distribute = PartitionedArrays.DebugArray + +np = (2,1) + +ranks = distribute(LinearIndices((prod(np),))) + +L = 1 +p0 = Point(0.0,0.0) +pmin = p0-L/2 +pmax = p0+L/2 + +R = 0.3 +x = VectorValue(1.0,0.0) + +d1 = VectorValue(0.4,0.0) +d2 = VectorValue(0.0,0.05) +geo = disk(R,x0=p0) +# geo2 = quadrilateral(;x0=p0-d2/2,d1=d1,d2=d2) +# geo = union(geo1,geo2) + +n = 9 +mesh_partition = (n,n) +bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) + +cutgeo = cut(bgmodel,geo) + +bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) + +Ω = Triangulation(cutgeo) + +writevtk(Ω,"trian") + +strategy = AggregateCutCellsByThreshold(1.0) +aggregates,aggregate_owner,aggregate_neig = distributed_aggregate( + strategy,cutgeo,geo,IN) + + +gids = get_cell_gids(bgmodel) +local_aggretation = map(aggregate_owner,own_to_local(gids),own_to_owner(gids)) do owns,o_to_l,o + owners = map(Reindex(owns),o_to_l) + all(lazy_map(==,owners,o)) +end +has_local_aggretation = reduction(&,local_aggretation) |> emit |> PartitionedArrays.getany + +remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l + lazy_map(agg) do a + iszero(a) || !iszero(g_to_l[a]) + end |> all |> ! +end +has_remote_aggregation = reduction(|,remote_aggregation) |> emit |> PartitionedArrays.getany + +@test !has_local_aggretation +@test !has_remote_aggregation + +cut_in = map(aggregates) do agg + findall(!iszero,agg) +end + +cut_owner = map(aggregate_owner,cut_in) do owners,cut_in + owners[cut_in] +end + +aggregates +gids = get_cell_gids(bgmodel) + +laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l + map(agg) do a + if iszero(a) + a + else + g_to_l[a] + end + end +end + +oaggregates = map(aggregates,own_to_local(gids)) do agg,o_to_l + map(Reindex(agg),o_to_l) +end + +oaggregate_owner = map(aggregate_owner,own_to_local(gids)) do agg,o_to_l + map(Reindex(agg),o_to_l) +end + + +Ωbg = Triangulation(bgmodel) +Ω = Triangulation(cutgeo) +Ωin = Triangulation(cutgeo,IN) +Γ = EmbeddedBoundary(cutgeo) +Ω_act = Triangulation(cutgeo,ACTIVE) + + + +n_Γ = get_normal_vector(Γ) + +order = 1 +degree = 2*order +dΩ = Measure(Ω,degree) +dΓ = Measure(Γ,degree) + +reffe = ReferenceFE(lagrangian,Float64,order) + +Vstd = FESpace(Ω_act,reffe) + + +map(num_cells,local_views(Ω_act)) + + +map(local_views(Ω_act),ranks) do trian,p + writevtk(trian,"trian_act_$p") +end + +V = AgFEMSpace(bgmodel,Vstd,laggregates) +# V = Vstd +U = TrialFESpace(V) + +h = 1.0/n +γd = 10.0 + +u(x) = x[1] - x[2] +f(x) = -Δ(u)(x) +ud(x) = u(x) + + +a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + +l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + +op = AffineFEOperator(a,l,U,V) +uh = solve(op) + +e = u - uh + +l2(u) = sqrt(sum( ∫( u*u )*dΩ )) +h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + +el2 = l2(e) +eh1 = h1(e) +ul2 = l2(uh) +uh1 = h1(uh) + + +writevtk(Ωin,"trian_in") +writevtk(Γ,"bnd") +writevtk(Ωbg,"bgtrian",celldata= + ["aggregate"=>oaggregates, + "aggregate_owner"=>oaggregate_owner]) + +writevtk(Ω,"trian",cellfields=["uh"=>uh]) +# TODO: +# - print aggregates [x] +# - move to src [x] +# - test aggregates with several geometries [x] +# - reconstruct paths +# - add remote roots to model + +# TODO: parallel aggregation (parallel aggfem article) +# 0. exchange measures and aggregates through gid +# 1. root in ghost layer +# 2. root in neighbors +# 3. root in neighbors of neighbors + + +# TODO: +# - test possion eq with root in ghost layer (simplify this geometry) +# - test graph reconstruction in 1D arrays +# - add remote roots to model +end From 3d678b22e33b481e22a5f6aafb366c0a94561a34 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Mon, 8 Apr 2024 18:12:33 +0200 Subject: [PATCH 13/31] [distributed] add remote aggregates to bgmodel --- src/Distributed/Distributed.jl | 113 ++++++++++++++++++++++- test/DistributedTests/DistributedDevs.jl | 38 +++++++- 2 files changed, 144 insertions(+), 7 deletions(-) diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 2c63c4a..839a346 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -374,8 +374,7 @@ function _distributed_aggregate_by_threshold_barrier( PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait - reduction!(&,all_aggregated,all_aggregated) - emit!(all_aggregated,all_aggregated) + reduction!(&,all_aggregated,all_aggregated,destination=:all) if PartitionedArrays.getany(all_aggregated) break @@ -532,5 +531,115 @@ function _get_cell_measure(trian1::Triangulation,trian2::Triangulation) end end +function add_remote_aggregates(model::DistributedDiscreteModel,aggregates,aggregate_owner) + gids = get_cell_gids(model) + remote_cells,remote_parts = _extract_remote_cells(gids,aggregates,aggregate_owner) + remote_cells,remote_parts = _group_remote_ids(remote_cells,remote_parts) + add_remote_cells(model,remote_cells,remote_parts) +end + +function _extract_remote_cells(gids::PRange,aggregates,aggregate_owner) + remote_aggids = map(aggregates,global_to_local(gids)) do agg,g_to_l + ids = findall(agg) do i + !iszero(i) && iszero(g_to_l[i]) + end + unique(Reindex(agg),ids) + end + + remote_cells = map(aggregates,remote_aggids) do agg,ids + map(Reindex(agg),ids) + end + + remote_parts = map(aggregate_owner,remote_aggids) do agg,ids + map(Reindex(agg),ids) + end + + remote_cells,remote_parts +end + +function _group_remote_ids(remote_ids,remote_parts) + new_parts = map(sort∘unique,remote_parts) + new_ids = map(remote_ids,remote_parts,new_parts) do ids,parts,uparts + grouped_ids = map(i->Int[],1:length(uparts)) + for (id,p) in zip(ids,parts) + j = findfirst(==(p),uparts) + union!(grouped_ids[j],id) + end + map!(sort!,grouped_ids,grouped_ids) + end + new_ids,new_parts +end + +function _ungroup_remote_ids(remote_ids,remote_parts) + new_ids = map(remote_ids) do ids + reduce(append!,ids,init=eltype(eltype(ids))[]) + end + new_parts = map(remote_ids,remote_parts) do ids,parts + n = map(length,ids) + parts_v = map((p,n)->Fill(p,n),parts,n) + reduce(append!,parts_v,init=eltype(parts)[]) + end + new_ids,new_parts +end + +function add_remote_ids(gids::PRange,remote_gids,remote_parts) + new_gids,new_parts = _ungroup_remote_ids(remote_gids,remote_parts) + lid_to_gid = map(vcat,local_to_global(gids),new_gids) + lid_to_part = map(vcat,local_to_owner(gids),new_parts) + p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p + LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) + end + PRange(p) +end + +function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_parts) + # Send remote gids to owners + snd_ids = remote_parts + snd_remotes = remote_cells + graph = ExchangeGraph(snd_ids) + rcv_remotes = allocate_exchange(snd_remotes,graph) + exchange!(rcv_remotes,snd_remotes,graph) |> wait + + # Send remote coordinates + gids = get_cell_gids(model) + snd_gids = rcv_remotes + snd_lids = map(global_to_local(gids),snd_gids) do g_to_l,gids + map(Reindex(g_to_l),gids) + end + snd_coords = map(local_views(model),snd_lids) do m,lids + map(lids) do lids + coords = map(Reindex(get_cell_coordinates(m)),lids) + reduce(append!,coords,init=eltype(eltype(coords))[]) + end + end + rgraph = reverse(graph) + rcv_coords = allocate_exchange(snd_coords,rgraph) + exchange!(rcv_coords,snd_coords,rgraph) |> wait + + # Build remote grids + ncells = map(remote_cells) do cells + sum(length,cells) + end + reffes = map(get_reffes,local_views(model)) + reffe = map(only,reffes) + ctypes = map(n->ones(Int,n),ncells) + coords = map(PartitionedArrays.getdata,rcv_coords) + conn = map(ncells,reffe) do ncells,reffe + n = num_nodes(reffe) + data = 1:n*ncells + ptrs = 1:n:n*ncells+1 + Table(data,ptrs) + end + rgrids = map(UnstructuredGrid,coords,conn,reffes,ctypes) + + # Build appended model + lgrids = map(get_grid,local_views(model)) + grids = map(lazy_append,lgrids,rgrids) + models = map(UnstructuredDiscreteModel,grids) + agids = add_remote_ids(gids,remote_cells,remote_parts) + DistributedDiscreteModel(models,agids) +end + + end # module diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl index 300435b..d969821 100644 --- a/test/DistributedTests/DistributedDevs.jl +++ b/test/DistributedTests/DistributedDevs.jl @@ -50,14 +50,14 @@ local_aggretation = map(aggregate_owner,own_to_local(gids),own_to_owner(gids)) d owners = map(Reindex(owns),o_to_l) all(lazy_map(==,owners,o)) end -has_local_aggretation = reduction(&,local_aggretation) |> emit |> PartitionedArrays.getany +has_local_aggretation = reduction(&,local_aggretation,destination=:all) |> PartitionedArrays.getany remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l lazy_map(agg) do a iszero(a) || !iszero(g_to_l[a]) end |> all |> ! end -has_remote_aggregation = reduction(|,remote_aggregation) |> emit |> PartitionedArrays.getany +has_remote_aggregation = reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany @test !has_local_aggretation @test !has_remote_aggregation @@ -160,12 +160,40 @@ writevtk(Ωbg,"bgtrian",celldata= "aggregate_owner"=>oaggregate_owner]) writevtk(Ω,"trian",cellfields=["uh"=>uh]) + + + + +using GridapEmbedded.Distributed: add_remote_aggregates + +np = (3,) + +ranks = distribute(LinearIndices((prod(np),))) +m = CartesianDiscreteModel(ranks,np,(0, 1),(9,)) + + +aggregates = DebugArray([[1, 9, 8, 5], [3, 4, 9, 6, 7], [6, 7, 3, 9]]) +aggregate_owner = DebugArray([[1, 3, 3, 2], [1, 2, 3, 2, 3], [2, 3, 1, 3]]) + +am = add_remote_aggregates(m,aggregates,aggregate_owner) + +gids = partition(get_cell_gids(am)) +test_gids = DebugArray([[1,2,3,4,5,8,9],[3,4,5,6,7,9],[6,7,8,9,3]]) +map(gids,test_gids) do gids,_gids + @test all(gids .==_gids) +end + +map(local_views(am),ranks) do m,p + writevtk(Triangulation(m),"m$p") +end + # TODO: # - print aggregates [x] # - move to src [x] # - test aggregates with several geometries [x] -# - reconstruct paths -# - add remote roots to model +# - reconstruct paths [x] +# - add remote roots to model [x] +# - migrate model # TODO: parallel aggregation (parallel aggfem article) # 0. exchange measures and aggregates through gid @@ -173,9 +201,9 @@ writevtk(Ω,"trian",cellfields=["uh"=>uh]) # 2. root in neighbors # 3. root in neighbors of neighbors - # TODO: # - test possion eq with root in ghost layer (simplify this geometry) # - test graph reconstruction in 1D arrays # - add remote roots to model + end From db1da3cac1782e7c57ddaf5de8d1604430a16fb5 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Tue, 9 Apr 2024 17:33:35 +0200 Subject: [PATCH 14/31] [distributed] testing remotes on bgmodels (not working) --- src/Distributed/Distributed.jl | 58 ++++++- test/DistributedTests/DistributedDevs.jl | 39 ++++- test/DistributedTests/testing_remote_aggs.jl | 164 +++++++++++++++++++ 3 files changed, 250 insertions(+), 11 deletions(-) create mode 100644 test/DistributedTests/testing_remote_aggs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 839a346..a47e8f4 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -241,10 +241,41 @@ function compute_bgcell_to_inoutcut(cutgeo::DistributedEmbeddedDiscretization,ar end end +function change_bgmodel( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel, + args...) + + cuts = _change_bgmodels(cutgeo,model,args...) + gids = get_cell_gids(model) + ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) + _consistent!(ls_to_bgcell_to_inoutcut,gids) + DistributedEmbeddedDiscretization(cuts,model) +end + +function _change_bgmodels( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel, + cell_to_newcell) + + map(local_views(cutgeo),local_views(model),cell_to_newcell) do c,m,c_to_nc + change_bgmodel(c,m,c_to_nc) + end +end + +function _change_bgmodels( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel) + + map(local_views(cutgeo),local_views(model)) do c,m + change_bgmodel(c,m) + end +end + function change_bgmodel( cut::EmbeddedDiscretization, newmodel::DiscreteModel, - cell_to_newcell) + cell_to_newcell=1:num_cells(get_background_model(cut))) ls_to_bgc_to_ioc = map(cut.ls_to_bgcell_to_inoutcut) do bgc_to_ioc new_bgc_to_ioc = Vector{Int8}(undef,num_cells(newmodel)) @@ -594,6 +625,7 @@ end function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_parts) # Send remote gids to owners + display(remote_cells) snd_ids = remote_parts snd_remotes = remote_cells graph = ExchangeGraph(snd_ids) @@ -607,10 +639,12 @@ function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_pa map(Reindex(g_to_l),gids) end snd_coords = map(local_views(model),snd_lids) do m,lids - map(lids) do lids + T = eltype(eltype(get_cell_coordinates(m))) + coords = map(lids) do lids coords = map(Reindex(get_cell_coordinates(m)),lids) - reduce(append!,coords,init=eltype(eltype(coords))[]) + reduce(append!,coords,init=T[]) end + Vector{Vector{T}}(coords) end rgraph = reverse(graph) rcv_coords = allocate_exchange(snd_coords,rgraph) @@ -618,7 +652,7 @@ function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_pa # Build remote grids ncells = map(remote_cells) do cells - sum(length,cells) + sum(length,cells,init=0) end reffes = map(get_reffes,local_views(model)) reffe = map(only,reffes) @@ -641,5 +675,21 @@ function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_pa end +function has_remote_aggregation(model::DistributedDiscreteModel,aggregates) + gids = get_cell_gids(model) + has_remote_aggregation(aggregates,gids) +end + +function has_remote_aggregation(aggregates,gids::PRange) + remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l + lazy_map(agg) do a + iszero(a) || !iszero(g_to_l[a]) + end |> all |> ! + end + display( remote_aggregation) + reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany +end + + end # module diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl index d969821..f25a417 100644 --- a/test/DistributedTests/DistributedDevs.jl +++ b/test/DistributedTests/DistributedDevs.jl @@ -7,6 +7,7 @@ using PartitionedArrays using Test using GridapEmbedded.Distributed: distributed_aggregate +using GridapEmbedded.Distributed: has_remote_aggregation distribute = PartitionedArrays.DebugArray @@ -46,21 +47,19 @@ aggregates,aggregate_owner,aggregate_neig = distributed_aggregate( gids = get_cell_gids(bgmodel) +# TODO: do not use owner for this query (has ghost aggregation?) local_aggretation = map(aggregate_owner,own_to_local(gids),own_to_owner(gids)) do owns,o_to_l,o owners = map(Reindex(owns),o_to_l) all(lazy_map(==,owners,o)) end has_local_aggretation = reduction(&,local_aggretation,destination=:all) |> PartitionedArrays.getany -remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l - lazy_map(agg) do a - iszero(a) || !iszero(g_to_l[a]) - end |> all |> ! -end -has_remote_aggregation = reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany + +@test !has_remote_aggregation(bgmodel,aggregates) + @test !has_local_aggretation -@test !has_remote_aggregation +# @test !has_remote_aggregation cut_in = map(aggregates) do agg findall(!iszero,agg) @@ -187,6 +186,32 @@ map(local_views(am),ranks) do m,p writevtk(Triangulation(m),"m$p") end + +# import GridapEmbedded.Distributed: change_bgmodel +# using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization +# using GridapDistributed: DistributedDiscreteModel + + + + + +# cutgeo::DistributedEmbeddedDiscretization +# map local_views(cutgeo) +# change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) +# end +# ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) +# _consistent!(ls_to_bgcell_to_inoutcut,gids) +# DistributedEmbeddedDiscretization(cuts,bgmodel) + +# change_bgmodel( +# cut::EmbeddedDiscretization, +# newmodel::DiscreteModel, +# cell_to_newcell) + +# cell_to_newcell = 1:num_cells(get_background_model(cut)) + + + # TODO: # - print aggregates [x] # - move to src [x] diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl new file mode 100644 index 0000000..41da5dc --- /dev/null +++ b/test/DistributedTests/testing_remote_aggs.jl @@ -0,0 +1,164 @@ +module testing_remote_aggs + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.Distributed: distributed_aggregate +using GridapEmbedded.Distributed: has_remote_aggregation +using GridapEmbedded.Distributed: add_remote_aggregates +using GridapEmbedded.Distributed: change_bgmodel + +distribute = PartitionedArrays.DebugArray + + +np = (3,1) +ranks = distribute(LinearIndices((prod(np),))) + +nc = (9,9) + +L = 1 + +x0 = Point(0.1,0.1) +d1 = VectorValue(0.8,0.0) +d2 = VectorValue(0.0,0.05) +geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +x0 = Point(0.1,0.1) +d1 = VectorValue(0.2,0.0) +d2 = VectorValue(0.0,0.8) +geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +x0 = Point(0.2,0.6) +d1 = VectorValue(0.7,0.0) +d2 = VectorValue(0.0,0.3) +geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) + + +geo12 = union(geo1,geo2) +geo = union(geo12,geo3) +# geo = geo12 + + +domain = (0, 1, 0, 1) +bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) + +cutgeo = cut(bgmodel,geo) + +strategy = AggregateCutCellsByThreshold(1.0) +aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN) + +@test has_remote_aggregation(bgmodel,aggregates) + +_bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) + +@test ! has_remote_aggregation(_bgmodel,aggregates) + +_cutgeo = change_bgmodel(cutgeo,_bgmodel) + + +gids = get_cell_gids(_bgmodel) +_aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g + _agg = zeros(Int,length(l_to_g)) + for (l,g) in enumerate(l_to_g) + _agg[l] = l > length(agg) ? g : agg[l] + end + _agg +end + + +bgmodel = _bgmodel +cutgeo = _cutgeo +aggregates = _aggregates + +gids = get_cell_gids(bgmodel) +laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l + map(agg) do i + iszero(i) ? i : g_to_l[i] + end +end + + +Ω_bg = Triangulation(bgmodel) +Ω_act = Triangulation(cutgeo,ACTIVE) +Ω = Triangulation(cutgeo,PHYSICAL) +Γ = EmbeddedBoundary(cutgeo) +n_Γ = get_normal_vector(Γ) + +order = 1 +degree = 2*order +dΩ = Measure(Ω,degree) +dΓ = Measure(Γ,degree) + +reffe = ReferenceFE(lagrangian,Float64,order) + +Vstd = FESpace(Ω_act,reffe) +map(length,laggregates) + +V = AgFEMSpace(bgmodel,Vstd,laggregates) +U = TrialFESpace(V) + + +u(x) = x[1] - x[2] +f(x) = -Δ(u)(x) +ud(x) = u(x) + +γd = 10.0 + +a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + +l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + +op = AffineFEOperator(a,l,U,V) +uh = solve(op) + + +e = u - uh + +l2(u) = sqrt(sum( ∫( u*u )*dΩ )) +h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + +el2 = l2(e) +eh1 = h1(e) +ul2 = l2(uh) +uh1 = h1(uh) + + + +Ω = Triangulation(cutgeo) +Γ = EmbeddedBoundary(cutgeo) +Ωbg = Triangulation(bgmodel) + +writevtk(Ω,"trian") +writevtk(Γ,"bnd") +writevtk(Ωbg,"bg_trian") + +map(local_views(_bgmodel),ranks) do m,p + writevtk(get_grid(m),"bgmodel_$p") +end + + + + + + + +# TODO: +# only own remotes, not ghost remotes +# add remotes (itself) to aggregate array + + + +# Implement C shape domain with one thin branch + +# Test remotes by changing bgmodel + + + +end # module From 276389c11fa38048f78e8d39d3f69f693e9bd9cb Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Wed, 10 Apr 2024 11:24:55 +0200 Subject: [PATCH 15/31] [distributed] bugfix and testing remote ghost with no aggregation (ag fe space with remote aggregates not working) --- src/Distributed/Distributed.jl | 17 +- test/DistributedTests/DistributedDevs.jl | 38 ++--- test/DistributedTests/testing_remote_aggs.jl | 78 ++++++--- .../testing_remote_no_aggs.jl | 150 ++++++++++++++++++ 4 files changed, 226 insertions(+), 57 deletions(-) create mode 100644 test/DistributedTests/testing_remote_no_aggs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index a47e8f4..80e6dca 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -184,11 +184,14 @@ function _consistent!( p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, prange::PRange) - tables = map(Table,p_to_i_to_a) - data = map(t->t.data,tables) - pdata = PVector(data,partition(prange)) - consistent!(pdata) |> wait - map(copyto!,p_to_i_to_a,tables) + n = map(length,p_to_i_to_a) |> PartitionedArrays.getany + for i in 1:n + p_to_a = map(i_to_a->i_to_a[i],p_to_i_to_a) + PVector(p_to_a,partition(prange)) |> consistent! |> wait + map(p_to_a,p_to_i_to_a) do p_to_a,p_to_ia + copyto!(p_to_ia[i],p_to_a) + end + end end @@ -625,7 +628,6 @@ end function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_parts) # Send remote gids to owners - display(remote_cells) snd_ids = remote_parts snd_remotes = remote_cells graph = ExchangeGraph(snd_ids) @@ -686,8 +688,7 @@ function has_remote_aggregation(aggregates,gids::PRange) iszero(a) || !iszero(g_to_l[a]) end |> all |> ! end - display( remote_aggregation) - reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany + reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany end diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl index f25a417..33f686e 100644 --- a/test/DistributedTests/DistributedDevs.jl +++ b/test/DistributedTests/DistributedDevs.jl @@ -191,44 +191,24 @@ end # using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization # using GridapDistributed: DistributedDiscreteModel - - - - -# cutgeo::DistributedEmbeddedDiscretization -# map local_views(cutgeo) -# change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) -# end -# ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) -# _consistent!(ls_to_bgcell_to_inoutcut,gids) -# DistributedEmbeddedDiscretization(cuts,bgmodel) - -# change_bgmodel( -# cut::EmbeddedDiscretization, -# newmodel::DiscreteModel, -# cell_to_newcell) - -# cell_to_newcell = 1:num_cells(get_background_model(cut)) - - - # TODO: # - print aggregates [x] # - move to src [x] # - test aggregates with several geometries [x] # - reconstruct paths [x] # - add remote roots to model [x] -# - migrate model +# - migrate model [x] +# - agfem fe space with remotes # TODO: parallel aggregation (parallel aggfem article) -# 0. exchange measures and aggregates through gid -# 1. root in ghost layer -# 2. root in neighbors -# 3. root in neighbors of neighbors +# 0. exchange measures and aggregates through gid [x] +# 1. root in ghost layer [x] +# 2. root in neighbors [x] +# 3. root in neighbors of neighbors [x] # TODO: -# - test possion eq with root in ghost layer (simplify this geometry) -# - test graph reconstruction in 1D arrays -# - add remote roots to model +# - test possion eq with root in ghost layer (simplify this geometry) [x] +# - test graph reconstruction in 1D arrays [x] +# - add remote roots to model [x] end diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl index 41da5dc..21bfdf3 100644 --- a/test/DistributedTests/testing_remote_aggs.jl +++ b/test/DistributedTests/testing_remote_aggs.jl @@ -21,26 +21,51 @@ nc = (9,9) L = 1 +# C shape x0 = Point(0.1,0.1) d1 = VectorValue(0.8,0.0) d2 = VectorValue(0.0,0.05) geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) -x0 = Point(0.1,0.1) +x0 = Point(0.15,0.125) d1 = VectorValue(0.2,0.0) -d2 = VectorValue(0.0,0.8) +d2 = VectorValue(0.0,0.6) geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) -x0 = Point(0.2,0.6) -d1 = VectorValue(0.7,0.0) +x0 = Point(0.1,0.6) +d1 = VectorValue(0.8,0.0) d2 = VectorValue(0.0,0.3) geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) - geo12 = union(geo1,geo2) geo = union(geo12,geo3) -# geo = geo12 +geo = geo12 + +# Cirlce (test) +R = 0.35 +p0 = Point(0.5,0.5) +geo = disk(R,x0=p0) +geo = geo3 + + +# Bigger cut cells (test) +x0 = Point(0.05,0.05) +d1 = VectorValue(0.9,0.0) +d2 = VectorValue(0.0,0.15) +geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) +x0 = Point(0.15,0.125) +d1 = VectorValue(0.25,0.0) +d2 = VectorValue(0.0,0.6) +geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +x0 = Point(0.05,0.6) +d1 = VectorValue(0.9,0.0) +d2 = VectorValue(0.0,0.35) +geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +geo12 = union(geo1,geo2) +geo = union(geo12,geo3) domain = (0, 1, 0, 1) bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) @@ -48,7 +73,8 @@ bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) cutgeo = cut(bgmodel,geo) strategy = AggregateCutCellsByThreshold(1.0) -aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN) +aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN); + @test has_remote_aggregation(bgmodel,aggregates) @@ -60,6 +86,7 @@ _cutgeo = change_bgmodel(cutgeo,_bgmodel) gids = get_cell_gids(_bgmodel) +# Add remote to aggregates _aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g _agg = zeros(Int,length(l_to_g)) for (l,g) in enumerate(l_to_g) @@ -74,6 +101,7 @@ cutgeo = _cutgeo aggregates = _aggregates gids = get_cell_gids(bgmodel) +# Local aggregates laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l map(agg) do i iszero(i) ? i : g_to_l[i] @@ -95,9 +123,9 @@ dΓ = Measure(Γ,degree) reffe = ReferenceFE(lagrangian,Float64,order) Vstd = FESpace(Ω_act,reffe) -map(length,laggregates) V = AgFEMSpace(bgmodel,Vstd,laggregates) +V = Vstd U = TrialFESpace(V) @@ -106,6 +134,7 @@ f(x) = -Δ(u)(x) ud(x) = u(x) γd = 10.0 +h = 1/nc[1] a(u,v) = ∫( ∇(v)⋅∇(u) ) * dΩ + @@ -130,35 +159,44 @@ ul2 = l2(uh) uh1 = h1(uh) - Ω = Triangulation(cutgeo) Γ = EmbeddedBoundary(cutgeo) Ωbg = Triangulation(bgmodel) +Ω_act = Triangulation(cutgeo,ACTIVE) + +writevtk(Ω,"trian"); +writevtk(Γ,"bnd"); +writevtk(Ωbg,"bg_trian"); +writevtk(Ω_act,"act_trian"); -writevtk(Ω,"trian") -writevtk(Γ,"bnd") -writevtk(Ωbg,"bg_trian") +writevtk(Ω,"trian", + cellfields=["uh"=>uh,"u"=>u,"e"=>e],); + + +# Debug VTK map(local_views(_bgmodel),ranks) do m,p writevtk(get_grid(m),"bgmodel_$p") end +map(local_views(uh),local_views(bgmodel),ranks) do uh,m,p + trian = Triangulation(m) + writevtk(trian,"ltrian_$p",cellfields=["uh"=>uh]) +end +map(local_views(cutgeo),ranks) do c,p + m = get_background_model(c) + c_io = compute_bgcell_to_inoutcut(c,c.geo) + writevtk(Triangulation(m),"bgtrian_$p",celldata=["inout"=>c_io]) +end # TODO: # only own remotes, not ghost remotes -# add remotes (itself) to aggregate array - - - -# Implement C shape domain with one thin branch - -# Test remotes by changing bgmodel - +# add remotes (itself) to aggregate array [x] end # module diff --git a/test/DistributedTests/testing_remote_no_aggs.jl b/test/DistributedTests/testing_remote_no_aggs.jl new file mode 100644 index 0000000..87e5d1a --- /dev/null +++ b/test/DistributedTests/testing_remote_no_aggs.jl @@ -0,0 +1,150 @@ +module testing_remote_no_aggs + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test + +using GridapEmbedded.Distributed: distributed_aggregate +using GridapEmbedded.Distributed: has_remote_aggregation +using GridapEmbedded.Distributed: add_remote_aggregates +using GridapEmbedded.Distributed: change_bgmodel + +distribute = PartitionedArrays.DebugArray + + +np = (3,1) +ranks = distribute(LinearIndices((prod(np),))) + +nc = (9,9) + +L = 1 + +x0 = Point(0.05,0.05) +d1 = VectorValue(0.9,0.0) +d2 = VectorValue(0.0,0.15) +geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +x0 = Point(0.15,0.125) +d1 = VectorValue(0.25,0.0) +d2 = VectorValue(0.0,0.6) +geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +x0 = Point(0.05,0.6) +d1 = VectorValue(0.9,0.0) +d2 = VectorValue(0.0,0.35) +geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) + +geo12 = union(geo1,geo2) +geo = union(geo12,geo3) + +domain = (0, 1, 0, 1) +bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) + +cutgeo = cut(bgmodel,geo) + +strategy = AggregateCutCellsByThreshold(1.0) +aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN); + + +@test has_remote_aggregation(bgmodel,aggregates) + +_bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) + +@test ! has_remote_aggregation(_bgmodel,aggregates) + +_cutgeo = change_bgmodel(cutgeo,_bgmodel) + +gids = get_cell_gids(_bgmodel) +# Add remote to aggregates +_aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g + _agg = zeros(Int,length(l_to_g)) + for (l,g) in enumerate(l_to_g) + _agg[l] = l > length(agg) ? g : agg[l] + end + _agg +end + +bgmodel = _bgmodel +cutgeo = _cutgeo +aggregates = _aggregates + +# Local aggregates +gids = get_cell_gids(bgmodel) +laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l + map(agg) do i + iszero(i) ? i : g_to_l[i] + end +end + +Ω_bg = Triangulation(bgmodel) +Ω_act = Triangulation(cutgeo,ACTIVE) +Ω = Triangulation(cutgeo,PHYSICAL) +Γ = EmbeddedBoundary(cutgeo) +n_Γ = get_normal_vector(Γ) + +order = 1 +degree = 2*order +dΩ = Measure(Ω,degree) +dΓ = Measure(Γ,degree) + +reffe = ReferenceFE(lagrangian,Float64,order) + +Vstd = FESpace(Ω_act,reffe) + +V = Vstd +U = TrialFESpace(V) + + +u(x) = x[1] - x[2] +f(x) = -Δ(u)(x) +ud(x) = u(x) + +γd = 10.0 +h = 1/nc[1] + +a(u,v) = + ∫( ∇(v)⋅∇(u) ) * dΩ + + ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ + +l(v) = + ∫( v*f ) * dΩ + + ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ + +op = AffineFEOperator(a,l,U,V) +uh = solve(op) + + +e = u - uh + +l2(u) = sqrt(sum( ∫( u*u )*dΩ )) +h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) + +el2 = l2(e) +eh1 = h1(e) +ul2 = l2(uh) +uh1 = h1(uh) + +@test el2/ul2 < 1e-9 +@test eh1/uh1 < 1e-9 + +Ω = Triangulation(cutgeo) +Γ = EmbeddedBoundary(cutgeo) +Ωbg = Triangulation(bgmodel) + +writevtk(Ω,"trian"); +writevtk(Γ,"bnd"); +writevtk(Ωbg,"bg_trian"); +writevtk(Ω_act,"act_trian"); + +writevtk(Ω,"trian", + cellfields=["uh"=>uh,"u"=>u,"e"=>e],); + + +map(local_views(uh),local_views(bgmodel),ranks) do uh,m,p + trian = Triangulation(m) + writevtk(trian,"ltrian_$p",cellfields=["uh"=>uh]) +end + +end # module From 1d3332acd6657017d8b18397aad52d3f53732f17 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 11 Apr 2024 14:17:00 +0200 Subject: [PATCH 16/31] [distributed] bugfix in generate agg gids --- src/Distributed/Distributed.jl | 31 ++++++++++++++++++++ test/DistributedTests/testing_remote_aggs.jl | 12 +++----- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 80e6dca..d08c603 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -146,8 +146,10 @@ function AgFEMSpace( trian = DistributedTriangulation(trians,bgmodel) trian = add_ghost_cells(trian) trian_gids = generate_cell_gids(trian) + cell_to_cellin = _active_aggregates(bgcell_to_bgcellin) cell_to_ldofs = map(get_cell_dof_ids,spaces) cell_to_ldofs = map(i->map(sort,i),cell_to_ldofs) + _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) nldofs = map(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) vector_type = _find_vector_type(spaces,gids) @@ -692,5 +694,34 @@ function has_remote_aggregation(aggregates,gids::PRange) end +function _active_aggregates(bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) + map(_active_aggregates,bgcell_to_bgcellin) +end + +function _active_aggregates(bgcell_to_bgcellin) + acell_to_bgcell = findall(!iszero,bgcell_to_bgcellin) + bgcell_to_acell = zeros(Int,length(bgcell_to_bgcellin)) + bgcell_to_acell[acell_to_bgcell] = 1:length(acell_to_bgcell) + acell_to_bgcellin = bgcell_to_bgcellin[ acell_to_bgcell ] + bgcell_to_acell[ acell_to_bgcellin ] +end + +function _remove_improper_cell_ldofs!( + cell_to_ldofs::AbstractVector{<:AbstractVector{<:AbstractVector}}, + bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) + + map(_remove_improper_cell_ldofs!,cell_to_ldofs,bgcell_to_bgcellin) +end + + +function _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) + for cell in 1:length(cell_to_ldofs) + cell_to_cellin[cell] != cell || continue + cell_to_ldofs[cell] = empty!(cell_to_ldofs[cell]) + end + cell_to_ldofs +end + + end # module diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl index 21bfdf3..4706cb4 100644 --- a/test/DistributedTests/testing_remote_aggs.jl +++ b/test/DistributedTests/testing_remote_aggs.jl @@ -123,9 +123,8 @@ dΓ = Measure(Γ,degree) reffe = ReferenceFE(lagrangian,Float64,order) Vstd = FESpace(Ω_act,reffe) - V = AgFEMSpace(bgmodel,Vstd,laggregates) -V = Vstd + U = TrialFESpace(V) @@ -193,10 +192,7 @@ map(local_views(cutgeo),ranks) do c,p end - -# TODO: -# only own remotes, not ghost remotes -# add remotes (itself) to aggregate array [x] - - +# TODO: encapsulate and reduce Interfaces +# add tests with mpi +# PR end # module From d30394c41e619f0d37973556856facf27d5ddebc Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 11 Apr 2024 17:02:40 +0200 Subject: [PATCH 17/31] [distributed] cleaning aggregation interface --- src/Distributed/Distributed.jl | 67 +++++++++----- test/DistributedTests/testing_remote_aggs.jl | 93 ++++++++++++-------- 2 files changed, 101 insertions(+), 59 deletions(-) diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index d08c603..2121938 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -105,28 +105,32 @@ function distributed_embedded_triangulation( end -function aggregate(stragegy,cutgeo::DistributedEmbeddedDiscretization) - aggregates = map(local_views(cutgeo)) do lcutgeo - aggregate(stragegy,lcutgeo) - end +function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) + aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,args...) bgmodel = get_background_model(cutgeo) - consistent_aggregates(aggregates,bgmodel) -end - -function consistent_aggregates(aggregates,bgmodel::DistributedDiscreteModel) - consistent_aggregates(aggregates,get_cell_gids(bgmodel)) -end - -function consistent_aggregates(aggregates,gids::PRange) - global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid - map(i-> iszero(i) ? i : gid[i],agg) - end - paggregates = PVector(global_aggregates,partition(gids)) - consistent!(paggregates) |> wait - map(local_values(paggregates),global_to_local(gids)) do agg,lgid - map(i-> iszero(i) ? i : lgid[i],agg) - end -end + if has_remote_aggregation(bgmodel,aggregates) + bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) + cutgeo = change_bgmodel(cutgeo,bgmodel) + aggregates = _change_model(aggregates,get_cell_gids(bgmodel)) + end + laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) + bgmodel,cutgeo,laggregates +end + +# function consistent_aggregates(aggregates,bgmodel::DistributedDiscreteModel) +# consistent_aggregates(aggregates,get_cell_gids(bgmodel)) +# end + +# function consistent_aggregates(aggregates,gids::PRange) +# global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid +# map(i-> iszero(i) ? i : gid[i],agg) +# end +# paggregates = PVector(global_aggregates,partition(gids)) +# consistent!(paggregates) |> wait +# map(local_values(paggregates),global_to_local(gids)) do agg,lgid +# map(i-> iszero(i) ? i : lgid[i],agg) +# end +# end function AgFEMSpace( bgmodel::DistributedDiscreteModel, @@ -722,6 +726,27 @@ function _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) cell_to_ldofs end +function _local_aggregates(cell_to_gcellin,gids::PRange) + map(_local_aggregates,cell_to_gcellin,global_to_local(gids)) +end + +function _local_aggregates(cell_to_gcellin,gcell_to_cell) + map(cell_to_gcellin) do gcin + iszero(gcin) ? gcin : gcell_to_cell[ gcin ] + end +end +function _change_model(cell_to_gcellin,gids::PRange) + map(_change_model,cell_to_gcellin,local_to_global(gids)) +end + +function _change_model(cell_to_gcellin,ncell_to_gcell) + ncells = length(cell_to_gcellin) + ncell_to_gcellin = zeros(Int,length(ncell_to_gcell)) + for (ncell,gcell) in enumerate(ncell_to_gcell) + ncell_to_gcellin[ncell] = ncell > ncells ? gcell : cell_to_gcellin[ncell] + end + ncell_to_gcellin +end end # module diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl index 4706cb4..abbeef4 100644 --- a/test/DistributedTests/testing_remote_aggs.jl +++ b/test/DistributedTests/testing_remote_aggs.jl @@ -67,46 +67,62 @@ geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) geo12 = union(geo1,geo2) geo = union(geo12,geo3) +geo = geo12 + domain = (0, 1, 0, 1) bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) cutgeo = cut(bgmodel,geo) strategy = AggregateCutCellsByThreshold(1.0) -aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN); -@test has_remote_aggregation(bgmodel,aggregates) +bgmodel,cutgeo,aggregates = aggregate(strategy,cutgeo,geo,IN); +# aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN); + +# if has_remote_aggregation(bgmodel,aggregates) +# bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) +# cutgeo = change_bgmodel(cutgeo,_bgmodel) +# aggregates = _change_model(aggregates,get_cell_gids(_bgmodel)) +# end + +# laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) + +# bgmodel,cutgeo,laggregates + + + +# @test has_remote_aggregation(bgmodel,aggregates) -_bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) +# _bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) -@test ! has_remote_aggregation(_bgmodel,aggregates) +# @test ! has_remote_aggregation(_bgmodel,aggregates) -_cutgeo = change_bgmodel(cutgeo,_bgmodel) +# _cutgeo = change_bgmodel(cutgeo,_bgmodel) -gids = get_cell_gids(_bgmodel) -# Add remote to aggregates -_aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g - _agg = zeros(Int,length(l_to_g)) - for (l,g) in enumerate(l_to_g) - _agg[l] = l > length(agg) ? g : agg[l] - end - _agg -end +# gids = get_cell_gids(_bgmodel) +# # Add remote to aggregates +# _aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g +# _agg = zeros(Int,length(l_to_g)) +# for (l,g) in enumerate(l_to_g) +# _agg[l] = l > length(agg) ? g : agg[l] +# end +# _agg +# end -bgmodel = _bgmodel -cutgeo = _cutgeo -aggregates = _aggregates +# bgmodel = _bgmodel +# cutgeo = _cutgeo +# aggregates = _aggregates -gids = get_cell_gids(bgmodel) -# Local aggregates -laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l - map(agg) do i - iszero(i) ? i : g_to_l[i] - end -end +# gids = get_cell_gids(bgmodel) +# # Local aggregates +# laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l +# map(agg) do i +# iszero(i) ? i : g_to_l[i] +# end +# end Ω_bg = Triangulation(bgmodel) @@ -123,7 +139,7 @@ dΓ = Measure(Γ,degree) reffe = ReferenceFE(lagrangian,Float64,order) Vstd = FESpace(Ω_act,reffe) -V = AgFEMSpace(bgmodel,Vstd,laggregates) +V = AgFEMSpace(bgmodel,Vstd,aggregates) U = TrialFESpace(V) @@ -173,26 +189,27 @@ writevtk(Ω,"trian", -# Debug VTK -map(local_views(_bgmodel),ranks) do m,p - writevtk(get_grid(m),"bgmodel_$p") -end +# # Debug VTK +# map(local_views(_bgmodel),ranks) do m,p +# writevtk(get_grid(m),"bgmodel_$p") +# end -map(local_views(uh),local_views(bgmodel),ranks) do uh,m,p - trian = Triangulation(m) - writevtk(trian,"ltrian_$p",cellfields=["uh"=>uh]) -end +# map(local_views(uh),local_views(bgmodel),ranks) do uh,m,p +# trian = Triangulation(m) +# writevtk(trian,"ltrian_$p",cellfields=["uh"=>uh]) +# end -map(local_views(cutgeo),ranks) do c,p - m = get_background_model(c) - c_io = compute_bgcell_to_inoutcut(c,c.geo) - writevtk(Triangulation(m),"bgtrian_$p",celldata=["inout"=>c_io]) -end +# map(local_views(cutgeo),ranks) do c,p +# m = get_background_model(c) +# c_io = compute_bgcell_to_inoutcut(c,c.geo) +# writevtk(Triangulation(m),"bgtrian_$p",celldata=["inout"=>c_io]) +# end # TODO: encapsulate and reduce Interfaces +# split files # add tests with mpi # PR end # module From d41de4663b903185d9bee0fc8f80452c3cf2e1f0 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Thu, 11 Apr 2024 17:43:49 +0200 Subject: [PATCH 18/31] [distributed] structured src --- src/Distributed/Distributed.jl | 708 +----------------- src/Distributed/DistributedAgFEM.jl | 467 ++++++++++++ src/Distributed/DistributedDiscretizations.jl | 222 ++++++ test/DistributedTests/testing_remote_aggs.jl | 54 +- 4 files changed, 693 insertions(+), 758 deletions(-) create mode 100644 src/Distributed/DistributedAgFEM.jl create mode 100644 src/Distributed/DistributedDiscretizations.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 2121938..6c3a7ef 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -41,712 +41,8 @@ import Gridap.Geometry: get_background_model import GridapDistributed: local_views import GridapDistributed: remove_ghost_cells +include("DistributedDiscretizations.jl") -struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: GridapType - discretizations::A - model::B - function DistributedEmbeddedDiscretization( - d::AbstractArray{<:EmbeddedDiscretization{Dp,T}}, - model::DistributedDiscreteModel) where {Dp,T} - A = typeof(d) - B = typeof(model) - new{Dp,T,A,B}(d,model) - end -end - -local_views(a::DistributedEmbeddedDiscretization) = a.discretizations - -get_background_model(a::DistributedEmbeddedDiscretization) = a.model - -function cut(bgmodel::DistributedDiscreteModel,args...) - cut(LevelSetCutter(),bgmodel,args...) -end - -function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) - gids = get_cell_gids(bgmodel) - cuts = map(local_views(bgmodel),local_views(gids)) do bgmodel,gids - ownmodel = remove_ghost_cells(bgmodel,gids) - cutgeo = cut(cutter,ownmodel,args...) - change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) - end - ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) - _consistent!(ls_to_bgcell_to_inoutcut,gids) - DistributedEmbeddedDiscretization(cuts,bgmodel) -end - -function Triangulation( - cutgeo::DistributedEmbeddedDiscretization, - in_or_out::ActiveInOrOut, - args...) - - distributed_embedded_triangulation(Triangulation,cutgeo,in_or_out,args...) -end - -function Triangulation(cutgeo::DistributedEmbeddedDiscretization,args...) - trian = distributed_embedded_triangulation(Triangulation,cutgeo,args...) - remove_ghost_cells(trian) -end - -function EmbeddedBoundary(cutgeo::DistributedEmbeddedDiscretization,args...) - trian = distributed_embedded_triangulation(EmbeddedBoundary,cutgeo,args...) - remove_ghost_cells(trian) -end - -function distributed_embedded_triangulation( - T, - cutgeo::DistributedEmbeddedDiscretization, - args...) - - trians = map(local_views(cutgeo)) do lcutgeo - T(lcutgeo,args...) - end - bgmodel = get_background_model(cutgeo) - DistributedTriangulation(trians,bgmodel) -end - - -function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) - aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,args...) - bgmodel = get_background_model(cutgeo) - if has_remote_aggregation(bgmodel,aggregates) - bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) - cutgeo = change_bgmodel(cutgeo,bgmodel) - aggregates = _change_model(aggregates,get_cell_gids(bgmodel)) - end - laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) - bgmodel,cutgeo,laggregates -end - -# function consistent_aggregates(aggregates,bgmodel::DistributedDiscreteModel) -# consistent_aggregates(aggregates,get_cell_gids(bgmodel)) -# end - -# function consistent_aggregates(aggregates,gids::PRange) -# global_aggregates = map(aggregates,local_to_global(gids)) do agg,gid -# map(i-> iszero(i) ? i : gid[i],agg) -# end -# paggregates = PVector(global_aggregates,partition(gids)) -# consistent!(paggregates) |> wait -# map(local_values(paggregates),global_to_local(gids)) do agg,lgid -# map(i-> iszero(i) ? i : lgid[i],agg) -# end -# end - -function AgFEMSpace( - bgmodel::DistributedDiscreteModel, - f::DistributedFESpace, - bgcell_to_bgcellin::AbstractArray{<:AbstractVector}, - g::DistributedFESpace=f) - - bgmodel_gids = get_cell_gids(bgmodel) - spaces = map( - local_views(f), - bgcell_to_bgcellin, - local_views(g), - local_views(bgmodel_gids)) do f,bgcell_to_bgcellin,g,gids - AgFEMSpace(f,bgcell_to_bgcellin,g,local_to_global(gids)) - end - trians = map(get_triangulation,local_views(f)) - trian = DistributedTriangulation(trians,bgmodel) - trian = add_ghost_cells(trian) - trian_gids = generate_cell_gids(trian) - cell_to_cellin = _active_aggregates(bgcell_to_bgcellin) - cell_to_ldofs = map(get_cell_dof_ids,spaces) - cell_to_ldofs = map(i->map(sort,i),cell_to_ldofs) - _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) - nldofs = map(num_free_dofs,spaces) - gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) - vector_type = _find_vector_type(spaces,gids) - DistributedSingleFieldFESpace(spaces,gids,vector_type) -end - -function remove_ghost_cells(trian::DistributedTriangulation) - model = get_background_model(trian) - gids = get_cell_gids(model) - trians = map(local_views(trian),local_views(gids)) do trian,gids - remove_ghost_cells(trian,gids) - end - DistributedTriangulation(trians,model) -end - -function remove_ghost_cells(trian::AppendedTriangulation,gids) - a = remove_ghost_cells(trian.a,gids) - b = remove_ghost_cells(trian.b,gids) - lazy_append(a,b) -end - -function remove_ghost_cells(trian::SubFacetTriangulation,gids) - model = get_background_model(trian) - D = num_cell_dims(model) - glue = get_glue(trian,Val{D}()) - remove_ghost_cells(glue,trian,gids) -end - -function remove_ghost_cells(model::DiscreteModel,gids::AbstractLocalIndices) - DiscreteModelPortion(model,own_to_local(gids)) -end - -function _consistent!( - p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, - prange::PRange) - - n = map(length,p_to_i_to_a) |> PartitionedArrays.getany - for i in 1:n - p_to_a = map(i_to_a->i_to_a[i],p_to_i_to_a) - PVector(p_to_a,partition(prange)) |> consistent! |> wait - map(p_to_a,p_to_i_to_a) do p_to_a,p_to_ia - copyto!(p_to_ia[i],p_to_a) - end - end -end - - -function compute_bgfacet_to_inoutcut( - bgmodel::DistributedDiscreteModel, - bgf_to_ioc::AbstractArray{<:AbstractVector}) - - D = num_dims(eltype(local_views(bgmodel))) - gids = get_cell_gids(bgmodel) - bgf_to_ioc = map( - local_views(bgmodel), - local_views(gids), - bgf_to_ioc) do bgmodel,gids,bgf_to_ioc - - ownmodel = remove_ghost_cells(bgmodel,gids) - f_to_pf = Gridap.Geometry.get_face_to_parent_face(ownmodel,D-1) - _bgf_to_ioc = Vector{eltype(bgf_to_ioc)}(undef,num_faces(bgmodel,D-1)) - _bgf_to_ioc[f_to_pf] .= bgf_to_ioc - _bgf_to_ioc - end - facet_gids = get_face_gids(bgmodel,D-1) - pbgf_to_ioc = PVector(bgf_to_ioc,partition(facet_gids)) - consistent!(pbgf_to_ioc) |> wait - local_values(pbgf_to_ioc) -end - - -function compute_bgfacet_to_inoutcut( - cutter::Cutter, - bgmodel::DistributedDiscreteModel, - geo) - - gids = get_cell_gids(bgmodel) - bgf_to_ioc = map(local_views(bgmodel),local_views(gids)) do model,gids - ownmodel = remove_ghost_cells(model,gids) - compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) - end - compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc) -end - - -function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) - cutter = LevelSetCutter() - compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) -end - -function compute_bgcell_to_inoutcut(cutgeo::DistributedEmbeddedDiscretization,args...) - map(local_views(cutgeo)) do cutgeo - compute_bgcell_to_inoutcut(cutgeo,args...) - end -end - -function change_bgmodel( - cutgeo::DistributedEmbeddedDiscretization, - model::DistributedDiscreteModel, - args...) - - cuts = _change_bgmodels(cutgeo,model,args...) - gids = get_cell_gids(model) - ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) - _consistent!(ls_to_bgcell_to_inoutcut,gids) - DistributedEmbeddedDiscretization(cuts,model) -end - -function _change_bgmodels( - cutgeo::DistributedEmbeddedDiscretization, - model::DistributedDiscreteModel, - cell_to_newcell) - - map(local_views(cutgeo),local_views(model),cell_to_newcell) do c,m,c_to_nc - change_bgmodel(c,m,c_to_nc) - end -end - -function _change_bgmodels( - cutgeo::DistributedEmbeddedDiscretization, - model::DistributedDiscreteModel) - - map(local_views(cutgeo),local_views(model)) do c,m - change_bgmodel(c,m) - end -end - -function change_bgmodel( - cut::EmbeddedDiscretization, - newmodel::DiscreteModel, - cell_to_newcell=1:num_cells(get_background_model(cut))) - - ls_to_bgc_to_ioc = map(cut.ls_to_bgcell_to_inoutcut) do bgc_to_ioc - new_bgc_to_ioc = Vector{Int8}(undef,num_cells(newmodel)) - new_bgc_to_ioc[cell_to_newcell] = bgc_to_ioc - new_bgc_to_ioc - end - subcells = change_bgmodel(cut.subcells,cell_to_newcell) - subfacets = change_bgmodel(cut.subfacets,cell_to_newcell) - EmbeddedDiscretization( - newmodel, - ls_to_bgc_to_ioc, - subcells, - cut.ls_to_subcell_to_inout, - subfacets, - cut.ls_to_subfacet_to_inout, - cut.oid_to_ls, - cut.geo) -end - -function change_bgmodel(cells::SubCellData,cell_to_newcell) - cell_to_bgcell = lazy_map(Reindex(cell_to_newcell),cells.cell_to_bgcell) - SubCellData( - cells.cell_to_points, - collect(Int32,cell_to_bgcell), - cells.point_to_coords, - cells.point_to_rcoords) -end - -function change_bgmodel(facets::SubFacetData,cell_to_newcell) - facet_to_bgcell = lazy_map(Reindex(cell_to_newcell),facets.facet_to_bgcell) - SubFacetData( - facets.facet_to_points, - facets.facet_to_normal, - collect(Int32,facet_to_bgcell), - facets.point_to_coords, - facets.point_to_rcoords) -end - -function distributed_aggregate( - strategy::AggregateCutCellsByThreshold, - cut::DistributedEmbeddedDiscretization, - geo::CSG.Geometry, - in_or_out=IN) - - bgmodel = get_background_model(cut) - facet_to_inoutcut = compute_bgfacet_to_inoutcut(bgmodel,geo) - _distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut) -end - - -function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_inoutcut) - @assert loc in (IN,OUT) - - cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) - trian = Triangulation(cutgeo,cutinorout,geo) - model = get_background_model(cutgeo) - bgtrian = get_triangulation(model) - cell_to_cut_meas = map(_get_cell_measure,local_views(trian),local_views(bgtrian)) - cell_to_meas = map(get_cell_measure,local_views(bgtrian)) - cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m - lazy_map(/,c_to_cm,c_to_m) - end - - cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) - - cell_to_coords = map(get_cell_coordinates,local_views(model)) - topo = get_grid_topology(model) - D = num_cell_dims(model) - cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) - face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) - gids = get_cell_gids(model) - - _distributed_aggregate_by_threshold_barrier( - threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, - loc,cell_to_coords,cell_to_faces,face_to_cells,gids) -end - - -function _distributed_aggregate_by_threshold_barrier( - threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, - loc,cell_to_coords,cell_to_faces,face_to_cells,gids) - - - ocell_to_touched = map(cell_to_unit_cut_meas) do c_to_m - map(≥,c_to_m,Fill(threshold,length(c_to_m))) - end - cell_to_touched = _add_ghost_values(ocell_to_touched,gids) - - cell_to_root_centroid = map(cell_to_coords) do cell_to_coords - map(i->sum(i)/length(i),cell_to_coords) - end - PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait - - n_cells = map(length,cell_to_touched) - touched_cells = map(findall,cell_to_touched) - - cell_to_cellin = map(n->zeros(Int32,n),n_cells) - map(cell_to_cellin,touched_cells,local_to_global(gids)) do c_to_ci,cells,l_to_g - gcells = lazy_map(Reindex(l_to_g),cells) - c_to_ci[cells] = gcells - end - - cell_to_neig = map(n->zeros(Int32,n),n_cells) - - cell_to_root_part = map(collect,local_to_owner(gids)) - - c1 = map(array_cache,cell_to_faces) - c2 = map(array_cache,face_to_cells) - - max_iters = 20 - for iter in 1:max_iters - all_aggregated = _aggregate_one_step!(c1,c2,gids, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - PVector(cell_to_touched,partition(gids)) |> consistent! |> wait - PVector(cell_to_neig,partition(gids)) |> consistent! |> wait - PVector(cell_to_cellin,partition(gids)) |> consistent! |> wait - PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait - PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait - - reduction!(&,all_aggregated,all_aggregated,destination=:all) - - if PartitionedArrays.getany(all_aggregated) - break - end - end - - cell_to_cellin, cell_to_root_part, cell_to_neig -end - -function _aggregate_one_step!(c1,c2,gids::PRange, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - map(c1,c2,own_to_local(gids), - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - local_to_global(gids), - cell_to_faces, - face_to_cells, - facet_to_inoutcut) do c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut - - _aggregate_one_step!( - c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - end -end - -function _aggregate_one_step!( - c1,c2,own_cells, - cell_to_inoutcut, - cell_to_touched, - cell_to_neig, - cell_to_cellin, - cell_to_root_centroid, - cell_to_root_part, - cell_to_gcell, - cell_to_faces, - face_to_cells, - facet_to_inoutcut, - loc) - - all_aggregated = true - for cell in own_cells - if ! cell_to_touched[cell] && cell_to_inoutcut[cell] == CUT - neigh_cell = _find_best_neighbor_from_centroid_distance( - c1,c2,cell, - cell_to_faces, - face_to_cells, - cell_to_touched, - cell_to_root_centroid, - facet_to_inoutcut, - loc) - if neigh_cell > 0 - cellin = cell_to_cellin[neigh_cell] - centroid = cell_to_root_centroid[neigh_cell] - part = cell_to_root_part[neigh_cell] - neigh_gcell = cell_to_gcell[neigh_cell] - - cell_to_neig[cell] = neigh_gcell - cell_to_cellin[cell] = cellin - cell_to_root_centroid[cell] = centroid - cell_to_root_part[cell] = part - else - all_aggregated = false - end - end - end - _touch_aggregated_cells!(cell_to_touched,cell_to_cellin) - all_aggregated -end - -function _find_best_neighbor_from_centroid_distance( - c1,c2,cell, - cell_to_faces, - face_to_cells, - cell_to_touched, - cell_to_root_centroid, - facet_to_inoutcut, - loc) - - faces = getindex!(c1,cell_to_faces,cell) - dmin = Inf - T = eltype(eltype(face_to_cells)) - best_neigh_cell = zero(T) - for face in faces - inoutcut = facet_to_inoutcut[face] - if inoutcut != CUT && inoutcut != loc - continue - end - neigh_cells = getindex!(c2,face_to_cells,face) - for neigh_cell in neigh_cells - if neigh_cell != cell && cell_to_touched[neigh_cell] - p = cell_to_root_centroid[neigh_cell] - q = cell_to_root_centroid[cell] - d = norm(p-q) - if (1.0+1.0e-9)*d < dmin - dmin = d - best_neigh_cell = neigh_cell - end - end - end - end - best_neigh_cell -end - -function _add_ghost_values(own_v,gids::PRange) - lens = map(length,local_views(gids)) - eltypes = map(eltype,own_v) - local_v = map(zeros,eltypes,lens) - map(local_v,own_v,own_to_local(gids)) do l,o,o_to_l - l[o_to_l] = o - end - PVector(local_v,partition(gids)) |> consistent! |> wait - local_v -end - -function _get_cell_measure(trian1::Triangulation,trian2::Triangulation) - if num_cells(trian1) == 0 - Fill(0.0,num_cells(trian2)) - else - get_cell_measure(trian1,trian2) - end -end - -function add_remote_aggregates(model::DistributedDiscreteModel,aggregates,aggregate_owner) - gids = get_cell_gids(model) - remote_cells,remote_parts = _extract_remote_cells(gids,aggregates,aggregate_owner) - remote_cells,remote_parts = _group_remote_ids(remote_cells,remote_parts) - add_remote_cells(model,remote_cells,remote_parts) -end - -function _extract_remote_cells(gids::PRange,aggregates,aggregate_owner) - remote_aggids = map(aggregates,global_to_local(gids)) do agg,g_to_l - ids = findall(agg) do i - !iszero(i) && iszero(g_to_l[i]) - end - unique(Reindex(agg),ids) - end - - remote_cells = map(aggregates,remote_aggids) do agg,ids - map(Reindex(agg),ids) - end - - remote_parts = map(aggregate_owner,remote_aggids) do agg,ids - map(Reindex(agg),ids) - end - - remote_cells,remote_parts -end - -function _group_remote_ids(remote_ids,remote_parts) - new_parts = map(sort∘unique,remote_parts) - new_ids = map(remote_ids,remote_parts,new_parts) do ids,parts,uparts - grouped_ids = map(i->Int[],1:length(uparts)) - for (id,p) in zip(ids,parts) - j = findfirst(==(p),uparts) - union!(grouped_ids[j],id) - end - map!(sort!,grouped_ids,grouped_ids) - end - new_ids,new_parts -end - -function _ungroup_remote_ids(remote_ids,remote_parts) - new_ids = map(remote_ids) do ids - reduce(append!,ids,init=eltype(eltype(ids))[]) - end - new_parts = map(remote_ids,remote_parts) do ids,parts - n = map(length,ids) - parts_v = map((p,n)->Fill(p,n),parts,n) - reduce(append!,parts_v,init=eltype(parts)[]) - end - new_ids,new_parts -end - -function add_remote_ids(gids::PRange,remote_gids,remote_parts) - new_gids,new_parts = _ungroup_remote_ids(remote_gids,remote_parts) - lid_to_gid = map(vcat,local_to_global(gids),new_gids) - lid_to_part = map(vcat,local_to_owner(gids),new_parts) - p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p - LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) - end - PRange(p) -end - -function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_parts) - # Send remote gids to owners - snd_ids = remote_parts - snd_remotes = remote_cells - graph = ExchangeGraph(snd_ids) - rcv_remotes = allocate_exchange(snd_remotes,graph) - exchange!(rcv_remotes,snd_remotes,graph) |> wait - - # Send remote coordinates - gids = get_cell_gids(model) - snd_gids = rcv_remotes - snd_lids = map(global_to_local(gids),snd_gids) do g_to_l,gids - map(Reindex(g_to_l),gids) - end - snd_coords = map(local_views(model),snd_lids) do m,lids - T = eltype(eltype(get_cell_coordinates(m))) - coords = map(lids) do lids - coords = map(Reindex(get_cell_coordinates(m)),lids) - reduce(append!,coords,init=T[]) - end - Vector{Vector{T}}(coords) - end - rgraph = reverse(graph) - rcv_coords = allocate_exchange(snd_coords,rgraph) - exchange!(rcv_coords,snd_coords,rgraph) |> wait - - # Build remote grids - ncells = map(remote_cells) do cells - sum(length,cells,init=0) - end - reffes = map(get_reffes,local_views(model)) - reffe = map(only,reffes) - ctypes = map(n->ones(Int,n),ncells) - coords = map(PartitionedArrays.getdata,rcv_coords) - conn = map(ncells,reffe) do ncells,reffe - n = num_nodes(reffe) - data = 1:n*ncells - ptrs = 1:n:n*ncells+1 - Table(data,ptrs) - end - rgrids = map(UnstructuredGrid,coords,conn,reffes,ctypes) - - # Build appended model - lgrids = map(get_grid,local_views(model)) - grids = map(lazy_append,lgrids,rgrids) - models = map(UnstructuredDiscreteModel,grids) - agids = add_remote_ids(gids,remote_cells,remote_parts) - DistributedDiscreteModel(models,agids) -end - - -function has_remote_aggregation(model::DistributedDiscreteModel,aggregates) - gids = get_cell_gids(model) - has_remote_aggregation(aggregates,gids) -end - -function has_remote_aggregation(aggregates,gids::PRange) - remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l - lazy_map(agg) do a - iszero(a) || !iszero(g_to_l[a]) - end |> all |> ! - end - reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany -end - - -function _active_aggregates(bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) - map(_active_aggregates,bgcell_to_bgcellin) -end - -function _active_aggregates(bgcell_to_bgcellin) - acell_to_bgcell = findall(!iszero,bgcell_to_bgcellin) - bgcell_to_acell = zeros(Int,length(bgcell_to_bgcellin)) - bgcell_to_acell[acell_to_bgcell] = 1:length(acell_to_bgcell) - acell_to_bgcellin = bgcell_to_bgcellin[ acell_to_bgcell ] - bgcell_to_acell[ acell_to_bgcellin ] -end - -function _remove_improper_cell_ldofs!( - cell_to_ldofs::AbstractVector{<:AbstractVector{<:AbstractVector}}, - bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) - - map(_remove_improper_cell_ldofs!,cell_to_ldofs,bgcell_to_bgcellin) -end - - -function _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) - for cell in 1:length(cell_to_ldofs) - cell_to_cellin[cell] != cell || continue - cell_to_ldofs[cell] = empty!(cell_to_ldofs[cell]) - end - cell_to_ldofs -end - -function _local_aggregates(cell_to_gcellin,gids::PRange) - map(_local_aggregates,cell_to_gcellin,global_to_local(gids)) -end - -function _local_aggregates(cell_to_gcellin,gcell_to_cell) - map(cell_to_gcellin) do gcin - iszero(gcin) ? gcin : gcell_to_cell[ gcin ] - end -end - -function _change_model(cell_to_gcellin,gids::PRange) - map(_change_model,cell_to_gcellin,local_to_global(gids)) -end - -function _change_model(cell_to_gcellin,ncell_to_gcell) - ncells = length(cell_to_gcellin) - ncell_to_gcellin = zeros(Int,length(ncell_to_gcell)) - for (ncell,gcell) in enumerate(ncell_to_gcell) - ncell_to_gcellin[ncell] = ncell > ncells ? gcell : cell_to_gcellin[ncell] - end - ncell_to_gcellin -end +include("DistributedAgFEM.jl") end # module diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl new file mode 100644 index 0000000..ab41727 --- /dev/null +++ b/src/Distributed/DistributedAgFEM.jl @@ -0,0 +1,467 @@ + +function AgFEMSpace( + bgmodel::DistributedDiscreteModel, + f::DistributedFESpace, + bgcell_to_bgcellin::AbstractArray{<:AbstractVector}, + g::DistributedFESpace=f) + + bgmodel_gids = get_cell_gids(bgmodel) + spaces = map( + local_views(f), + bgcell_to_bgcellin, + local_views(g), + local_views(bgmodel_gids)) do f,bgcell_to_bgcellin,g,gids + AgFEMSpace(f,bgcell_to_bgcellin,g,local_to_global(gids)) + end + trians = map(get_triangulation,local_views(f)) + trian = DistributedTriangulation(trians,bgmodel) + trian = add_ghost_cells(trian) + trian_gids = generate_cell_gids(trian) + cell_to_cellin = _active_aggregates(bgcell_to_bgcellin) + cell_to_ldofs = map(get_cell_dof_ids,spaces) + cell_to_ldofs = map(i->map(sort,i),cell_to_ldofs) + _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) + nldofs = map(num_free_dofs,spaces) + gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) + vector_type = _find_vector_type(spaces,gids) + DistributedSingleFieldFESpace(spaces,gids,vector_type) +end + +function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) + aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,args...) + bgmodel = get_background_model(cutgeo) + if has_remote_aggregation(bgmodel,aggregates) + bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) + cutgeo = change_bgmodel(cutgeo,bgmodel) + aggregates = _change_model(aggregates,get_cell_gids(bgmodel)) + end + laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) + bgmodel,cutgeo,laggregates +end + + +function distributed_aggregate( + strategy::AggregateCutCellsByThreshold, + cut::DistributedEmbeddedDiscretization, + geo::CSG.Geometry, + in_or_out=IN) + + bgmodel = get_background_model(cut) + facet_to_inoutcut = compute_bgfacet_to_inoutcut(bgmodel,geo) + _distributed_aggregate_by_threshold(strategy.threshold,cut,geo,in_or_out,facet_to_inoutcut) +end + + +function _distributed_aggregate_by_threshold(threshold,cutgeo,geo,loc,facet_to_inoutcut) + @assert loc in (IN,OUT) + + cutinorout = loc == IN ? (CUT_IN,IN) : (CUT_OUT,OUT) + trian = Triangulation(cutgeo,cutinorout,geo) + model = get_background_model(cutgeo) + bgtrian = get_triangulation(model) + cell_to_cut_meas = map(_get_cell_measure,local_views(trian),local_views(bgtrian)) + cell_to_meas = map(get_cell_measure,local_views(bgtrian)) + cell_to_unit_cut_meas = map(cell_to_cut_meas,cell_to_meas) do c_to_cm,c_to_m + lazy_map(/,c_to_cm,c_to_m) + end + + cell_to_inoutcut = compute_bgcell_to_inoutcut(cutgeo,geo) + + cell_to_coords = map(get_cell_coordinates,local_views(model)) + topo = get_grid_topology(model) + D = num_cell_dims(model) + cell_to_faces = map(t->get_faces(t,D,D-1),local_views(topo)) + face_to_cells = map(t->get_faces(t,D-1,D),local_views(topo)) + gids = get_cell_gids(model) + + _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) +end + + +function _distributed_aggregate_by_threshold_barrier( + threshold,cell_to_unit_cut_meas,facet_to_inoutcut,cell_to_inoutcut, + loc,cell_to_coords,cell_to_faces,face_to_cells,gids) + + + ocell_to_touched = map(cell_to_unit_cut_meas) do c_to_m + map(≥,c_to_m,Fill(threshold,length(c_to_m))) + end + cell_to_touched = _add_ghost_values(ocell_to_touched,gids) + + cell_to_root_centroid = map(cell_to_coords) do cell_to_coords + map(i->sum(i)/length(i),cell_to_coords) + end + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + + n_cells = map(length,cell_to_touched) + touched_cells = map(findall,cell_to_touched) + + cell_to_cellin = map(n->zeros(Int32,n),n_cells) + map(cell_to_cellin,touched_cells,local_to_global(gids)) do c_to_ci,cells,l_to_g + gcells = lazy_map(Reindex(l_to_g),cells) + c_to_ci[cells] = gcells + end + + cell_to_neig = map(n->zeros(Int32,n),n_cells) + + cell_to_root_part = map(collect,local_to_owner(gids)) + + c1 = map(array_cache,cell_to_faces) + c2 = map(array_cache,face_to_cells) + + max_iters = 20 + for iter in 1:max_iters + all_aggregated = _aggregate_one_step!(c1,c2,gids, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + PVector(cell_to_touched,partition(gids)) |> consistent! |> wait + PVector(cell_to_neig,partition(gids)) |> consistent! |> wait + PVector(cell_to_cellin,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_centroid,partition(gids)) |> consistent! |> wait + PVector(cell_to_root_part,partition(gids)) |> consistent! |> wait + + reduction!(&,all_aggregated,all_aggregated,destination=:all) + + if PartitionedArrays.getany(all_aggregated) + break + end + end + + cell_to_cellin, cell_to_root_part, cell_to_neig +end + +function _aggregate_one_step!(c1,c2,gids::PRange, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + map(c1,c2,own_to_local(gids), + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + local_to_global(gids), + cell_to_faces, + face_to_cells, + facet_to_inoutcut) do c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut + + _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + end +end + +function _aggregate_one_step!( + c1,c2,own_cells, + cell_to_inoutcut, + cell_to_touched, + cell_to_neig, + cell_to_cellin, + cell_to_root_centroid, + cell_to_root_part, + cell_to_gcell, + cell_to_faces, + face_to_cells, + facet_to_inoutcut, + loc) + + all_aggregated = true + for cell in own_cells + if ! cell_to_touched[cell] && cell_to_inoutcut[cell] == CUT + neigh_cell = _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + if neigh_cell > 0 + cellin = cell_to_cellin[neigh_cell] + centroid = cell_to_root_centroid[neigh_cell] + part = cell_to_root_part[neigh_cell] + neigh_gcell = cell_to_gcell[neigh_cell] + + cell_to_neig[cell] = neigh_gcell + cell_to_cellin[cell] = cellin + cell_to_root_centroid[cell] = centroid + cell_to_root_part[cell] = part + else + all_aggregated = false + end + end + end + _touch_aggregated_cells!(cell_to_touched,cell_to_cellin) + all_aggregated +end + +function _find_best_neighbor_from_centroid_distance( + c1,c2,cell, + cell_to_faces, + face_to_cells, + cell_to_touched, + cell_to_root_centroid, + facet_to_inoutcut, + loc) + + faces = getindex!(c1,cell_to_faces,cell) + dmin = Inf + T = eltype(eltype(face_to_cells)) + best_neigh_cell = zero(T) + for face in faces + inoutcut = facet_to_inoutcut[face] + if inoutcut != CUT && inoutcut != loc + continue + end + neigh_cells = getindex!(c2,face_to_cells,face) + for neigh_cell in neigh_cells + if neigh_cell != cell && cell_to_touched[neigh_cell] + p = cell_to_root_centroid[neigh_cell] + q = cell_to_root_centroid[cell] + d = norm(p-q) + if (1.0+1.0e-9)*d < dmin + dmin = d + best_neigh_cell = neigh_cell + end + end + end + end + best_neigh_cell +end + +function _add_ghost_values(own_v,gids::PRange) + lens = map(length,local_views(gids)) + eltypes = map(eltype,own_v) + local_v = map(zeros,eltypes,lens) + map(local_v,own_v,own_to_local(gids)) do l,o,o_to_l + l[o_to_l] = o + end + PVector(local_v,partition(gids)) |> consistent! |> wait + local_v +end + +function _get_cell_measure(trian1::Triangulation,trian2::Triangulation) + if num_cells(trian1) == 0 + Fill(0.0,num_cells(trian2)) + else + get_cell_measure(trian1,trian2) + end +end + +function add_remote_cells(model::DistributedDiscreteModel,remote_cells,remote_parts) + # Send remote gids to owners + snd_ids = remote_parts + snd_remotes = remote_cells + graph = ExchangeGraph(snd_ids) + rcv_remotes = allocate_exchange(snd_remotes,graph) + exchange!(rcv_remotes,snd_remotes,graph) |> wait + + # Send remote coordinates + gids = get_cell_gids(model) + snd_gids = rcv_remotes + snd_lids = map(global_to_local(gids),snd_gids) do g_to_l,gids + map(Reindex(g_to_l),gids) + end + snd_coords = map(local_views(model),snd_lids) do m,lids + T = eltype(eltype(get_cell_coordinates(m))) + coords = map(lids) do lids + coords = map(Reindex(get_cell_coordinates(m)),lids) + reduce(append!,coords,init=T[]) + end + Vector{Vector{T}}(coords) + end + rgraph = reverse(graph) + rcv_coords = allocate_exchange(snd_coords,rgraph) + exchange!(rcv_coords,snd_coords,rgraph) |> wait + + # Build remote grids + ncells = map(remote_cells) do cells + sum(length,cells,init=0) + end + reffes = map(get_reffes,local_views(model)) + reffe = map(only,reffes) + ctypes = map(n->ones(Int,n),ncells) + coords = map(PartitionedArrays.getdata,rcv_coords) + conn = map(ncells,reffe) do ncells,reffe + n = num_nodes(reffe) + data = 1:n*ncells + ptrs = 1:n:n*ncells+1 + Table(data,ptrs) + end + rgrids = map(UnstructuredGrid,coords,conn,reffes,ctypes) + + # Build appended model + lgrids = map(get_grid,local_views(model)) + grids = map(lazy_append,lgrids,rgrids) + models = map(UnstructuredDiscreteModel,grids) + agids = add_remote_ids(gids,remote_cells,remote_parts) + DistributedDiscreteModel(models,agids) +end + +function add_remote_aggregates(model::DistributedDiscreteModel,aggregates,aggregate_owner) + gids = get_cell_gids(model) + remote_cells,remote_parts = _extract_remote_cells(gids,aggregates,aggregate_owner) + remote_cells,remote_parts = _group_remote_ids(remote_cells,remote_parts) + add_remote_cells(model,remote_cells,remote_parts) +end + +function _extract_remote_cells(gids::PRange,aggregates,aggregate_owner) + remote_aggids = map(aggregates,global_to_local(gids)) do agg,g_to_l + ids = findall(agg) do i + !iszero(i) && iszero(g_to_l[i]) + end + unique(Reindex(agg),ids) + end + + remote_cells = map(aggregates,remote_aggids) do agg,ids + map(Reindex(agg),ids) + end + + remote_parts = map(aggregate_owner,remote_aggids) do agg,ids + map(Reindex(agg),ids) + end + + remote_cells,remote_parts +end + +function _group_remote_ids(remote_ids,remote_parts) + new_parts = map(sort∘unique,remote_parts) + new_ids = map(remote_ids,remote_parts,new_parts) do ids,parts,uparts + grouped_ids = map(i->Int[],1:length(uparts)) + for (id,p) in zip(ids,parts) + j = findfirst(==(p),uparts) + union!(grouped_ids[j],id) + end + map!(sort!,grouped_ids,grouped_ids) + end + new_ids,new_parts +end + +function _ungroup_remote_ids(remote_ids,remote_parts) + new_ids = map(remote_ids) do ids + reduce(append!,ids,init=eltype(eltype(ids))[]) + end + new_parts = map(remote_ids,remote_parts) do ids,parts + n = map(length,ids) + parts_v = map((p,n)->Fill(p,n),parts,n) + reduce(append!,parts_v,init=eltype(parts)[]) + end + new_ids,new_parts +end + +function add_remote_ids(gids::PRange,remote_gids,remote_parts) + new_gids,new_parts = _ungroup_remote_ids(remote_gids,remote_parts) + lid_to_gid = map(vcat,local_to_global(gids),new_gids) + lid_to_part = map(vcat,local_to_owner(gids),new_parts) + p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p + LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) + end + PRange(p) +end + + +function has_remote_aggregation(model::DistributedDiscreteModel,aggregates) + gids = get_cell_gids(model) + has_remote_aggregation(aggregates,gids) +end + +function has_remote_aggregation(aggregates,gids::PRange) + remote_aggregation = map(aggregates,global_to_local(gids)) do agg,g_to_l + lazy_map(agg) do a + iszero(a) || !iszero(g_to_l[a]) + end |> all |> ! + end + reduction(|,remote_aggregation,destination=:all) |> PartitionedArrays.getany +end + + +function _active_aggregates(bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) + map(_active_aggregates,bgcell_to_bgcellin) +end + +function _active_aggregates(bgcell_to_bgcellin) + acell_to_bgcell = findall(!iszero,bgcell_to_bgcellin) + bgcell_to_acell = zeros(Int,length(bgcell_to_bgcellin)) + bgcell_to_acell[acell_to_bgcell] = 1:length(acell_to_bgcell) + acell_to_bgcellin = bgcell_to_bgcellin[ acell_to_bgcell ] + bgcell_to_acell[ acell_to_bgcellin ] +end + +function _remove_improper_cell_ldofs!( + cell_to_ldofs::AbstractVector{<:AbstractVector{<:AbstractVector}}, + bgcell_to_bgcellin::AbstractVector{<:AbstractVector}) + + map(_remove_improper_cell_ldofs!,cell_to_ldofs,bgcell_to_bgcellin) +end + + +function _remove_improper_cell_ldofs!(cell_to_ldofs,cell_to_cellin) + for cell in 1:length(cell_to_ldofs) + cell_to_cellin[cell] != cell || continue + cell_to_ldofs[cell] = empty!(cell_to_ldofs[cell]) + end + cell_to_ldofs +end + +function _local_aggregates(cell_to_gcellin,gids::PRange) + map(_local_aggregates,cell_to_gcellin,global_to_local(gids)) +end + +function _local_aggregates(cell_to_gcellin,gcell_to_cell) + map(cell_to_gcellin) do gcin + iszero(gcin) ? gcin : gcell_to_cell[ gcin ] + end +end + +function _change_model(cell_to_gcellin,gids::PRange) + map(_change_model,cell_to_gcellin,local_to_global(gids)) +end + +function _change_model(cell_to_gcellin,ncell_to_gcell) + ncells = length(cell_to_gcellin) + ncell_to_gcellin = zeros(Int,length(ncell_to_gcell)) + for (ncell,gcell) in enumerate(ncell_to_gcell) + ncell_to_gcellin[ncell] = ncell > ncells ? gcell : cell_to_gcellin[ncell] + end + ncell_to_gcellin +end diff --git a/src/Distributed/DistributedDiscretizations.jl b/src/Distributed/DistributedDiscretizations.jl new file mode 100644 index 0000000..2ceef14 --- /dev/null +++ b/src/Distributed/DistributedDiscretizations.jl @@ -0,0 +1,222 @@ + +struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: GridapType + discretizations::A + model::B + function DistributedEmbeddedDiscretization( + d::AbstractArray{<:EmbeddedDiscretization{Dp,T}}, + model::DistributedDiscreteModel) where {Dp,T} + A = typeof(d) + B = typeof(model) + new{Dp,T,A,B}(d,model) + end +end + +local_views(a::DistributedEmbeddedDiscretization) = a.discretizations + +get_background_model(a::DistributedEmbeddedDiscretization) = a.model + +function cut(bgmodel::DistributedDiscreteModel,args...) + cut(LevelSetCutter(),bgmodel,args...) +end + +function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + gids = get_cell_gids(bgmodel) + cuts = map(local_views(bgmodel),local_views(gids)) do bgmodel,gids + ownmodel = remove_ghost_cells(bgmodel,gids) + cutgeo = cut(cutter,ownmodel,args...) + change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) + end + ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) + _consistent!(ls_to_bgcell_to_inoutcut,gids) + DistributedEmbeddedDiscretization(cuts,bgmodel) +end + +function Triangulation( + cutgeo::DistributedEmbeddedDiscretization, + in_or_out::ActiveInOrOut, + args...) + + distributed_embedded_triangulation(Triangulation,cutgeo,in_or_out,args...) +end + +function Triangulation(cutgeo::DistributedEmbeddedDiscretization,args...) + trian = distributed_embedded_triangulation(Triangulation,cutgeo,args...) + remove_ghost_cells(trian) +end + +function EmbeddedBoundary(cutgeo::DistributedEmbeddedDiscretization,args...) + trian = distributed_embedded_triangulation(EmbeddedBoundary,cutgeo,args...) + remove_ghost_cells(trian) +end + +function distributed_embedded_triangulation( + T, + cutgeo::DistributedEmbeddedDiscretization, + args...) + + trians = map(local_views(cutgeo)) do lcutgeo + T(lcutgeo,args...) + end + bgmodel = get_background_model(cutgeo) + DistributedTriangulation(trians,bgmodel) +end + +function compute_bgfacet_to_inoutcut( + bgmodel::DistributedDiscreteModel, + bgf_to_ioc::AbstractArray{<:AbstractVector}) + + D = num_dims(eltype(local_views(bgmodel))) + gids = get_cell_gids(bgmodel) + bgf_to_ioc = map( + local_views(bgmodel), + local_views(gids), + bgf_to_ioc) do bgmodel,gids,bgf_to_ioc + + ownmodel = remove_ghost_cells(bgmodel,gids) + f_to_pf = Gridap.Geometry.get_face_to_parent_face(ownmodel,D-1) + _bgf_to_ioc = Vector{eltype(bgf_to_ioc)}(undef,num_faces(bgmodel,D-1)) + _bgf_to_ioc[f_to_pf] .= bgf_to_ioc + _bgf_to_ioc + end + facet_gids = get_face_gids(bgmodel,D-1) + pbgf_to_ioc = PVector(bgf_to_ioc,partition(facet_gids)) + consistent!(pbgf_to_ioc) |> wait + local_values(pbgf_to_ioc) +end + +function compute_bgfacet_to_inoutcut( + cutter::Cutter, + bgmodel::DistributedDiscreteModel, + geo) + + gids = get_cell_gids(bgmodel) + bgf_to_ioc = map(local_views(bgmodel),local_views(gids)) do model,gids + ownmodel = remove_ghost_cells(model,gids) + compute_bgfacet_to_inoutcut(cutter,ownmodel,geo) + end + compute_bgfacet_to_inoutcut(bgmodel,bgf_to_ioc) +end + +function compute_bgfacet_to_inoutcut(bgmodel::DistributedDiscreteModel,args...) + cutter = LevelSetCutter() + compute_bgfacet_to_inoutcut(cutter,bgmodel,args...) +end + +function compute_bgcell_to_inoutcut(cutgeo::DistributedEmbeddedDiscretization,args...) + map(local_views(cutgeo)) do cutgeo + compute_bgcell_to_inoutcut(cutgeo,args...) + end +end + +function remove_ghost_cells(trian::DistributedTriangulation) + model = get_background_model(trian) + gids = get_cell_gids(model) + trians = map(local_views(trian),local_views(gids)) do trian,gids + remove_ghost_cells(trian,gids) + end + DistributedTriangulation(trians,model) +end + +function remove_ghost_cells(trian::AppendedTriangulation,gids) + a = remove_ghost_cells(trian.a,gids) + b = remove_ghost_cells(trian.b,gids) + lazy_append(a,b) +end + +function remove_ghost_cells(trian::SubFacetTriangulation,gids) + model = get_background_model(trian) + D = num_cell_dims(model) + glue = get_glue(trian,Val{D}()) + remove_ghost_cells(glue,trian,gids) +end + +function remove_ghost_cells(model::DiscreteModel,gids::AbstractLocalIndices) + DiscreteModelPortion(model,own_to_local(gids)) +end + +function _consistent!( + p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, + prange::PRange) + + n = map(length,p_to_i_to_a) |> PartitionedArrays.getany + for i in 1:n + p_to_a = map(i_to_a->i_to_a[i],p_to_i_to_a) + PVector(p_to_a,partition(prange)) |> consistent! |> wait + map(p_to_a,p_to_i_to_a) do p_to_a,p_to_ia + copyto!(p_to_ia[i],p_to_a) + end + end +end + +function change_bgmodel( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel, + args...) + + cuts = _change_bgmodels(cutgeo,model,args...) + gids = get_cell_gids(model) + ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) + _consistent!(ls_to_bgcell_to_inoutcut,gids) + DistributedEmbeddedDiscretization(cuts,model) +end + +function _change_bgmodels( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel, + cell_to_newcell) + + map(local_views(cutgeo),local_views(model),cell_to_newcell) do c,m,c_to_nc + change_bgmodel(c,m,c_to_nc) + end +end + +function _change_bgmodels( + cutgeo::DistributedEmbeddedDiscretization, + model::DistributedDiscreteModel) + + map(local_views(cutgeo),local_views(model)) do c,m + change_bgmodel(c,m) + end +end + +function change_bgmodel( + cut::EmbeddedDiscretization, + newmodel::DiscreteModel, + cell_to_newcell=1:num_cells(get_background_model(cut))) + + ls_to_bgc_to_ioc = map(cut.ls_to_bgcell_to_inoutcut) do bgc_to_ioc + new_bgc_to_ioc = Vector{Int8}(undef,num_cells(newmodel)) + new_bgc_to_ioc[cell_to_newcell] = bgc_to_ioc + new_bgc_to_ioc + end + subcells = change_bgmodel(cut.subcells,cell_to_newcell) + subfacets = change_bgmodel(cut.subfacets,cell_to_newcell) + EmbeddedDiscretization( + newmodel, + ls_to_bgc_to_ioc, + subcells, + cut.ls_to_subcell_to_inout, + subfacets, + cut.ls_to_subfacet_to_inout, + cut.oid_to_ls, + cut.geo) +end + +function change_bgmodel(cells::SubCellData,cell_to_newcell) + cell_to_bgcell = lazy_map(Reindex(cell_to_newcell),cells.cell_to_bgcell) + SubCellData( + cells.cell_to_points, + collect(Int32,cell_to_bgcell), + cells.point_to_coords, + cells.point_to_rcoords) +end + +function change_bgmodel(facets::SubFacetData,cell_to_newcell) + facet_to_bgcell = lazy_map(Reindex(cell_to_newcell),facets.facet_to_bgcell) + SubFacetData( + facets.facet_to_points, + facets.facet_to_normal, + collect(Int32,facet_to_bgcell), + facets.point_to_coords, + facets.point_to_rcoords) +end diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl index abbeef4..d03e3fc 100644 --- a/test/DistributedTests/testing_remote_aggs.jl +++ b/test/DistributedTests/testing_remote_aggs.jl @@ -6,11 +6,6 @@ using GridapDistributed using PartitionedArrays using Test -using GridapEmbedded.Distributed: distributed_aggregate -using GridapEmbedded.Distributed: has_remote_aggregation -using GridapEmbedded.Distributed: add_remote_aggregates -using GridapEmbedded.Distributed: change_bgmodel - distribute = PartitionedArrays.DebugArray @@ -78,51 +73,6 @@ strategy = AggregateCutCellsByThreshold(1.0) bgmodel,cutgeo,aggregates = aggregate(strategy,cutgeo,geo,IN); -# aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo,geo,IN); - -# if has_remote_aggregation(bgmodel,aggregates) -# bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) -# cutgeo = change_bgmodel(cutgeo,_bgmodel) -# aggregates = _change_model(aggregates,get_cell_gids(_bgmodel)) -# end - -# laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) - -# bgmodel,cutgeo,laggregates - - - -# @test has_remote_aggregation(bgmodel,aggregates) - -# _bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) - -# @test ! has_remote_aggregation(_bgmodel,aggregates) - -# _cutgeo = change_bgmodel(cutgeo,_bgmodel) - - -# gids = get_cell_gids(_bgmodel) -# # Add remote to aggregates -# _aggregates = map(aggregates,local_to_global(gids)) do agg,l_to_g -# _agg = zeros(Int,length(l_to_g)) -# for (l,g) in enumerate(l_to_g) -# _agg[l] = l > length(agg) ? g : agg[l] -# end -# _agg -# end - - -# bgmodel = _bgmodel -# cutgeo = _cutgeo -# aggregates = _aggregates - -# gids = get_cell_gids(bgmodel) -# # Local aggregates -# laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l -# map(agg) do i -# iszero(i) ? i : g_to_l[i] -# end -# end Ω_bg = Triangulation(bgmodel) @@ -208,8 +158,8 @@ writevtk(Ω,"trian", # end -# TODO: encapsulate and reduce Interfaces -# split files +# TODO: +# add geometries # add tests with mpi # PR end # module From 92be5c6b53b9f34368f55f1634dc240adca667d7 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 12 Apr 2024 13:03:56 +0200 Subject: [PATCH 19/31] [distribute] unify sequential tests --- src/Distributed/Distributed.jl | 1 + src/Distributed/DistributedAgFEM.jl | 8 + src/Distributed/DistributedDiscretizations.jl | 5 + test/DistributedTests/PoissonTests.jl | 83 +++++---- .../sequential/PoissonTests.jl | 1 + test/DistributedTests/testing_remote_aggs.jl | 165 ------------------ 6 files changed, 61 insertions(+), 202 deletions(-) delete mode 100644 test/DistributedTests/testing_remote_aggs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 6c3a7ef..54d7669 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -36,6 +36,7 @@ import GridapEmbedded.Interfaces: cut import GridapEmbedded.Interfaces: EmbeddedBoundary import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut import GridapEmbedded.Interfaces: compute_bgcell_to_inoutcut +import GridapEmbedded.CSG: get_geometry import Gridap.Geometry: Triangulation import Gridap.Geometry: get_background_model import GridapDistributed: local_views diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index ab41727..3a45915 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -39,6 +39,14 @@ function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) bgmodel,cutgeo,laggregates end +function distributed_aggregate( + strategy::AggregateCutCellsByThreshold, + cut::DistributedEmbeddedDiscretization, + in_or_out=IN) + + geo = get_geometry(cut) + distributed_aggregate(strategy,cut,geo,in_or_out) +end function distributed_aggregate( strategy::AggregateCutCellsByThreshold, diff --git a/src/Distributed/DistributedDiscretizations.jl b/src/Distributed/DistributedDiscretizations.jl index 2ceef14..e2a20a5 100644 --- a/src/Distributed/DistributedDiscretizations.jl +++ b/src/Distributed/DistributedDiscretizations.jl @@ -15,6 +15,11 @@ local_views(a::DistributedEmbeddedDiscretization) = a.discretizations get_background_model(a::DistributedEmbeddedDiscretization) = a.model +function get_geometry(a::DistributedEmbeddedDiscretization) + cut = local_views(a) |> PartitionedArrays.getany + get_geometry(cut) +end + function cut(bgmodel::DistributedDiscreteModel,args...) cut(LevelSetCutter(),bgmodel,args...) end diff --git a/test/DistributedTests/PoissonTests.jl b/test/DistributedTests/PoissonTests.jl index 0047d92..7286b4e 100644 --- a/test/DistributedTests/PoissonTests.jl +++ b/test/DistributedTests/PoissonTests.jl @@ -9,8 +9,10 @@ using Test using GridapEmbedded.CSG function main(distribute,parts; + threshold=1, n=8, - cells=(n,n)) + cells=(n,n), + geometry=:circle) ranks = distribute(LinearIndices((prod(parts),))) @@ -18,55 +20,31 @@ function main(distribute,parts; f(x) = -Δ(u)(x) ud(x) = u(x) - L = 1 - p0 = Point(0.0,0.0) - pmin = p0-L/2 - pmax = p0+L/2 - R = 0.35 - geo = disk(R,x0=p0) + geometries = Dict( + :circle => circle_geometry, + :remotes => remotes_geometry, + ) - R = 0.15 - d = L/4 - geo1 = disk(R,x0=p0-d) - geo2 = disk(R,x0=p0+d) - #geo = !union(geo1,geo2) - bgmodel = CartesianDiscreteModel(ranks,parts,pmin,pmax,cells) - # bgtrian = Triangulation(bgmodel) - # writevtk(bgtrian,"bgtrian") + bgmodel,geo = geometries[geometry](ranks,parts,cells) - dp = pmax - pmin - h = dp[1]/n + D = 2 + cell_meas = map(get_cell_measure∘Triangulation,local_views(bgmodel)) + meas = map(first,cell_meas) |> PartitionedArrays.getany + h = meas^(1/D) cutgeo = cut(bgmodel,geo) - strategy = AggregateAllCutCells() - strategy = AggregateCutCellsByThreshold(0.5) - aggregates = aggregate(strategy,cutgeo) - + strategy = AggregateCutCellsByThreshold(threshold) + bgmodel,cutgeo,aggregates = aggregate(strategy,cutgeo) Ω_bg = Triangulation(bgmodel) Ω_act = Triangulation(cutgeo,ACTIVE) Ω = Triangulation(cutgeo,PHYSICAL) Γ = EmbeddedBoundary(cutgeo) - # using GridapEmbedded.Interfaces: CUT - # Ω_in = Triangulation(cutgeo,IN) - # Ω_out = Triangulation(cutgeo,OUT) - # Ω_cut = Triangulation(cutgeo,CUT) - - # writevtk(Ω_in,"trian_in") - # writevtk(Ω_out,"trian_out") - # writevtk(Ω_cut,"trian_cut") - - - writevtk(Ω_bg,"trian") - writevtk(Ω_act,"trian_act") - writevtk(Ω,"trian_O") - writevtk(Γ,"trian_G") - n_Γ = get_normal_vector(Γ) order = 1 @@ -120,7 +98,6 @@ function main(distribute,parts; map(Reindex(col),oid) end - writevtk(Ω_bg,"trian", celldata=[ "aggregate"=>own_aggregates, @@ -135,4 +112,36 @@ function main(distribute,parts; end +function circle_geometry(ranks,parts,cells) + L = 1 + p0 = Point(0.0,0.0) + pmin = p0-L/2 + pmax = p0+L/2 + R = 0.35 + geo = disk(R,x0=p0) + bgmodel = CartesianDiscreteModel(ranks,parts,pmin,pmax,cells) + bgmodel,geo +end + +function remotes_geometry(ranks,parts,cells) + x0 = Point(0.05,0.05) + d1 = VectorValue(0.9,0.0) + d2 = VectorValue(0.0,0.1) + geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) + + x0 = Point(0.15,0.1) + d1 = VectorValue(0.25,0.0) + d2 = VectorValue(0.0,0.6) + geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) + geo12 = union(geo1,geo2) + geo = union(geo12,geo3) + + domain = (0, 1, 0, 1) + bgmodel = CartesianDiscreteModel(ranks,parts,domain,cells) + bgmodel,geo +end + + + + end # module diff --git a/test/DistributedTests/sequential/PoissonTests.jl b/test/DistributedTests/sequential/PoissonTests.jl index 8436c65..0caf1cd 100644 --- a/test/DistributedTests/sequential/PoissonTests.jl +++ b/test/DistributedTests/sequential/PoissonTests.jl @@ -3,5 +3,6 @@ using PartitionedArrays include("../PoissonTests.jl") with_debug() do distribute PoissonTests.main(distribute,(2,2)) + PoissonTests.main(distribute,(4,1),cells=(9,9),geometry=:remotes) end end diff --git a/test/DistributedTests/testing_remote_aggs.jl b/test/DistributedTests/testing_remote_aggs.jl deleted file mode 100644 index d03e3fc..0000000 --- a/test/DistributedTests/testing_remote_aggs.jl +++ /dev/null @@ -1,165 +0,0 @@ -module testing_remote_aggs - -using Gridap -using GridapEmbedded -using GridapDistributed -using PartitionedArrays -using Test - -distribute = PartitionedArrays.DebugArray - - -np = (3,1) -ranks = distribute(LinearIndices((prod(np),))) - -nc = (9,9) - -L = 1 - -# C shape -x0 = Point(0.1,0.1) -d1 = VectorValue(0.8,0.0) -d2 = VectorValue(0.0,0.05) -geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -x0 = Point(0.15,0.125) -d1 = VectorValue(0.2,0.0) -d2 = VectorValue(0.0,0.6) -geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -x0 = Point(0.1,0.6) -d1 = VectorValue(0.8,0.0) -d2 = VectorValue(0.0,0.3) -geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -geo12 = union(geo1,geo2) -geo = union(geo12,geo3) -geo = geo12 - -# Cirlce (test) -R = 0.35 -p0 = Point(0.5,0.5) -geo = disk(R,x0=p0) -geo = geo3 - - -# Bigger cut cells (test) -x0 = Point(0.05,0.05) -d1 = VectorValue(0.9,0.0) -d2 = VectorValue(0.0,0.15) -geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -x0 = Point(0.15,0.125) -d1 = VectorValue(0.25,0.0) -d2 = VectorValue(0.0,0.6) -geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -x0 = Point(0.05,0.6) -d1 = VectorValue(0.9,0.0) -d2 = VectorValue(0.0,0.35) -geo3 = quadrilateral(;x0=x0,d1=d1,d2=d2) - -geo12 = union(geo1,geo2) -geo = union(geo12,geo3) - -geo = geo12 - -domain = (0, 1, 0, 1) -bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) - -cutgeo = cut(bgmodel,geo) - -strategy = AggregateCutCellsByThreshold(1.0) - - -bgmodel,cutgeo,aggregates = aggregate(strategy,cutgeo,geo,IN); - - -Ω_bg = Triangulation(bgmodel) -Ω_act = Triangulation(cutgeo,ACTIVE) -Ω = Triangulation(cutgeo,PHYSICAL) -Γ = EmbeddedBoundary(cutgeo) -n_Γ = get_normal_vector(Γ) - -order = 1 -degree = 2*order -dΩ = Measure(Ω,degree) -dΓ = Measure(Γ,degree) - -reffe = ReferenceFE(lagrangian,Float64,order) - -Vstd = FESpace(Ω_act,reffe) -V = AgFEMSpace(bgmodel,Vstd,aggregates) - -U = TrialFESpace(V) - - -u(x) = x[1] - x[2] -f(x) = -Δ(u)(x) -ud(x) = u(x) - -γd = 10.0 -h = 1/nc[1] - -a(u,v) = - ∫( ∇(v)⋅∇(u) ) * dΩ + - ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ - -l(v) = - ∫( v*f ) * dΩ + - ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ - -op = AffineFEOperator(a,l,U,V) -uh = solve(op) - - -e = u - uh - -l2(u) = sqrt(sum( ∫( u*u )*dΩ )) -h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) - -el2 = l2(e) -eh1 = h1(e) -ul2 = l2(uh) -uh1 = h1(uh) - - -Ω = Triangulation(cutgeo) -Γ = EmbeddedBoundary(cutgeo) -Ωbg = Triangulation(bgmodel) -Ω_act = Triangulation(cutgeo,ACTIVE) - -writevtk(Ω,"trian"); -writevtk(Γ,"bnd"); -writevtk(Ωbg,"bg_trian"); -writevtk(Ω_act,"act_trian"); - -writevtk(Ω,"trian", - cellfields=["uh"=>uh,"u"=>u,"e"=>e],); - - - -# # Debug VTK -# map(local_views(_bgmodel),ranks) do m,p -# writevtk(get_grid(m),"bgmodel_$p") -# end - - -# map(local_views(uh),local_views(bgmodel),ranks) do uh,m,p -# trian = Triangulation(m) -# writevtk(trian,"ltrian_$p",cellfields=["uh"=>uh]) -# end - - -# map(local_views(cutgeo),ranks) do c,p -# m = get_background_model(c) -# c_io = compute_bgcell_to_inoutcut(c,c.geo) -# writevtk(Triangulation(m),"bgtrian_$p",celldata=["inout"=>c_io]) -# end - - -# TODO: -# add geometries -# add tests with mpi -# PR -end # module From 70a05e2d5c59f4ba2e2f08ea1292a9f1b97a6f7a Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 12 Apr 2024 14:45:02 +0200 Subject: [PATCH 20/31] [distributed] setup mpi tests --- Project.toml | 1 + test/DistributedTests/PoissonTests.jl | 3 +- test/DistributedTests/mpi/runtests.jl | 38 ++++++++++++++++++ test/DistributedTests/mpi/runtests_body.jl | 39 +++++++++++++++++++ .../sequential/PoissonTests.jl | 2 +- 5 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 test/DistributedTests/mpi/runtests.jl create mode 100644 test/DistributedTests/mpi/runtests_body.jl diff --git a/Project.toml b/Project.toml index 32969f7..6f8976a 100644 --- a/Project.toml +++ b/Project.toml @@ -13,6 +13,7 @@ Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355" LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" MiniQhull = "978d7f02-9e05-4691-894f-ae31a51d76ca" PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/DistributedTests/PoissonTests.jl b/test/DistributedTests/PoissonTests.jl index 7286b4e..1dd27bf 100644 --- a/test/DistributedTests/PoissonTests.jl +++ b/test/DistributedTests/PoissonTests.jl @@ -133,8 +133,7 @@ function remotes_geometry(ranks,parts,cells) d1 = VectorValue(0.25,0.0) d2 = VectorValue(0.0,0.6) geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) - geo12 = union(geo1,geo2) - geo = union(geo12,geo3) + geo = union(geo1,geo2) domain = (0, 1, 0, 1) bgmodel = CartesianDiscreteModel(ranks,parts,domain,cells) diff --git a/test/DistributedTests/mpi/runtests.jl b/test/DistributedTests/mpi/runtests.jl new file mode 100644 index 0000000..15d3b1e --- /dev/null +++ b/test/DistributedTests/mpi/runtests.jl @@ -0,0 +1,38 @@ +module MPITests + +using MPI +using Test + +#Sysimage +sysimage=nothing +if length(ARGS)==1 + @assert isfile(ARGS[1]) "$(ARGS[1]) must be a valid Julia sysimage file" + sysimage=ARGS[1] +end + +mpidir = @__DIR__ +testdir = joinpath(mpidir,"..") +repodir = joinpath(testdir,"..","..") + +function run_driver(procs,file,sysimage) + mpiexec() do cmd + if sysimage!=nothing + extra_args="-J$(sysimage)" + run(`$cmd -n $procs $(Base.julia_cmd()) $(extra_args) --project=$repodir $(joinpath(mpidir,file))`) + else + run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`) + end + @test true + end +end + +run_driver(4,"runtests_body.jl",sysimage) +run_driver(1,"runtests_body.jl",sysimage) # Check that the degenerated case works + +procs = 4 +mpiexec() do cmd + run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir -e "import Pkg; Pkg.status()" `) +end + + +end # module diff --git a/test/DistributedTests/mpi/runtests_body.jl b/test/DistributedTests/mpi/runtests_body.jl new file mode 100644 index 0000000..df8ffdc --- /dev/null +++ b/test/DistributedTests/mpi/runtests_body.jl @@ -0,0 +1,39 @@ +module MPITestsBody + +using PartitionedArrays +const PArrays = PartitionedArrays +using MPI + +include("../PoissonTests.jl") + +if ! MPI.Initialized() + MPI.Init() +end + +function all_tests(distribute,parts) + ranks = distribute(LinearIndices((prod(parts),))) + + t = PArrays.PTimer(ranks,verbose=true) + PArrays.tic!(t) + + PoissonTests.main(distribute,parts) + PoissonTests.main(distribute,(prod(parts),1),cells=(12,12),geometry=:remotes) + + PArrays.toc!(t,"Geometry") + + display(t) +end + +if MPI.Comm_size(MPI.COMM_WORLD) == 4 + with_mpi() do distribute + all_tests(distribute,(2,2)) + end +elseif MPI.Comm_size(MPI.COMM_WORLD) == 1 + with_mpi() do distribute + all_tests(distribute,(1,1)) + end +else + MPI.Abort(MPI.COMM_WORLD,0) +end + +end #module diff --git a/test/DistributedTests/sequential/PoissonTests.jl b/test/DistributedTests/sequential/PoissonTests.jl index 0caf1cd..d52e007 100644 --- a/test/DistributedTests/sequential/PoissonTests.jl +++ b/test/DistributedTests/sequential/PoissonTests.jl @@ -3,6 +3,6 @@ using PartitionedArrays include("../PoissonTests.jl") with_debug() do distribute PoissonTests.main(distribute,(2,2)) - PoissonTests.main(distribute,(4,1),cells=(9,9),geometry=:remotes) + PoissonTests.main(distribute,(4,1),cells=(12,12),geometry=:remotes) end end From b828ea938d38c130e6cf0d689d2c95dd7ac69976 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 12 Apr 2024 17:08:08 +0200 Subject: [PATCH 21/31] [distributed] adding aggregation tests --- Project.toml | 6 +- src/Distributed/DistributedAgFEM.jl | 8 +- test/DistributedTests/AggregatesTests.jl | 89 ++++++++++++++++++++++ test/DistributedTests/mpi/runtests.jl | 6 -- test/DistributedTests/mpi/runtests_body.jl | 7 +- 5 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 test/DistributedTests/AggregatesTests.jl diff --git a/Project.toml b/Project.toml index 6f8976a..d941914 100644 --- a/Project.toml +++ b/Project.toml @@ -21,17 +21,17 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" algoimWrapper_jll = "3c43aa7b-5398-51f3-8d75-8f051e6faa4d" [compat] -Algoim = "0.2" AbstractTrees = "0.3.3, 0.4" +Algoim = "0.2" Combinatorics = "1" CxxWrap = "0.14" FillArrays = "0.10, 0.11, 0.12, 0.13, 1" Gridap = "0.17.13" +GridapDistributed = "0.3.6" LightGraphs = "1.3.3" MiniQhull = "0.1.0, 0.2, 0.3, 0.4" -julia = "1.3" -GridapDistributed = "0.3.6" PartitionedArrays = "0.3.4" +julia = "1.3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 3a45915..a228902 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -33,7 +33,7 @@ function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) if has_remote_aggregation(bgmodel,aggregates) bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) cutgeo = change_bgmodel(cutgeo,bgmodel) - aggregates = _change_model(aggregates,get_cell_gids(bgmodel)) + aggregates = change_bgmodel(aggregates,get_cell_gids(bgmodel)) end laggregates = _local_aggregates(aggregates,get_cell_gids(bgmodel)) bgmodel,cutgeo,laggregates @@ -461,11 +461,11 @@ function _local_aggregates(cell_to_gcellin,gcell_to_cell) end end -function _change_model(cell_to_gcellin,gids::PRange) - map(_change_model,cell_to_gcellin,local_to_global(gids)) +function change_bgmodel(cell_to_gcellin,gids::PRange) + map(change_bgmodel,cell_to_gcellin,local_to_global(gids)) end -function _change_model(cell_to_gcellin,ncell_to_gcell) +function change_bgmodel(cell_to_gcellin,ncell_to_gcell) ncells = length(cell_to_gcellin) ncell_to_gcellin = zeros(Int,length(ncell_to_gcell)) for (ncell,gcell) in enumerate(ncell_to_gcell) diff --git a/test/DistributedTests/AggregatesTests.jl b/test/DistributedTests/AggregatesTests.jl new file mode 100644 index 0000000..e773775 --- /dev/null +++ b/test/DistributedTests/AggregatesTests.jl @@ -0,0 +1,89 @@ +module DistributedAggregatesTests + +using Gridap +using GridapEmbedded +using GridapDistributed +using PartitionedArrays +using Test +using GridapEmbedded.Distributed: distributed_aggregate +using GridapEmbedded.Distributed: has_remote_aggregation +using GridapEmbedded.Distributed: add_remote_aggregates +using GridapEmbedded.Distributed: change_bgmodel +using GridapEmbedded.Distributed: _extract_remote_cells + + +function main(distribute,parts;vtk=false) + + @assert prod(parts) == 4 + ranks = distribute(LinearIndices((4,))) + + np = (4,1) + nc = (12,12) + + x0 = Point(0.05,0.05) + d1 = VectorValue(0.9,0.0) + d2 = VectorValue(0.0,0.1) + geo1 = quadrilateral(;x0=x0,d1=d1,d2=d2) + + x0 = Point(0.15,0.1) + d1 = VectorValue(0.25,0.0) + d2 = VectorValue(0.0,0.6) + geo2 = quadrilateral(;x0=x0,d1=d1,d2=d2) + + geo = union(geo1,geo2) + + domain = (0, 1, 0, 1) + bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) + cutgeo = cut(bgmodel,geo) + strategy = AggregateCutCellsByThreshold(1.0) + aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo) + + gids = get_cell_gids(bgmodel) + remote_cells,remote_parts = _extract_remote_cells(gids,aggregates,aggregate_owner) + _bgmodel = add_remote_aggregates(bgmodel,aggregates,aggregate_owner) + _cutgeo = change_bgmodel(cutgeo,bgmodel) + _aggregates = change_bgmodel(aggregates,get_cell_gids(bgmodel)) + + @test has_remote_aggregation(bgmodel,aggregates) + @test ! has_remote_aggregation(_bgmodel,_aggregates) + + _remote_cells = distribute([[],[],[28],[28]]) + map(remote_cells,_remote_cells) do a,b + @test a == b + end + _remote_parts = distribute([[],[],[2],[2]]) + map(remote_parts,_remote_parts) do a,b + @test a == b + end + + if vtk + gids = get_cell_gids(bgmodel) + Ωbg = Triangulation(bgmodel) + Γ = EmbeddedBoundary(cutgeo) + Ω = Triangulation(cutgeo) + own_aggregates = map(aggregates,own_to_local(gids)) do agg,o_to_l + agg[o_to_l] + end + writevtk(Ωbg,"bgmodel", + celldata=["gcell"=>own_to_global(gids),"aggregate"=>own_aggregates]) + writevtk(Γ,"boudary") + writevtk(Ω,"trian") + end + + # No remotes + + np = (2,2) + nc = (8,8) + R = 0.4 + geo = disk(R,x0=Point(0.5,0.5)) + domain = (0, 1, 0, 1) + bgmodel = CartesianDiscreteModel(ranks,np,domain,nc) + cutgeo = cut(bgmodel,geo) + strategy = AggregateCutCellsByThreshold(1.0) + aggregates,aggregate_owner = distributed_aggregate(strategy,cutgeo) + + @test ! has_remote_aggregation(bgmodel,aggregates) + +end + +end # module diff --git a/test/DistributedTests/mpi/runtests.jl b/test/DistributedTests/mpi/runtests.jl index 15d3b1e..6e1a679 100644 --- a/test/DistributedTests/mpi/runtests.jl +++ b/test/DistributedTests/mpi/runtests.jl @@ -29,10 +29,4 @@ end run_driver(4,"runtests_body.jl",sysimage) run_driver(1,"runtests_body.jl",sysimage) # Check that the degenerated case works -procs = 4 -mpiexec() do cmd - run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir -e "import Pkg; Pkg.status()" `) -end - - end # module diff --git a/test/DistributedTests/mpi/runtests_body.jl b/test/DistributedTests/mpi/runtests_body.jl index df8ffdc..fe600b7 100644 --- a/test/DistributedTests/mpi/runtests_body.jl +++ b/test/DistributedTests/mpi/runtests_body.jl @@ -5,6 +5,7 @@ const PArrays = PartitionedArrays using MPI include("../PoissonTests.jl") +include("../AggregatesTests.jl") if ! MPI.Initialized() MPI.Init() @@ -18,8 +19,12 @@ function all_tests(distribute,parts) PoissonTests.main(distribute,parts) PoissonTests.main(distribute,(prod(parts),1),cells=(12,12),geometry=:remotes) + PArrays.toc!(t,"Poisson") - PArrays.toc!(t,"Geometry") + if prod(parts) == 4 + DistributedAggregatesTests.main(distribute,parts) + end + PArrays.toc!(t,"Aggregates") display(t) end From e63b612d2b5be8c3446707bd29b7a683bccfff9c Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 12 Apr 2024 17:54:41 +0200 Subject: [PATCH 22/31] [distributed] missing file --- test/DistributedTests/sequential/AggregatesTestsSeq.jl | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 test/DistributedTests/sequential/AggregatesTestsSeq.jl diff --git a/test/DistributedTests/sequential/AggregatesTestsSeq.jl b/test/DistributedTests/sequential/AggregatesTestsSeq.jl new file mode 100644 index 0000000..8828ac7 --- /dev/null +++ b/test/DistributedTests/sequential/AggregatesTestsSeq.jl @@ -0,0 +1,7 @@ +module AggregatesTestsSeq +using PartitionedArrays +include("../AggregatesTests.jl") +with_debug() do distribute + DistributedAggregatesTests.main(distribute,(2,2)) +end +end From b93f42fa3b66d66cb86dcaceba86d44be73bffe4 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Fri, 12 Apr 2024 18:01:08 +0200 Subject: [PATCH 23/31] [debug] try to fix ci error --- src/Distributed/DistributedAgFEM.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index a228902..8ee72fa 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -402,6 +402,8 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) lid_to_gid = map(vcat,local_to_global(gids),new_gids) lid_to_part = map(vcat,local_to_owner(gids),new_parts) p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p + l_to_g = collect(Int64,l_to_g) + l_to_p = collect(Int64,l_to_p) LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) end PRange(p) From d510d2b74da7f9fb628e1821ab9547b0a7e8c62d Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Tue, 16 Apr 2024 11:08:37 +0200 Subject: [PATCH 24/31] [distributed] missing cut_facets interfaces --- src/Distributed/Distributed.jl | 10 + src/Distributed/DistributedDiscretizations.jl | 106 ++++++++- src/Distributed/DistributedQuadratures.jl | 18 ++ .../MomentFittedQuadratures.jl | 32 ++- test/DistributedTests/DistributedDevs.jl | 214 ------------------ test/DistributedTests/PoissonTests.jl | 4 +- 6 files changed, 156 insertions(+), 228 deletions(-) create mode 100644 src/Distributed/DistributedQuadratures.jl delete mode 100644 test/DistributedTests/DistributedDevs.jl diff --git a/src/Distributed/Distributed.jl b/src/Distributed/Distributed.jl index 54d7669..aadb4d5 100644 --- a/src/Distributed/Distributed.jl +++ b/src/Distributed/Distributed.jl @@ -6,6 +6,7 @@ using PartitionedArrays using FillArrays using Gridap.Arrays +using Gridap.CellData using Gridap.Geometry using Gridap.Helpers using Gridap.ReferenceFEs @@ -18,13 +19,17 @@ using GridapEmbedded.Interfaces: ActiveInOrOut using GridapEmbedded.Interfaces: SubFacetTriangulation using GridapEmbedded.Interfaces: SubCellData using GridapEmbedded.Interfaces: SubFacetData +using GridapEmbedded.Interfaces: AbstractEmbeddedDiscretization using GridapEmbedded.AgFEM: _touch_aggregated_cells! using GridapEmbedded.AgFEM: AggregateCutCellsByThreshold +using GridapEmbedded.MomentFittedQuadratures: MomentFitted using Gridap.Geometry: AppendedTriangulation +using Gridap.Geometry: get_face_to_parent_face using GridapDistributed: DistributedDiscreteModel using GridapDistributed: DistributedTriangulation using GridapDistributed: DistributedFESpace using GridapDistributed: DistributedSingleFieldFESpace +using GridapDistributed: DistributedMeasure using GridapDistributed: add_ghost_cells using GridapDistributed: generate_gids using GridapDistributed: generate_cell_gids @@ -33,11 +38,14 @@ using GridapDistributed: _find_vector_type import GridapEmbedded.AgFEM: aggregate import GridapEmbedded.AgFEM: AgFEMSpace import GridapEmbedded.Interfaces: cut +import GridapEmbedded.Interfaces: cut_facets import GridapEmbedded.Interfaces: EmbeddedBoundary import GridapEmbedded.Interfaces: compute_bgfacet_to_inoutcut import GridapEmbedded.Interfaces: compute_bgcell_to_inoutcut import GridapEmbedded.CSG: get_geometry import Gridap.Geometry: Triangulation +import Gridap.Geometry: SkeletonTriangulation +import Gridap.Geometry: BoundaryTriangulation import Gridap.Geometry: get_background_model import GridapDistributed: local_views import GridapDistributed: remove_ghost_cells @@ -46,4 +54,6 @@ include("DistributedDiscretizations.jl") include("DistributedAgFEM.jl") +include("DistributedQuadratures.jl") + end # module diff --git a/src/Distributed/DistributedDiscretizations.jl b/src/Distributed/DistributedDiscretizations.jl index e2a20a5..850eb18 100644 --- a/src/Distributed/DistributedDiscretizations.jl +++ b/src/Distributed/DistributedDiscretizations.jl @@ -1,13 +1,13 @@ -struct DistributedEmbeddedDiscretization{Dp,T,A,B} <: GridapType +struct DistributedEmbeddedDiscretization{A,B} <: GridapType discretizations::A model::B function DistributedEmbeddedDiscretization( - d::AbstractArray{<:EmbeddedDiscretization{Dp,T}}, - model::DistributedDiscreteModel) where {Dp,T} + d::AbstractArray{<:AbstractEmbeddedDiscretization}, + model::DistributedDiscreteModel) A = typeof(d) B = typeof(model) - new{Dp,T,A,B}(d,model) + new{A,B}(d,model) end end @@ -31,8 +31,29 @@ function cut(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) cutgeo = cut(cutter,ownmodel,args...) change_bgmodel(cutgeo,bgmodel,own_to_local(gids)) end - ls_to_bgcell_to_inoutcut = map(c->c.ls_to_bgcell_to_inoutcut,cuts) - _consistent!(ls_to_bgcell_to_inoutcut,gids) + consistent_bgcell_to_inoutcut!(cuts,gids) + DistributedEmbeddedDiscretization(cuts,bgmodel) +end + +function cut_facets(bgmodel::DistributedDiscreteModel,args...) + cut_facets(LevelSetCutter(),bgmodel,args...) +end + +function cut_facets(cutter::Cutter,bgmodel::DistributedDiscreteModel,args...) + D = map(num_dims,local_views(bgmodel)) |> PartitionedArrays.getany + cell_gids = get_cell_gids(bgmodel) + facet_gids = get_face_gids(bgmodel,D-1) + cuts = map( + local_views(bgmodel), + local_views(cell_gids), + local_views(facet_gids)) do bgmodel,cell_gids,facet_gids + ownmodel = remove_ghost_cells(bgmodel,cell_gids) + facet_to_pfacet = get_face_to_parent_face(ownmodel,D-1) + cutfacets = cut_facets(cutter,ownmodel,args...) + cutfacets = change_bgmodel(cutfacets,bgmodel,facet_to_pfacet) + remove_ghost_subfacets(cutfacets,facet_gids) + end + consistent_bgfacet_to_inoutcut!(cuts,facet_gids) DistributedEmbeddedDiscretization(cuts,bgmodel) end @@ -54,6 +75,16 @@ function EmbeddedBoundary(cutgeo::DistributedEmbeddedDiscretization,args...) remove_ghost_cells(trian) end +function SkeletonTriangulation(cutgeo::DistributedEmbeddedDiscretization,args...) + trian = distributed_embedded_triangulation(SkeletonTriangulation,cutgeo,args...) + remove_ghost_cells(trian) +end + +function BoundaryTriangulation(cutgeo::DistributedEmbeddedDiscretization,args...) + trian = distributed_embedded_triangulation(BoundaryTriangulation,cutgeo,args...) + remove_ghost_cells(trian) +end + function distributed_embedded_triangulation( T, cutgeo::DistributedEmbeddedDiscretization, @@ -139,6 +170,30 @@ function remove_ghost_cells(model::DiscreteModel,gids::AbstractLocalIndices) DiscreteModelPortion(model,own_to_local(gids)) end +function consistent_bgcell_to_inoutcut!( + cuts::AbstractArray{<:AbstractEmbeddedDiscretization}, + gids::PRange) + + ls_to_bgcell_to_inoutcut = map(get_ls_to_bgcell_to_inoutcut,cuts) + _consistent!(ls_to_bgcell_to_inoutcut,gids) +end + +function get_ls_to_bgcell_to_inoutcut(cut::EmbeddedDiscretization) + cut.ls_to_bgcell_to_inoutcut +end + +function consistent_bgfacet_to_inoutcut!( + cuts::AbstractArray{<:AbstractEmbeddedDiscretization}, + gids::PRange) + + ls_to_bgfacet_to_inoutcut = map(get_ls_to_bgfacet_to_inoutcut,cuts) + _consistent!(ls_to_bgfacet_to_inoutcut,gids) +end + +function get_ls_to_bgfacet_to_inoutcut(cut::EmbeddedFacetDiscretization) + cut.ls_to_facet_to_inoutcut +end + function _consistent!( p_to_i_to_a::AbstractArray{<:Vector{<:Vector}}, prange::PRange) @@ -207,6 +262,28 @@ function change_bgmodel( cut.geo) end +function change_bgmodel( + cut::EmbeddedFacetDiscretization, + newmodel::DiscreteModel, + facet_to_newfacet=1:num_facets(get_background_model(cut))) + + nfacets = num_facets(newmodel) + + ls_to_bgf_to_ioc = map(cut.ls_to_facet_to_inoutcut) do bgf_to_ioc + new_bgf_to_ioc = Vector{Int8}(undef,nfacets) + new_bgf_to_ioc[facet_to_newfacet] = bgf_to_ioc + new_bgf_to_ioc + end + subfacets = change_bgmodel(cut.subfacets,facet_to_newfacet) + EmbeddedFacetDiscretization( + newmodel, + ls_to_bgf_to_ioc, + subfacets, + cut.ls_to_subfacet_to_inout, + cut.oid_to_ls, + cut.geo) +end + function change_bgmodel(cells::SubCellData,cell_to_newcell) cell_to_bgcell = lazy_map(Reindex(cell_to_newcell),cells.cell_to_bgcell) SubCellData( @@ -225,3 +302,20 @@ function change_bgmodel(facets::SubFacetData,cell_to_newcell) facets.point_to_coords, facets.point_to_rcoords) end + +function remove_ghost_subfacets(cut::EmbeddedFacetDiscretization,facet_gids) + bgfacet_mask = map(!iszero,local_to_owner(facet_gids)) + subfacet_mask = map(Reindex(bgfacet_mask),cut.subfacets.cell_to_bgcell) + new_subfacets = findall(subfacet_mask) + subfacets = SubCellData(cut.subfacets,new_subfacets) + ls_to_subfacet_to_inout = map(cut.ls_to_subfacet_to_inout) do sf_to_io + map(Reindex(sf_to_io),new_subfacets) + end + EmbeddedFacetDiscretization( + cut.bgmodel, + cut.ls_to_facet_to_inoutcut, + subfacets, + ls_to_subfacet_to_inout, + cut.oid_to_ls, + cut.geo) +end diff --git a/src/Distributed/DistributedQuadratures.jl b/src/Distributed/DistributedQuadratures.jl new file mode 100644 index 0000000..06a40a4 --- /dev/null +++ b/src/Distributed/DistributedQuadratures.jl @@ -0,0 +1,18 @@ + +function CellData.Measure( + t::DistributedTriangulation, + quad::Tuple{MomentFitted,Vararg}; + kwargs...) + @notimplemented + name, _args, _kwargs = quad + cut,cutfacets,_args... = _args + t = remove_ghost_cells(t) + measures = map( + local_views(t), + local_views(cut), + local_views(cutfacets)) do trian,cut,cutfacets + quad = name, (cut,cutfacets,_args...), _kwargs + Measure(trian,quad;kwargs...) + end + DistributedMeasure(measures) +end diff --git a/src/MomentFittedQuadratures/MomentFittedQuadratures.jl b/src/MomentFittedQuadratures/MomentFittedQuadratures.jl index 5c6f114..2102263 100644 --- a/src/MomentFittedQuadratures/MomentFittedQuadratures.jl +++ b/src/MomentFittedQuadratures/MomentFittedQuadratures.jl @@ -43,15 +43,36 @@ function Quadrature(trian::Grid, Quadrature(trian,momentfitted,cut,geo,degree;in_or_out=in_or_out) end +function Quadrature(trian::Grid, + ::MomentFitted, + cut::AbstractEmbeddedDiscretization, + cutf::AbstractEmbeddedDiscretization, + degree::Int; + in_or_out=IN) + geo = get_geometry(cut) + Quadrature(trian,momentfitted,cut,cutf,geo,degree;in_or_out=in_or_out) +end + +function Quadrature(trian::Grid, + ::MomentFitted, + cut::AbstractEmbeddedDiscretization, + geo::CSG.Geometry, + degree::Int; + in_or_out=IN) + cutf = cut_facets(cut,geo) + Quadrature(trian,momentfitted,cut,cutf,geo,degree;in_or_out=in_or_out) +end + function Quadrature(active_mesh::Grid, ::MomentFitted, cut::AbstractEmbeddedDiscretization, + cutf::AbstractEmbeddedDiscretization, geo::CSG.Geometry, degree::Int; in_or_out=IN) acell_to_point_vals, acell_to_weight_vals = # - compute_lag_moments_from_leg(cut,geo,in_or_out,degree) + compute_lag_moments_from_leg(cut,cutf,geo,in_or_out,degree) acell_to_weight_vals = collect(get_array(acell_to_weight_vals)) D = num_dims(active_mesh) @@ -131,6 +152,7 @@ function legendreToMonomial(n::Int,d::Int) end function compute_lag_moments_from_leg(cut::AbstractEmbeddedDiscretization, + cutf::AbstractEmbeddedDiscretization, geo::CSG.Geometry, in_or_out, degree::Int) @@ -138,7 +160,7 @@ function compute_lag_moments_from_leg(cut::AbstractEmbeddedDiscretization, T = eltype(eltype(get_node_coordinates(cut_trian))) D = num_dims(cut_trian) b = MonomialBasis{D}(T,degree) - mon_contribs = compute_monomial_domain_contribution(cut,geo,in_or_out,b,degree) + mon_contribs = compute_monomial_domain_contribution(cut,cutf,geo,in_or_out,b,degree) mon_moments = compute_monomial_cut_cell_moments(cut_trian,mon_contribs,b) mon_to_leg = Fill(legendreToMonomial(degree,D),num_cells(cut_trian)) leg_moments = lazy_map(*,mon_to_leg,mon_moments) @@ -149,14 +171,16 @@ function compute_lag_moments_from_leg(cut::AbstractEmbeddedDiscretization, end function compute_monomial_domain_contribution(cut::AbstractEmbeddedDiscretization, + cutf::AbstractEmbeddedDiscretization, in_or_out, b::MonomialBasis, deg::Int) geo = get_geometry(cut) - compute_monomial_domain_contribution(cut,geo,in_or_out,b,deg) + compute_monomial_domain_contribution(cut,cutf,geo,in_or_out,b,deg) end function compute_monomial_domain_contribution(cut::AbstractEmbeddedDiscretization, + cutf::AbstractEmbeddedDiscretization, geo::CSG.Geometry, in_or_out::Integer, b::MonomialBasis, @@ -164,7 +188,6 @@ function compute_monomial_domain_contribution(cut::AbstractEmbeddedDiscretizatio cut_io = CutInOrOut(in_or_out) dir_Γᵉ = (-1)^(in_or_out==OUT) - cutf = cut_facets(cut,geo) # Embedded facets Γᵉ = EmbeddedBoundary(cut,geo) # Interior fitted cut facets @@ -354,4 +377,3 @@ function _get_terms_degrees(c::CartesianIndex) end end #module - diff --git a/test/DistributedTests/DistributedDevs.jl b/test/DistributedTests/DistributedDevs.jl deleted file mode 100644 index 33f686e..0000000 --- a/test/DistributedTests/DistributedDevs.jl +++ /dev/null @@ -1,214 +0,0 @@ -module DistributedDevs - -using Gridap -using GridapEmbedded -using GridapDistributed -using PartitionedArrays -using Test - -using GridapEmbedded.Distributed: distributed_aggregate -using GridapEmbedded.Distributed: has_remote_aggregation - -distribute = PartitionedArrays.DebugArray - -np = (2,1) - -ranks = distribute(LinearIndices((prod(np),))) - -L = 1 -p0 = Point(0.0,0.0) -pmin = p0-L/2 -pmax = p0+L/2 - -R = 0.3 -x = VectorValue(1.0,0.0) - -d1 = VectorValue(0.4,0.0) -d2 = VectorValue(0.0,0.05) -geo = disk(R,x0=p0) -# geo2 = quadrilateral(;x0=p0-d2/2,d1=d1,d2=d2) -# geo = union(geo1,geo2) - -n = 9 -mesh_partition = (n,n) -bgmodel = CartesianDiscreteModel(ranks,np,pmin,pmax,mesh_partition) - -cutgeo = cut(bgmodel,geo) - -bgf_to_ioc = compute_bgfacet_to_inoutcut(bgmodel,geo) - -Ω = Triangulation(cutgeo) - -writevtk(Ω,"trian") - -strategy = AggregateCutCellsByThreshold(1.0) -aggregates,aggregate_owner,aggregate_neig = distributed_aggregate( - strategy,cutgeo,geo,IN) - - -gids = get_cell_gids(bgmodel) -# TODO: do not use owner for this query (has ghost aggregation?) -local_aggretation = map(aggregate_owner,own_to_local(gids),own_to_owner(gids)) do owns,o_to_l,o - owners = map(Reindex(owns),o_to_l) - all(lazy_map(==,owners,o)) -end -has_local_aggretation = reduction(&,local_aggretation,destination=:all) |> PartitionedArrays.getany - - -@test !has_remote_aggregation(bgmodel,aggregates) - - -@test !has_local_aggretation -# @test !has_remote_aggregation - -cut_in = map(aggregates) do agg - findall(!iszero,agg) -end - -cut_owner = map(aggregate_owner,cut_in) do owners,cut_in - owners[cut_in] -end - -aggregates -gids = get_cell_gids(bgmodel) - -laggregates = map(aggregates,global_to_local(gids)) do agg,g_to_l - map(agg) do a - if iszero(a) - a - else - g_to_l[a] - end - end -end - -oaggregates = map(aggregates,own_to_local(gids)) do agg,o_to_l - map(Reindex(agg),o_to_l) -end - -oaggregate_owner = map(aggregate_owner,own_to_local(gids)) do agg,o_to_l - map(Reindex(agg),o_to_l) -end - - -Ωbg = Triangulation(bgmodel) -Ω = Triangulation(cutgeo) -Ωin = Triangulation(cutgeo,IN) -Γ = EmbeddedBoundary(cutgeo) -Ω_act = Triangulation(cutgeo,ACTIVE) - - - -n_Γ = get_normal_vector(Γ) - -order = 1 -degree = 2*order -dΩ = Measure(Ω,degree) -dΓ = Measure(Γ,degree) - -reffe = ReferenceFE(lagrangian,Float64,order) - -Vstd = FESpace(Ω_act,reffe) - - -map(num_cells,local_views(Ω_act)) - - -map(local_views(Ω_act),ranks) do trian,p - writevtk(trian,"trian_act_$p") -end - -V = AgFEMSpace(bgmodel,Vstd,laggregates) -# V = Vstd -U = TrialFESpace(V) - -h = 1.0/n -γd = 10.0 - -u(x) = x[1] - x[2] -f(x) = -Δ(u)(x) -ud(x) = u(x) - - -a(u,v) = - ∫( ∇(v)⋅∇(u) ) * dΩ + - ∫( (γd/h)*v*u - v*(n_Γ⋅∇(u)) - (n_Γ⋅∇(v))*u ) * dΓ - -l(v) = - ∫( v*f ) * dΩ + - ∫( (γd/h)*v*ud - (n_Γ⋅∇(v))*ud ) * dΓ - -op = AffineFEOperator(a,l,U,V) -uh = solve(op) - -e = u - uh - -l2(u) = sqrt(sum( ∫( u*u )*dΩ )) -h1(u) = sqrt(sum( ∫( u*u + ∇(u)⋅∇(u) )*dΩ )) - -el2 = l2(e) -eh1 = h1(e) -ul2 = l2(uh) -uh1 = h1(uh) - - -writevtk(Ωin,"trian_in") -writevtk(Γ,"bnd") -writevtk(Ωbg,"bgtrian",celldata= - ["aggregate"=>oaggregates, - "aggregate_owner"=>oaggregate_owner]) - -writevtk(Ω,"trian",cellfields=["uh"=>uh]) - - - - -using GridapEmbedded.Distributed: add_remote_aggregates - -np = (3,) - -ranks = distribute(LinearIndices((prod(np),))) -m = CartesianDiscreteModel(ranks,np,(0, 1),(9,)) - - -aggregates = DebugArray([[1, 9, 8, 5], [3, 4, 9, 6, 7], [6, 7, 3, 9]]) -aggregate_owner = DebugArray([[1, 3, 3, 2], [1, 2, 3, 2, 3], [2, 3, 1, 3]]) - -am = add_remote_aggregates(m,aggregates,aggregate_owner) - -gids = partition(get_cell_gids(am)) -test_gids = DebugArray([[1,2,3,4,5,8,9],[3,4,5,6,7,9],[6,7,8,9,3]]) -map(gids,test_gids) do gids,_gids - @test all(gids .==_gids) -end - -map(local_views(am),ranks) do m,p - writevtk(Triangulation(m),"m$p") -end - - -# import GridapEmbedded.Distributed: change_bgmodel -# using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization -# using GridapDistributed: DistributedDiscreteModel - -# TODO: -# - print aggregates [x] -# - move to src [x] -# - test aggregates with several geometries [x] -# - reconstruct paths [x] -# - add remote roots to model [x] -# - migrate model [x] -# - agfem fe space with remotes - -# TODO: parallel aggregation (parallel aggfem article) -# 0. exchange measures and aggregates through gid [x] -# 1. root in ghost layer [x] -# 2. root in neighbors [x] -# 3. root in neighbors of neighbors [x] - -# TODO: -# - test possion eq with root in ghost layer (simplify this geometry) [x] -# - test graph reconstruction in 1D arrays [x] -# - add remote roots to model [x] - -end diff --git a/test/DistributedTests/PoissonTests.jl b/test/DistributedTests/PoissonTests.jl index 1dd27bf..4c736c3 100644 --- a/test/DistributedTests/PoissonTests.jl +++ b/test/DistributedTests/PoissonTests.jl @@ -20,14 +20,11 @@ function main(distribute,parts; f(x) = -Δ(u)(x) ud(x) = u(x) - - geometries = Dict( :circle => circle_geometry, :remotes => remotes_geometry, ) - bgmodel,geo = geometries[geometry](ranks,parts,cells) D = 2 @@ -36,6 +33,7 @@ function main(distribute,parts; h = meas^(1/D) cutgeo = cut(bgmodel,geo) + cutgeo_facets = cut_facets(bgmodel,geo) strategy = AggregateCutCellsByThreshold(threshold) bgmodel,cutgeo,aggregates = aggregate(strategy,cutgeo) From 03931d9128eb7ab24c0de9422c8c1de6d408e38a Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Tue, 16 Apr 2024 11:19:16 +0200 Subject: [PATCH 25/31] update GridapEmbedded --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 28ca3a3..82db8d9 100644 --- a/Project.toml +++ b/Project.toml @@ -26,7 +26,7 @@ Algoim = "0.2" Combinatorics = "1" CxxWrap = "0.14" FillArrays = "0.10, 0.11, 0.12, 0.13, 1" -GridapDistributed = "0.3.6" +GridapDistributed = "0.3, 0.4" Gridap = "0.17, 0.18" LightGraphs = "1.3.3" MiniQhull = "0.1.0, 0.2, 0.3, 0.4" From d43b730a14606926a83fdd8878bab8856cfa58df Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Tue, 16 Apr 2024 11:22:29 +0200 Subject: [PATCH 26/31] [distributed] Int bug in x86 test --- src/Distributed/DistributedAgFEM.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 8ee72fa..01e23c5 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -402,8 +402,8 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) lid_to_gid = map(vcat,local_to_global(gids),new_gids) lid_to_part = map(vcat,local_to_owner(gids),new_parts) p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p - l_to_g = collect(Int64,l_to_g) - l_to_p = collect(Int64,l_to_p) + l_to_g = collect(Int,l_to_g) + l_to_p = collect(Int,l_to_p) LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) end PRange(p) From 3732bde23b87680de4062573053a9c5f5a95c3f0 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Wed, 17 Apr 2024 10:23:12 +0200 Subject: [PATCH 27/31] [distributed] bugfix --- src/Distributed/DistributedAgFEM.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 01e23c5..1da5b9f 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -24,7 +24,7 @@ function AgFEMSpace( nldofs = map(num_free_dofs,spaces) gids = generate_gids(trian_gids,cell_to_ldofs,nldofs) vector_type = _find_vector_type(spaces,gids) - DistributedSingleFieldFESpace(spaces,gids,vector_type) + DistributedSingleFieldFESpace(spaces,gids,trian,vector_type) end function aggregate(strategy,cutgeo::DistributedEmbeddedDiscretization,args...) From b51524a5040c48e9fd5ceecd44f53e83c64c769e Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Wed, 17 Apr 2024 14:42:58 +0200 Subject: [PATCH 28/31] [distributed] minor --- src/Distributed/DistributedAgFEM.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 1da5b9f..139e8c8 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -403,7 +403,7 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) lid_to_part = map(vcat,local_to_owner(gids),new_parts) p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p l_to_g = collect(Int,l_to_g) - l_to_p = collect(Int,l_to_p) + l_to_p = collect(Int32,l_to_p) LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) end PRange(p) From 189f5dd975158f9ef6c2b433a947f5b5bd137184 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Mon, 22 Apr 2024 13:09:08 +0200 Subject: [PATCH 29/31] [distributed] debug ci 32 bit --- src/Distributed/DistributedAgFEM.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 139e8c8..5e08abc 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -404,12 +404,14 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p l_to_g = collect(Int,l_to_g) l_to_p = collect(Int32,l_to_p) - LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) + ids = LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) + @show typeof(l_to_g) + @show typeof(local_to_global(ids)) + ids end PRange(p) end - function has_remote_aggregation(model::DistributedDiscreteModel,aggregates) gids = get_cell_gids(model) has_remote_aggregation(aggregates,gids) From 396df80176206ef7e20705181763269e2481b498 Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Mon, 22 Apr 2024 16:23:38 +0200 Subject: [PATCH 30/31] [distributed] debug x86 --- src/Distributed/DistributedAgFEM.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 5e08abc..6414cef 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -407,6 +407,7 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) ids = LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) @show typeof(l_to_g) @show typeof(local_to_global(ids)) + @show typeof(ghost_to_global(ids)) ids end PRange(p) From d3e60791bf8ac2bfbfbc49ee56d745ed935f70df Mon Sep 17 00:00:00 2001 From: Pere Antoni Martorell Date: Wed, 24 Apr 2024 17:40:47 +0200 Subject: [PATCH 31/31] [distributed] do not test distributed with 32-bit machines --- src/Distributed/DistributedAgFEM.jl | 6 +----- test/DistributedTests/runtests.jl | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Distributed/DistributedAgFEM.jl b/src/Distributed/DistributedAgFEM.jl index 6414cef..b4f0961 100644 --- a/src/Distributed/DistributedAgFEM.jl +++ b/src/Distributed/DistributedAgFEM.jl @@ -404,11 +404,7 @@ function add_remote_ids(gids::PRange,remote_gids,remote_parts) p = map(lid_to_gid,lid_to_part,partition(gids)) do l_to_g,l_to_p,p l_to_g = collect(Int,l_to_g) l_to_p = collect(Int32,l_to_p) - ids = LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) - @show typeof(l_to_g) - @show typeof(local_to_global(ids)) - @show typeof(ghost_to_global(ids)) - ids + LocalIndices(length(gids),part_id(p),l_to_g,l_to_p) end PRange(p) end diff --git a/test/DistributedTests/runtests.jl b/test/DistributedTests/runtests.jl index e4b3489..fd13110 100644 --- a/test/DistributedTests/runtests.jl +++ b/test/DistributedTests/runtests.jl @@ -1,5 +1,8 @@ module DistributedTests -include("sequential/runtests.jl") +if Int === Int64 + include("sequential/runtests.jl") + include("mpi/runtests.jl") +end end # module