[distributed] setup distributed interface

Project.toml

@@ -9,18 +9,20 @@ Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
 Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e"
+GridapDistributed = "f9701e48-63b3-45aa-9a63-9bc6c271f355"
 GridapEmbedded = "8838a6a3-0006-4405-b874-385995508d5d"
 IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MeshIO = "7269a6da-0436-5bbc-96c2-40638cbb6118"
+PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 
 [compat]
 AlgebraicMultigrid = "0.5,0.6"
 Downloads = "1.4"
 FileIO = "1.6"
 FillArrays = "0.11, 0.13, 1"
-Gridap = "0.17.16"
+Gridap = "0.17.16, 0.18"
 GridapEmbedded = "0.8, 0.9"
 IterativeSolvers = "0.9"
 MeshIO = "0.4"

src/Embedded.jl

@@ -234,7 +234,7 @@ function send_to_ref_space(
 end
 
 function send_to_ref_space(grid::Grid,cell_to_bgcell::Vector,subgrid::Grid)
-  bgcell_map = get_cell_map(grid)
+  bgcell_map = _get_cell_affine_map(grid)
   bgcell_invmap = lazy_map(inverse_map,bgcell_map)
   cell_invmap = lazy_map(Reindex(bgcell_invmap),cell_to_bgcell)
   cell_nodes = get_cell_node_ids(subgrid)
@@ -293,3 +293,66 @@ function _collect(a::LazyArray)
   end
   b
 end
+
+function _get_cell_affine_map(grid::Grid)
+  @assert all(==(1),map(get_order,get_reffes(grid)))
+  D = num_dims(grid)
+  x0 = Point(tfill(0.0,Val{D}()))
+  ncells = num_cells(grid)
+  cell_map = get_cell_map(grid)
+  cell_map_gradient = lazy_map(∇,cell_map)
+  origins = lazy_map(evaluate,cell_map,Fill(x0,ncells))
+  gradiens = lazy_map(evaluate,cell_map_gradient,Fill(x0,ncells))
+  lazy_map(affine_map,gradiens,origins)
+end
+
+
+function cut(cutter::STLCutter,bgmodel::DistributedDiscreteModel,args...)
+  D = map(num_dims,local_views(bgmodel)) |> PartitionedArrays.getany
+  cell_gids = get_cell_gids(bgmodel)
+  facet_gids = get_face_gids(bgmodel,D-1)
+  cuts = map(
+    local_views(bgmodel),
+    local_views(cell_gids),
+    local_views(facet_gids)) do bgmodel,cell_gids,facet_gids
+    ownmodel = remove_ghost_cells(bgmodel,cell_gids)
+    cell_to_pcell = get_cell_to_parent_cell(ownmodel)
+    facet_to_pfacet = get_face_to_parent_face(ownmodel,D-1)
+    cutgeo = cut(cutter,ownmodel,args...)
+    cutgeo = change_bgmodel(cutgeo,bgmodel,cell_to_pcell,facet_to_pfacet)
+    remove_ghost_subfacets(cutgeo,facet_gids)
+  end
+  consistent_bgcell_to_inoutcut!(cuts,cell_gids)
+  consistent_bgfacet_to_inoutcut!(cuts,facet_gids)
+  DistributedEmbeddedDiscretization(cuts,bgmodel)
+end
+
+function cut_facets(cut::DistributedEmbeddedDiscretization)
+  bgmodel = get_background_model(cut)
+  cutfacets = map(cut_facets,local_views(cut))
+  DistributedEmbeddedDiscretization(cutfacets,bgmodel)
+end
+
+function change_bgmodel(
+  cutgeo::STLEmbeddedDiscretization,
+  model::DiscreteModel,
+  cell_to_newcell=1:num_cells(get_background_model(cutgeo)),
+  facet_to_newfacet=1:num_facets(get_background_model(cutgeo)))
+
+  _cut = change_bgmodel(cutgeo.cut,model,cell_to_newcell)
+  _cutfacets = change_bgmodel(cutgeo.cutfacets,model,facet_to_newfacet)
+  STLEmbeddedDiscretization(_cut,_cutfacets)
+end
+
+function remove_ghost_subfacets(cut::STLEmbeddedDiscretization,facet_gids)
+  cutfacets = remove_ghost_subfacets(cut.cutfacets,facet_gids)
+  STLEmbeddedDiscretization(cut.cut,cutfacets)
+end
+
+function get_ls_to_bgcell_to_inoutcut(cut::STLEmbeddedDiscretization)
+  get_ls_to_bgcell_to_inoutcut(cut.cut)
+end
+
+function get_ls_to_bgfacet_to_inoutcut(cut::STLEmbeddedDiscretization)
+  get_ls_to_bgfacet_to_inoutcut(cut.cutfacets)
+end

src/STLCutters.jl

@@ -6,6 +6,7 @@ import Downloads
 using Gridap
 using Gridap.Arrays
 using Gridap.Geometry
+using Gridap.Fields
 using Gridap.Helpers
 using Gridap.ReferenceFEs
 using GridapEmbedded
@@ -16,6 +17,16 @@ using LinearAlgebra
 using FillArrays
 using FileIO
 using ProgressMeter
+using GridapDistributed
+using PartitionedArrays
+
+using Gridap.Geometry: get_face_to_parent_face
+using Gridap.Geometry: get_cell_to_parent_cell
+using GridapEmbedded.Distributed: consistent_bgcell_to_inoutcut!
+using GridapEmbedded.Distributed: consistent_bgfacet_to_inoutcut!
+using GridapEmbedded.Distributed: DistributedEmbeddedDiscretization
+using GridapDistributed: DistributedDiscreteModel
+using GridapDistributed: remove_ghost_cells
 
 import Gridap: writevtk
 import Gridap.ReferenceFEs: get_polytope
@@ -55,6 +66,11 @@ import GridapEmbedded.CSG: get_tree
 import GridapEmbedded.CSG: compatible_geometries
 import GridapEmbedded.CSG: similar_geometry
 import GridapEmbedded.AgFEM: aggregate
+import GridapEmbedded.Distributed: change_bgmodel
+import GridapEmbedded.Distributed: get_ls_to_bgcell_to_inoutcut
+import GridapEmbedded.Distributed: get_ls_to_bgfacet_to_inoutcut
+import GridapEmbedded.Distributed: remove_ghost_subfacets
+
 
 import Base: split
 

src/STLs.jl

@@ -939,8 +939,11 @@ function compute_cell_to_facets(model_a::DiscreteModel,grid_b)
   compute_cell_to_facets(grid_a,grid_b)
 end
 
+function compute_cell_to_facets(
+  grid::CartesianGrid,
+  stl,
+  cell_mask=Trues(num_cells(grid)))
 
-function compute_cell_to_facets(grid::CartesianGrid,stl)
   CELL_EXPANSION_FACTOR = 1e-3
   desc = get_cartesian_descriptor(grid)
   @assert length(get_reffes(grid)) == 1
@@ -962,6 +965,7 @@ function compute_cell_to_facets(grid::CartesianGrid,stl)
     pmin,pmax = get_bounding_box(f)
     for cid in get_cells_around(desc,pmin,pmax)
       cell = LinearIndices(desc.partition)[cid.I...]
+      cell_mask[cell] || continue
       nodes = getindex!(nc,cell_to_nodes,cell)
       _pmin = coords[nodes[1]]
       _pmax = coords[nodes[end]]
@@ -978,6 +982,15 @@ function compute_cell_to_facets(grid::CartesianGrid,stl)
   assemble_threaded_sparse_map(thread_to_cells,thread_to_stl_facets,ncells)
 end
 
+function compute_cell_to_facets(grid::GridPortion,stl)
+  pgrid = grid.parent
+  pcell_mask = falses(num_cells(pgrid))
+  pcell_mask[grid.cell_to_parent_cell] .= true
+  pcell_to_facets = compute_cell_to_facets(pgrid,stl,pcell_mask)
+  map(Reindex(pcell_to_facets),grid.cell_to_parent_cell)
+end
+
+
 function compute_cell_to_facets(a::UnstructuredGrid,b)
   tmp = cartesian_bounding_model(a)
   tmp_to_a = compute_cell_to_facets(tmp,a)

src/SubTriangulations.jl

@@ -281,6 +281,10 @@ function max_length(model::CartesianDiscreteModel)
   float(get_cartesian_descriptor(model).sizes[1])
 end
 
+function max_length(model::DiscreteModelPortion)
+  max_length(model.parent_model)
+end
+
 function delete_small_subfaces!(bgmodel,X,T,p::Polytope{D},arrays...) where D
   h = max_length(bgmodel)
   c = array_cache(T)