20 #include <stk_util/parallel/ParallelComm.hpp> 21 #include <stk_util/parallel/ParallelReduce.hpp> 23 #include <stk_mesh/base/BulkData.hpp> 24 #include <stk_mesh/base/MetaData.hpp> 25 #include <stk_mesh/base/Entity.hpp> 26 #include <stk_mesh/base/EntityComm.hpp> 27 #include <stk_mesh/base/Trace.hpp> 38 void insert_closure_ghost( Entity *
const entity ,
39 const unsigned proc_local ,
40 std::set<Entity*,EntityLess> & work_list )
42 if ( ! in_owned_closure( *entity , proc_local ) ) {
46 std::pair< std::set<Entity*,EntityLess>::iterator ,
bool >
47 result = work_list.insert( entity );
49 if ( result.second ) {
55 for ( PairIterRelation
56 irel = entity->
relations() ; ! irel.empty() ; ++irel ) {
58 insert_closure_ghost( irel->entity() , proc_local ,work_list );
67 void insert_transitive_ghost( Entity *
const entity ,
68 const unsigned proc_local ,
69 std::set<Entity*,EntityLess> & work_list )
71 insert_closure_ghost( entity , proc_local , work_list );
80 for ( PairIterRelation rel = entity->
relations(); ! rel.empty() ; ++rel ) {
82 insert_transitive_ghost( rel->entity() , proc_local , work_list );
92 void insert_closure_send(
94 std::set<EntityProc,EntityLess> & send_list )
96 ThrowRequireMsg( send_entry.first->log_query() != EntityLogDeleted,
97 "Cannot send destroyed entity " << print_entity_key(send_entry.first));
99 std::pair< std::set<EntityProc,EntityLess>::iterator ,
bool >
100 result = send_list.insert( send_entry );
102 if ( result.second ) {
105 const unsigned erank = send_entry.first->entity_rank();
106 PairIterRelation irel = send_entry.first->relations();
109 for ( ; ! irel.empty() ; ++irel ) {
110 if ( irel->entity_rank() < erank ) {
111 const EntityProc rel_send_entry( irel->entity(), send_entry.second );
113 insert_closure_send( rel_send_entry , send_list );
121 bool member_of_owned_closure(
const Entity & e ,
const unsigned p_rank )
123 bool result = p_rank == e.owner_rank();
128 for ( PairIterRelation
129 irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
131 p_rank == irel->entity()->owner_rank();
135 for ( PairIterRelation
136 irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
138 member_of_owned_closure( * irel->entity() , p_rank );
148 void clean_and_verify_parallel_change(
149 const BulkData & mesh ,
150 std::vector<EntityProc> & local_change )
152 const MetaData & meta = MetaData::get(mesh);
153 const unsigned p_rank = mesh.parallel_rank();
154 const unsigned p_size = mesh.parallel_size();
157 size_t error_count = 0 ;
159 std::ostringstream error_msg ;
163 std::vector<EntityProc>::iterator i = local_change.begin() ,
164 j = local_change.end() ;
165 std::sort( i , j , EntityLess() );
166 i = std::unique( i , j );
167 local_change.erase( i , j );
170 for ( std::vector<EntityProc>::iterator
171 i = local_change.begin() ; i != local_change.end() ; ++i ) {
172 std::vector<EntityProc>::iterator next = i+1 ;
173 Entity *
const entity = i->first ;
174 const unsigned new_owner = i->second ;
182 const bool bad_null = NULL == entity ;
185 const bool bad_marked_deleted = ! bad_null && EntityLogDeleted == entity->
log_query();
188 const bool bad_process_not_entity_owner = ! bad_null && entity->
owner_rank() != p_rank ;
191 const bool bad_new_owner_does_not_exist = p_size <= new_owner ;
194 const bool bad_inconsistent_change = ! bad_null && next != local_change.end() && entity == next->first ;
197 bad_process_not_entity_owner ||
198 bad_new_owner_does_not_exist ||
199 bad_inconsistent_change ||
204 error_msg <<
" P" << p_rank <<
": " ;
205 if ( bad_null ) { error_msg <<
" NULL ENTITY" ; }
206 else { print_entity_key( error_msg , meta , entity->
key() ); }
207 if ( bad_marked_deleted ) { error_msg <<
" HAS_BEEN_DELETED" ; }
208 if ( bad_process_not_entity_owner ) { error_msg <<
" NOT_CURRENT_OWNER" ; }
209 if ( bad_new_owner_does_not_exist ) {
210 error_msg <<
" BAD_NEW_OWNER( " << new_owner <<
" )" ;
212 if ( bad_inconsistent_change ) {
213 error_msg <<
" CONFLICTING_NEW_OWNER( " << new_owner ;
214 error_msg <<
" != " << next->second <<
" )" ;
216 error_msg << std::endl ;
218 else if ( new_owner == p_rank ) {
225 all_reduce( p_comm , ReduceSum<1>( & error_count ) );
230 ThrowErrorMsg(
"Bad change ownership directives\n");
235 std::vector<EntityProc>::iterator i = local_change.begin(),
236 j = local_change.end();
238 local_change.erase( i , j );
248 void generate_parallel_change(
const BulkData & mesh ,
249 const std::vector<EntityProc> & local_change ,
250 std::vector<EntityProc> & shared_change ,
251 std::vector<EntityProc> & ghosted_change )
253 const unsigned p_size = mesh.parallel_size();
255 CommAll comm( mesh.parallel() );
257 std::vector<unsigned> procs ;
261 for (
int phase = 0; phase < 2; ++phase) {
262 for ( std::vector<EntityProc>::const_iterator
263 ip = local_change.begin() ; ip != local_change.end() ; ++ip ) {
264 Entity & entity = * ip->first ;
265 unsigned new_owner = ip->second;
266 comm_procs( entity , procs );
267 for ( std::vector<unsigned>::iterator
268 j = procs.begin() ; j != procs.end() ; ++j )
270 comm.send_buffer( *j )
271 .pack<EntityKey>( entity.
key() )
272 .pack<unsigned>( new_owner );
276 comm.allocate_buffers( p_size / 4 , 0 );
285 for (
unsigned ip = 0 ; ip < p_size ; ++ip ) {
286 CommBuffer & buf = comm.recv_buffer( ip );
287 while ( buf.remaining() ) {
290 buf.unpack<EntityKey>( key )
291 .unpack<unsigned>( entry.second );
293 entry.first = mesh.get_entity( key );
295 if ( in_receive_ghost( * entry.first ) ) {
296 ghosted_change.push_back( entry );
299 shared_change.push_back( entry );
304 std::sort( shared_change.begin() , shared_change.end() , EntityLess() );
305 std::sort( ghosted_change.begin() , ghosted_change.end() , EntityLess() );
315 Trace_(
"stk_classic::mesh::BulkData::change_entity_owner");
316 DiagIf(LOG_ENTITY,
"arg_change: " << arg_change);
318 const MetaData & meta = m_mesh_meta_data ;
319 const unsigned p_rank = m_parallel_rank ;
320 const unsigned p_size = m_parallel_size ;
328 std::vector<EntityProc> local_change( arg_change );
331 clean_and_verify_parallel_change( *
this , local_change );
339 std::vector<EntityProc> ghosted_change ;
340 std::vector<EntityProc> shared_change ;
342 generate_parallel_change( *
this , local_change ,
343 shared_change , ghosted_change );
351 typedef std::set<EntityProc,EntityLess> EntityProcSet;
352 typedef std::set<Entity*,EntityLess> EntitySet;
355 EntityProcSet send_closure ;
356 for ( std::vector<EntityProc>::iterator
357 i = local_change.begin() ; i != local_change.end() ; ++i ) {
358 insert_closure_send( *i , send_closure );
366 EntitySet modified_ghosts ;
368 for ( std::vector<EntityProc>::const_iterator
369 i = ghosted_change.begin() ; i != ghosted_change.end() ; ++i ) {
370 insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
373 for ( std::vector<EntityProc>::const_iterator
374 i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
375 insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
378 for ( EntityProcSet::iterator
379 i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
380 insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
384 ghosted_change.clear();
386 std::vector<EntityProc> empty_add ;
387 std::vector<Entity*> remove_modified_ghosts( modified_ghosts.begin() ,
388 modified_ghosts.end() );
391 for ( std::vector<Ghosting*>::iterator
392 ig = m_ghosting.begin() + 1; ig != m_ghosting.end() ; ++ig ) {
394 internal_change_ghosting( **ig , empty_add , remove_modified_ghosts );
407 for ( std::vector<EntityProc>::iterator
408 i = local_change.begin() ; i != local_change.end() ; ++i ) {
411 Entity* entity = i->first;
415 m_entity_repo.set_entity_owner_rank( *entity, i->second );
418 for ( std::vector<EntityProc>::iterator
419 i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
420 Entity* entity = i->first;
421 m_entity_repo.set_entity_owner_rank( *entity, i->second);
422 if ( p_rank == i->second ) {
431 std::ostringstream error_msg ;
432 int error_count = 0 ;
434 CommAll comm( p_comm );
436 for ( std::set<EntityProc,EntityLess>::iterator
437 i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
438 CommBuffer & buffer = comm.send_buffer( i->second );
439 Entity & entity = * i->first ;
440 pack_entity_info( buffer , entity );
441 pack_field_values( buffer , entity );
444 comm.allocate_buffers( p_size / 4 );
446 for ( std::set<EntityProc,EntityLess>::iterator
447 i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
448 CommBuffer & buffer = comm.send_buffer( i->second );
449 Entity & entity = * i->first ;
450 pack_entity_info( buffer , entity );
451 pack_field_values( buffer , entity );
456 for (
unsigned p = 0 ; p < p_size ; ++p ) {
457 CommBuffer & buf = comm.recv_buffer(p);
458 while ( buf.remaining() ) {
460 std::vector<Relation> relations ;
462 unsigned owner = ~0u ;
464 unpack_entity_info( buf, *
this, key, owner, parts, relations );
471 if ( owner == p_rank ) {
480 std::pair<Entity*,bool> result =
481 m_entity_repo.internal_create_entity( key );
483 Entity* entity = result.first;
485 m_entity_repo.log_created_parallel_copy( *entity );
489 m_entity_repo.set_entity_owner_rank( *entity, owner);
491 internal_change_entity_parts( *entity , parts ,
PartVector() );
495 if ( ! unpack_field_values( buf , *entity , error_msg ) ) {
501 all_reduce( p_comm , ReduceSum<1>( & error_count ) );
502 ThrowErrorMsgIf( error_count, error_msg.str() );
512 for ( std::set<EntityProc,EntityLess>::iterator
513 i = send_closure.end() ; i != send_closure.begin() ; ) {
515 Entity * e = (--i)->first ;
522 if ( ! member_of_owned_closure( *e , p_rank ) ) {
524 "Failed to destroy entity " << print_entity_key(e) );
530 send_closure.clear();
void declare_relation(Entity &e_from, Entity &e_to, const RelationIdentifier local_id)
Declare a relation and its converse between entities in the same mesh.
void remove(PartVector &v, Part &part)
Remove a part from a properly ordered collection of parts.
const EntityKey & key() const
The globally unique key ( entity type + identifier ) of this entity.
Integer type for the entity keys, which is an encoding of the entity type and entity identifier...
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
EntityModificationLog log_query() const
Query the current state of the entity log.
void change_entity_parts(Entity &entity, const PartVector &add_parts, const PartVector &remove_parts=PartVector())
Change the parallel-locally-owned entity's part membership by adding and/or removing parts...
PairIterRelation relations() const
All Entity relations for which this entity is a member. The relations are ordered from lowest entity-...
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
void all_write_string(ParallelMachine arg_comm, std::ostream &arg_root_os, const std::string &arg_msg)
Write string from any or all processors to the ostream on the root processor.
void change_entity_owner(const std::vector< EntityProc > &arg_change)
Give away ownership of entities to other parallel processes.
EntityRank entity_rank() const
The rank of this entity.
std::vector< Part *> PartVector
Collections of parts are frequently maintained as a vector of Part pointers.
EntityRank entity_rank(const EntityKey &key)
Given an entity key, return an entity type (rank).
bool destroy_entity(Entity *&entity)
Request the destruction an entity on the local process.
unsigned owner_rank() const
Parallel processor rank of the processor which owns this entity.
bool insert(PartVector &v, Part &part)
Insert a part into a properly ordered collection of parts. Returns true if this is a new insertion...