58 size_indices_from_(0),
74 total_recv_length_(0),
93 size_indices_to_(Distributor.size_indices_to_),
97 size_indices_from_(Distributor.size_indices_from_),
107 indices_from_ptr_(0),
108 nrecvs_(Distributor.nrecvs_),
109 nsends_(Distributor.nsends_),
110 nexports_(Distributor.nexports_),
111 self_msg_(Distributor.self_msg_),
112 max_send_length_(Distributor.max_send_length_),
113 total_recv_length_(Distributor.total_recv_length_),
114 tag_(Distributor.tag_),
115 epComm_(Distributor.epComm_),
116 comm_(Distributor.comm_),
119 no_delete_(Distributor.no_delete_),
122 comm_plan_reverse_(0)
223 const int * ExportPIDs,
229 MPI_Comm_rank(
comm_, &my_proc );
232 MPI_Comm_size(
comm_, &nprocs );
254 const int * RemoteGIDs,
255 const int * RemotePIDs,
262 MPI_Comm_rank(
comm_, &my_proc );
265 MPI_Comm_size(
comm_, &nprocs );
268 ExportGIDs, ExportPIDs, my_proc) );
270 int testNumRemoteIDs;
272 Deterministic, testNumRemoteIDs ) );
282 #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES 284 const long long * RemoteGIDs,
285 const int * RemotePIDs,
288 long long *& ExportGIDs,
292 MPI_Comm_rank(
comm_, &my_proc );
295 MPI_Comm_size(
comm_, &nprocs );
298 ExportGIDs, ExportPIDs, my_proc) );
300 int testNumRemoteIDs;
302 Deterministic, testNumRemoteIDs ) );
315 const int * ExportPIDs,
316 const int & NumRemoteIDs,
317 const int * RemoteGIDs,
318 const int * RemotePIDs,
326 MPI_Comm_rank(
comm_, &my_proc );
328 MPI_Comm_size(
comm_, &nprocs );
338 #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES 340 const int * ExportPIDs,
341 const int & NumRemoteIDs,
342 const long long * RemoteGIDs,
343 const int * RemotePIDs,
351 MPI_Comm_rank(
comm_, &my_proc );
353 MPI_Comm_size(
comm_, &nprocs );
372 const int & NumExportIDs,
373 const int * ExportPIDs)
383 int * starts =
new int[ nprocs + 1 ];
384 for( i = 0; i < nprocs; i++ )
388 bool no_send_buff =
true;
389 int numDeadIndices = 0;
391 for( i = 0; i < NumExportIDs; i++ )
393 if( no_send_buff && i && (ExportPIDs[i] < ExportPIDs[i-1]) )
394 no_send_buff =
false;
395 if( ExportPIDs[i] >= 0 )
397 ++starts[ ExportPIDs[i] ];
400 else numDeadIndices++;
403 self_msg_ = ( starts[my_proc] != 0 ) ? 1 : 0;
409 for( i = 0; i < nprocs; ++i )
419 int index = numDeadIndices;
424 proc = ExportPIDs[index];
426 index += starts[proc];
444 if( starts[0] != 0 )
nsends_ = 1;
446 for( i = 1; i < nprocs; i++ )
448 if( starts[i] != 0 ) ++
nsends_;
449 starts[i] += starts[i-1];
452 for( i = nprocs-1; i != 0; i-- )
453 starts[i] = starts[i-1];
462 for( i = 0; i < NumExportIDs; i++ )
463 if( ExportPIDs[i] >= 0 )
466 ++starts[ ExportPIDs[i] ];
471 for( i = nprocs-1; i != 0; i-- )
472 starts[i] = starts[i-1];
474 starts[nprocs] = nactive;
485 for( i = 0; i < nprocs; i++ )
486 if( starts[i+1] != starts[i] )
506 const int * RemotePIDs)
512 std::vector<int> recv_list;
516 for(i=0; i<NumRemoteIDs; i++) {
517 if(RemotePIDs[i]>last_pid) {
518 recv_list.push_back(RemotePIDs[i]);
519 last_pid = RemotePIDs[i];
521 else if (RemotePIDs[i]<last_pid)
522 throw std::runtime_error(
"Epetra_MpiDistributor::CreateRecvStructures_ expected RemotePIDs to be in sorted order");
538 for( ; j<NumRemoteIDs && RemotePIDs[jlast]==RemotePIDs[j] ; j++){;}
556 int * msg_count =
new int[ nprocs ];
557 int * counts =
new int[ nprocs ];
562 for( i = 0; i < nprocs; i++ )
571 #if defined(REDUCE_SCATTER_BUG) 573 MPI_Reduce(msg_count, counts, nprocs, MPI_INT, MPI_SUM, 0,
comm_);
574 MPI_Scatter(counts, 1, MPI_INT, &
nrecvs_, 1, MPI_INT, 0,
comm_);
576 MPI_Reduce_scatter( msg_count, &
nrecvs_, counts, MPI_INT, MPI_SUM,
comm_ );
591 #ifndef NEW_COMM_PATTERN 609 MPI_Barrier(
comm_ );
621 MPI_Barrier(
comm_ );
669 MPI_Barrier(
comm_ );
678 template<
typename id_type>
680 const id_type *& import_ids,
681 const int *& import_procs,
683 id_type *& export_ids,
691 int * import_objs = 0;
692 char * c_export_objs = 0;
693 const int pack_size = (1 +
sizeof(id_type)/
sizeof(
int));
695 if( num_imports > 0 )
697 proc_list =
new int[ num_imports ];
698 import_objs =
new int[ num_imports * pack_size];
700 for( i = 0; i < num_imports; i++ )
702 proc_list[i] = import_procs[i];
704 *(id_type*)(import_objs + pack_size*i) = import_ids[i];
705 *(import_objs + pack_size*i + (pack_size-1)) = my_proc;
710 true, num_exports) );
711 if( num_exports > 0 )
714 export_ids =
new id_type[ num_exports ];
715 export_procs =
new int[ num_exports ];
723 int len_c_export_objs = 0;
725 pack_size * (
int)
sizeof(
int ),
728 int * export_objs =
reinterpret_cast<int *
>(c_export_objs);
730 for( i = 0; i < num_exports; i++ ) {
731 export_ids[i] = *(id_type*)(export_objs + pack_size*i);
732 export_procs[i] = *(export_objs + pack_size*i + (pack_size-1));
735 if( proc_list != 0 )
delete [] proc_list;
736 if( import_objs != 0 )
delete [] import_objs;
737 if( len_c_export_objs != 0 )
delete [] c_export_objs;
746 int & len_import_objs,
747 char *& import_objs )
764 int & len_import_objs,
765 char *& import_objs )
768 len_import_objs, import_objs) );
784 int & len_import_objs,
785 char *& import_objs )
790 int self_recv_address = 0;
792 MPI_Comm_rank(
comm_, &my_proc );
796 if( import_objs!=0 ) {
delete [] import_objs; import_objs = 0;}
798 if (len_import_objs>0) import_objs =
new char[len_import_objs];
799 for( i=0; i<len_import_objs; ++i ) import_objs[i]=0;
809 MPI_Irecv( &(import_objs[j]),
817 self_recv_address = j;
822 #ifndef EPETRA_NO_READY_SEND_IN_DO_POSTS 832 MPI_Barrier(
comm_ );
833 #endif // EPETRA_NO_READY_SEND_IN_DO_POSTS 839 while( proc_index < nblocks &&
procs_to_[proc_index] < my_proc )
841 if( proc_index == nblocks ) proc_index = 0;
843 int self_num = 0, self_index = 0;
848 for( i = 0; i < nblocks; ++i )
851 if( p > (nblocks-1) ) p -= nblocks;
855 #ifndef EPETRA_NO_READY_SEND_IN_DO_POSTS 856 MPI_Rsend( &export_objs[
starts_to_[p]*obj_size],
863 MPI_Send( &export_objs[
starts_to_[p]*obj_size],
869 #endif // EPETRA_NO_READY_SEND_IN_DO_POSTS 877 memcpy( &import_objs[self_recv_address],
891 for( i = 0; i < nblocks; i++ )
894 if( p > (nblocks-1) ) p -= nblocks;
907 #ifndef EPETRA_NO_READY_SEND_IN_DO_POSTS 919 #endif // EPETRA_NO_READY_SEND_IN_DO_POSTS 931 memcpy( &(import_objs[self_recv_address]),
935 self_recv_address += obj_size;
952 int & len_import_objs,
953 char *& import_objs )
989 MPI_Comm_rank (
comm_, &my_proc);
991 MPI_Comm_size(
comm_, &nprocs );
998 match = match && (
sizes_[i]==sizes[i]);
999 int matched = match?1:0;
1000 int match_count = 0;
1001 MPI_Allreduce( &matched, &match_count, 1, MPI_INT, MPI_SUM,
comm_ );
1002 if( match_count == nprocs )
1052 if (index!=0) {
delete [] index; index = 0;}
1053 if (sort_val!=0) {
delete [] sort_val; sort_val = 0;}
1084 if (offset!=0) {
delete [] offset; offset = 0;}
1088 int self_index_to = -1;
1092 #ifndef EPETRA_NEW_COMM_PATTERN 1135 MPI_Barrier(
comm_ );
1170 int & len_import_objs,
1171 char *& import_objs )
1174 len_import_objs, import_objs) );
1184 int & len_import_objs,
1185 char *& import_objs )
1188 len_import_objs, import_objs) );
1198 int & len_import_objs,
1199 char *& import_objs )
1206 MPI_Barrier(
comm_ );
1211 int self_recv_address = 0;
1213 MPI_Comm_rank(
comm_, &my_proc );
1217 if( import_objs!=0 ) {
delete [] import_objs; import_objs = 0;}
1219 if (len_import_objs>0) import_objs =
new char[len_import_objs];
1237 MPI_Barrier(
comm_ );
1243 while( proc_index < nblocks &&
procs_to_[proc_index] < my_proc )
1245 if( proc_index == nblocks ) proc_index = 0;
1252 for( i = 0; i < nblocks; ++i )
1255 if( p > (nblocks-1) ) p -= nblocks;
1269 memcpy( &import_objs[self_recv_address],
1287 for( i=0; i<nblocks; ++i )
1290 if( p > (nblocks-1) ) p -= nblocks;
1318 memcpy( &import_objs[self_recv_address],
1319 &export_objs[jj*obj_size],
1334 int & len_import_objs,
1335 char *& import_objs )
1354 int myRank = 0, numProcs = 1;
1355 MPI_Comm_rank (
comm_, &myRank);
1356 MPI_Comm_size (
comm_, &numProcs);
1359 os <<
"Epetra_MpiDistributor (implements Epetra_Distributor)" << std::endl;
1365 for (
int p = 0; p < numProcs; ++p) {
1367 os <<
"[Node " << p <<
" of " << numProcs <<
"]" << std::endl;
1368 os <<
" selfMessage: " <<
self_msg_ << std::endl;
1369 os <<
" numSends: " <<
nsends_ << std::endl;
1371 os <<
" imagesTo: [";
1372 for (
int i = 0; i <
nsends_; ++i) {
1378 os <<
"]" << std::endl;
1380 os <<
" lengthsTo: [";
1381 for (
int i = 0; i <
nsends_; ++i) {
1387 os <<
"]" << std::endl;
1391 os <<
" startsTo: ";
1393 os <<
"(NULL)" << std::endl;
1396 for (
int i = 0; i <
nsends_; ++i) {
1402 os <<
"]" << std::endl;
1405 os <<
" indicesTo: ";
1407 os <<
"(NULL)" << std::endl;
1411 for (
int i = 0; i <
nsends_; ++i) {
1417 os <<
"]" << std::endl;
1420 os <<
" numReceives: " <<
nrecvs_ << std::endl;
1423 os <<
" lengthsFrom: [";
1424 for (
int i = 0; i <
nrecvs_; ++i) {
1430 os <<
"]" << std::endl;
1432 os <<
" startsFrom: [";
1433 for (
int i = 0; i <
nrecvs_; ++i) {
1439 os <<
"]" << std::endl;
1441 os <<
" imagesFrom: [";
1442 for (
int i = 0; i <
nrecvs_; ++i) {
1448 os <<
"]" << std::endl;
1470 MPI_Barrier (
comm_);
1471 MPI_Barrier (
comm_);
1472 MPI_Barrier (
comm_);
1489 if (nvals <= 1)
return 0;
1495 for (i = 0; i < nvals; i++)
1496 if (
n < vals_sort[i])
n = vals_sort[i];
1497 int *pos =
new int [
n+2];
1498 for (i = 0; i <
n+2; i++) pos[i] = 0;
1501 int *copy_sort =
new int [nvals];
1502 int *copy_other =
new int [nvals];
1503 for (i = 0; i < nvals; i++)
1505 copy_sort[i] = vals_sort[i];
1506 copy_other[i] = vals_other[i];
1511 for (i = 0; i < nvals; i++) p[copy_sort[i]]++;
1514 for (i = 1; i <
n; i++) p[i] += p[i-1];
1518 for (i = 0; i < nvals; i++)
1520 vals_sort [p[copy_sort [i]]] = copy_sort[i];
1521 vals_other [p[copy_sort [i]]++] = copy_other[i];
1524 delete [] copy_sort;
1525 delete [] copy_other;
1536 bool throw_error =
true;
1538 throw ReportError(
"Epetra_MpiDistributor::operator= not supported.",-1);
1549 MPI_Comm_rank(
comm_, &my_proc );
1552 int total_send_length = 0;
1556 int max_recv_length = 0;
1557 for( i = 0; i <
nrecvs_; i++ )
int CreateSendStructures_(int my_proc, int nprocs, const int &NumExportIDs, const int *ExportPIDs)
int ComputeSends_(int num_imports, const id_type *&import_ids, const int *&import_procs, int &num_exports, id_type *&export_ids, int *&export_procs, int my_proc)
int ComputeRecvs_(int my_proc, int nprocs)
Epetra_MpiDistributor & operator=(const Epetra_MpiDistributor &src)
int DoPosts(char *export_objs, int obj_size, int &len_import_objs, char *&import_objs)
Post buffer of export objects (can do other local work before executing Waits)
void CreateReverseDistributor()
MPI_Comm Comm() const
Extract MPI Communicator from a Epetra_MpiComm object.
Epetra_Distributor: The Epetra Gather/Scatter Setup Base Class.
int CreateFromRecvs(const int &NumRemoteIDs, const int *RemoteGIDs, const int *RemotePIDs, bool Deterministic, int &NumExportIDs, int *&ExportGIDs, int *&ExportPIDs)
Create a communication plan from receive list.
int DoReverseWaits()
Wait on a reverse set of posts.
#define EPETRA_CHK_ERR(a)
int CreateFromSendsAndRecvs(const int &NumExportIDs, const int *ExportPIDs, const int &NumRemoteIDs, const int *RemoteGIDs, const int *RemotePIDs, bool Deterministic)
Create a communication plan from send list and a recv list.
MPI implementation of Epetra_Distributor.
virtual int ReportError(const std::string Message, int ErrorCode) const
Error reporting method.
Epetra_MpiComm: The Epetra MPI Communication Class.
int CreateRecvStructures_(const int &NumRemoteIDs, const int *RemotePIDs)
Epetra_Object: The base Epetra class.
MPI_Comm GetMpiComm() const
Get the MPI Communicator (identical to Comm() method; used when we know we are MPI.
virtual ~Epetra_MpiDistributor()
Destructor (declared virtual for memory safety).
int CreateFromSends(const int &NumExportIDs, const int *ExportPIDs, bool Deterministic, int &NumRemoteIDs)
Create a communication plan from send list.
int DoWaits()
Wait on a set of posts.
int DoReversePosts(char *export_objs, int obj_size, int &len_import_objs, char *&import_objs)
Do reverse post of buffer of export objects (can do other local work before executing Waits) ...
Epetra_Distributor * ReverseClone()
Create and extract the reverse version of the distributor.
int NumSends() const
The number of procs to which we will send data.
Epetra_MpiDistributor * comm_plan_reverse_
int Sort_ints_(int *vals, int *other, int nvals)
int GetMpiTag() const
Acquire an MPI tag from the Epetra range of 24050-24099, increment tag.
int Do(char *export_objs, int obj_size, int &len_import_objs, char *&import_objs)
Execute plan on buffer of export objects in a single step.
Epetra_MpiDistributor(const Epetra_MpiComm &Comm)
Default constructor.
int DoReverse(char *export_objs, int obj_size, int &len_import_objs, char *&import_objs)
Execute reverse of plan on buffer of export objects in a single step.
const Epetra_MpiComm * epComm_
int NumReceives() const
The number of procs from which we will receive data.
void Print(std::ostream &os) const