42 #ifndef TEUCHOS_MPI_COMM_HPP 43 #define TEUCHOS_MPI_COMM_HPP 53 #ifdef HAVE_TEUCHOS_MPI 71 #ifdef TEUCHOS_MPI_COMM_DUMP 79 mpiErrorCodeToString (
const int err);
95 void safeCommFree (MPI_Comm* comm);
101 int setCommErrhandler (MPI_Comm comm, MPI_Errhandler handler);
105 #ifdef TEUCHOS_MPI_COMM_DUMP 106 template<
typename Ordinal,
typename T>
108 const std::string &funcName,
const std::string &buffName
109 ,
const Ordinal bytes,
const T buff[]
116 <<
"\n" << funcName <<
"::" << buffName <<
":\n";
118 for( Ordinal i = 0; i < bytes; ++i ) {
119 *out << buffName <<
"[" << i <<
"] = '" << buff[i] <<
"'\n";
123 #endif // TEUCHOS_MPI_COMM_DUMP 136 template<
class OrdinalType>
137 class MpiCommStatus :
public CommStatus<OrdinalType> {
139 MpiCommStatus (MPI_Status status) : status_ (status) {}
142 virtual ~MpiCommStatus() {}
145 OrdinalType getSourceRank () {
return status_.MPI_SOURCE; }
148 OrdinalType getTag () {
return status_.MPI_TAG; }
151 OrdinalType getError () {
return status_.MPI_ERROR; }
164 template<
class OrdinalType>
165 inline RCP<MpiCommStatus<OrdinalType> >
166 mpiCommStatus (MPI_Status rawMpiStatus)
168 return rcp (
new MpiCommStatus<OrdinalType> (rawMpiStatus));
186 template<
class OrdinalType>
187 class MpiCommRequestBase :
public CommRequest<OrdinalType> {
190 MpiCommRequestBase () :
191 rawMpiRequest_ (MPI_REQUEST_NULL)
195 MpiCommRequestBase (MPI_Request rawMpiRequest) :
196 rawMpiRequest_ (rawMpiRequest)
206 MPI_Request releaseRawMpiRequest()
208 MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
209 rawMpiRequest_ = MPI_REQUEST_NULL;
210 return tmp_rawMpiRequest;
214 bool isNull()
const {
215 return rawMpiRequest_ == MPI_REQUEST_NULL;
223 RCP<CommStatus<OrdinalType> > wait () {
224 MPI_Status rawMpiStatus;
227 const int err = MPI_Wait (&rawMpiRequest_, &rawMpiStatus);
229 err != MPI_SUCCESS, std::runtime_error,
230 "Teuchos: MPI_Wait() failed with error \"" 231 << mpiErrorCodeToString (err));
233 return mpiCommStatus<OrdinalType> (rawMpiStatus);
240 RCP<CommStatus<OrdinalType> > cancel () {
241 if (rawMpiRequest_ == MPI_REQUEST_NULL) {
245 int err = MPI_Cancel (&rawMpiRequest_);
247 err != MPI_SUCCESS, std::runtime_error,
248 "Teuchos: MPI_Cancel failed with the following error: " 249 << mpiErrorCodeToString (err));
256 err = MPI_Wait (&rawMpiRequest_, &status);
258 "Teuchos::MpiCommStatus::cancel: MPI_Wait failed with the following " 259 "error: " << mpiErrorCodeToString (err));
260 return mpiCommStatus<OrdinalType> (status);
265 virtual ~MpiCommRequestBase () {
266 if (rawMpiRequest_ != MPI_REQUEST_NULL) {
269 const int err = MPI_Cancel (&rawMpiRequest_);
270 if (err == MPI_SUCCESS) {
287 (void) MPI_Wait (&rawMpiRequest_, MPI_STATUS_IGNORE);
294 MPI_Request rawMpiRequest_;
312 template<
class OrdinalType>
313 class MpiCommRequest :
public MpiCommRequestBase<OrdinalType> {
317 MpiCommRequestBase<OrdinalType> (MPI_REQUEST_NULL),
322 MpiCommRequest (MPI_Request rawMpiRequest,
324 MpiCommRequestBase<OrdinalType> (rawMpiRequest),
325 numBytes_ (numBytesInMessage)
338 virtual ~MpiCommRequest () {}
353 template<
class OrdinalType>
354 inline RCP<MpiCommRequest<OrdinalType> >
358 return rcp (
new MpiCommRequest<OrdinalType> (rawMpiRequest, numBytes));
376 template<
typename Ordinal>
377 class MpiComm :
public Comm<Ordinal> {
402 explicit MpiComm (MPI_Comm rawMpiComm);
418 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm);
437 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
438 const int defaultTag);
456 MpiComm (
const MpiComm<Ordinal>& other);
459 RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm ()
const {
527 void setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler);
534 virtual int getRank()
const;
537 virtual int getSize()
const;
540 virtual void barrier()
const;
543 virtual void broadcast(
544 const int rootRank,
const Ordinal bytes,
char buffer[]
549 gather (
const Ordinal sendBytes,
const char sendBuffer[],
550 const Ordinal recvBytes,
char recvBuffer[],
551 const int root)
const;
553 virtual void gatherAll(
554 const Ordinal sendBytes,
const char sendBuffer[]
555 ,
const Ordinal recvBytes,
char recvBuffer[]
559 const ValueTypeReductionOp<Ordinal,char> &reductOp
560 ,
const Ordinal bytes,
const char sendBuffer[],
char globalReducts[]
564 const ValueTypeReductionOp<Ordinal,char> &reductOp
565 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
569 const Ordinal bytes,
const char sendBuffer[],
const int destRank
573 send (
const Ordinal bytes,
574 const char sendBuffer[],
576 const int tag)
const;
579 const Ordinal bytes,
const char sendBuffer[],
const int destRank
583 ssend (
const Ordinal bytes,
584 const char sendBuffer[],
586 const int tag)
const;
589 const int sourceRank,
const Ordinal bytes,
char recvBuffer[]
593 const ArrayView<const char> &sendBuffer,
599 const char sendBuffer[],
601 const int tag)
const;
603 virtual RCP<CommRequest<Ordinal> >
isend(
604 const ArrayView<const char> &sendBuffer,
608 virtual RCP<CommRequest<Ordinal> >
609 isend (
const ArrayView<const char> &sendBuffer,
611 const int tag)
const;
613 virtual RCP<CommRequest<Ordinal> >
ireceive(
614 const ArrayView<char> &Buffer,
618 virtual RCP<CommRequest<Ordinal> >
619 ireceive (
const ArrayView<char> &Buffer,
620 const int sourceRank,
621 const int tag)
const;
623 virtual void waitAll(
624 const ArrayView<RCP<CommRequest<Ordinal> > > &requests
628 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
629 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const;
631 virtual RCP<CommStatus<Ordinal> >
632 wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const;
634 virtual RCP< Comm<Ordinal> > duplicate()
const;
636 virtual RCP< Comm<Ordinal> >
split(
const int color,
const int key)
const;
638 virtual RCP< Comm<Ordinal> > createSubcommunicator(
639 const ArrayView<const int>& ranks)
const;
646 std::string description()
const;
652 static int const minTag_ = 26000;
653 static int const maxTag_ = 26099;
660 int getTag ()
const {
return tag_; }
667 void setupMembersFromComm();
668 static int tagCounter_;
677 RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
695 RCP<const OpaqueWrapper<MPI_Errhandler> > customErrorHandler_;
697 void assertRank(
const int rank,
const std::string &rankName)
const;
702 #ifdef TEUCHOS_MPI_COMM_DUMP 704 static bool show_dump;
705 #endif // TEUCHOS_MPI_COMM_DUMP 723 template<
typename Ordinal>
724 RCP<MpiComm<Ordinal> >
726 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
757 template<
typename Ordinal>
759 getRawMpiComm(
const Comm<Ordinal> &comm);
769 template<
typename Ordinal>
770 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
776 template<
typename Ordinal>
778 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm)
781 rawMpiComm.get () == NULL, std::invalid_argument,
782 "Teuchos::MpiComm constructor: The input RCP is null.");
784 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
785 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
787 rawMpiComm_ = rawMpiComm;
801 setupMembersFromComm ();
805 template<
typename Ordinal>
807 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
808 const int defaultTag)
811 rawMpiComm.get () == NULL, std::invalid_argument,
812 "Teuchos::MpiComm constructor: The input RCP is null.");
814 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
815 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
817 rawMpiComm_ = rawMpiComm;
819 int err = MPI_Comm_size (*rawMpiComm_, &size_);
821 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 822 "error \"" << mpiErrorCodeToString (err) <<
"\".");
824 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
826 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 827 "error \"" << mpiErrorCodeToString (err) <<
"\".");
832 template<
typename Ordinal>
833 MpiComm<Ordinal>::MpiComm (MPI_Comm rawMpiComm)
836 std::invalid_argument,
"Teuchos::MpiComm constructor: The given MPI_Comm " 837 "is MPI_COMM_NULL.");
841 rawMpiComm_ = opaqueWrapper<MPI_Comm> (rawMpiComm);
855 setupMembersFromComm ();
859 template<
typename Ordinal>
860 MpiComm<Ordinal>::MpiComm (
const MpiComm<Ordinal>& other) :
861 rawMpiComm_ (opaqueWrapper<MPI_Comm> (MPI_COMM_NULL))
864 RCP<const OpaqueWrapper<MPI_Comm> > origCommPtr = other.getRawMpiComm ();
866 "Teuchos::MpiComm copy constructor: " 867 "The input's getRawMpiComm() method returns null.");
868 MPI_Comm origComm = *origCommPtr;
870 "Teuchos::MpiComm copy constructor: " 871 "The input's raw MPI_Comm is MPI_COMM_NULL.");
879 rawMpiComm_ = origCommPtr;
883 const int err = MPI_Comm_dup (origComm, &newComm);
885 "Teuchos::MpiComm copy constructor: MPI_Comm_dup failed with " 886 "the following error: " << mpiErrorCodeToString (err));
888 rawMpiComm_ = opaqueWrapper (newComm, details::safeCommFree);
891 setupMembersFromComm ();
895 template<
typename Ordinal>
896 void MpiComm<Ordinal>::setupMembersFromComm ()
898 int err = MPI_Comm_size (*rawMpiComm_, &size_);
900 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 901 "error \"" << mpiErrorCodeToString (err) <<
"\".");
902 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
904 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 905 "error \"" << mpiErrorCodeToString (err) <<
"\".");
908 if (tagCounter_ > maxTag_) {
909 tagCounter_ = minTag_;
911 tag_ = tagCounter_++;
921 MPI_Bcast (&tag_, 1, MPI_INT, 0, *rawMpiComm_);
925 template<
typename Ordinal>
928 setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler)
931 const int err = details::setCommErrhandler (*getRawMpiComm (), *errHandler);
933 "Teuchos::MpiComm: Setting the MPI_Comm's error handler failed with " 934 "error \"" << mpiErrorCodeToString (err) <<
"\".");
938 customErrorHandler_ = errHandler;
945 template<
typename Ordinal>
946 int MpiComm<Ordinal>::getRank()
const 952 template<
typename Ordinal>
953 int MpiComm<Ordinal>::getSize()
const 959 template<
typename Ordinal>
960 void MpiComm<Ordinal>::barrier()
const 965 const int err = MPI_Barrier (*rawMpiComm_);
967 "Teuchos::MpiComm::barrier: MPI_Barrier failed with error \"" 968 << mpiErrorCodeToString (err) <<
"\".");
972 template<
typename Ordinal>
973 void MpiComm<Ordinal>::broadcast(
974 const int rootRank,
const Ordinal bytes,
char buffer[]
980 const int err = MPI_Bcast (buffer, bytes, MPI_CHAR, rootRank, *rawMpiComm_);
982 "Teuchos::MpiComm::broadcast: MPI_Bcast failed with error \"" 983 << mpiErrorCodeToString (err) <<
"\".");
987 template<
typename Ordinal>
988 void MpiComm<Ordinal>::gatherAll(
989 const Ordinal sendBytes,
const char sendBuffer[],
990 const Ordinal recvBytes,
char recvBuffer[]
998 MPI_Allgather (const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
999 recvBuffer, sendBytes, MPI_CHAR, *rawMpiComm_);
1004 "Teuchos::MpiComm::gatherAll: MPI_Allgather failed with error \"" 1005 << mpiErrorCodeToString (err) <<
"\".");
1009 template<
typename Ordinal>
1011 MpiComm<Ordinal>::gather (
const Ordinal sendBytes,
1012 const char sendBuffer[],
1013 const Ordinal recvBytes,
1015 const int root)
const 1023 MPI_Gather (const_cast<char *> (sendBuffer), sendBytes, MPI_CHAR,
1024 recvBuffer, sendBytes, MPI_CHAR, root, *rawMpiComm_);
1026 "Teuchos::MpiComm::gather: MPI_Gather failed with error \"" 1027 << mpiErrorCodeToString (err) <<
"\".");
1031 template<
typename Ordinal>
1034 reduceAll (
const ValueTypeReductionOp<Ordinal,char> &reductOp,
1035 const Ordinal bytes,
1036 const char sendBuffer[],
1037 char globalReducts[])
const 1040 int err = MPI_SUCCESS;
1042 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1043 MPI_Op op = Details::setMpiReductionOp (opWrap);
1052 MPI_Datatype char_block;
1053 err = MPI_Type_contiguous (bytes, MPI_CHAR, &char_block);
1055 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1056 "MPI_Type_contiguous failed with error \"" << mpiErrorCodeToString (err)
1058 err = MPI_Type_commit (&char_block);
1060 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1061 "MPI_Type_commit failed with error \"" << mpiErrorCodeToString (err)
1064 err = MPI_Allreduce (const_cast<char*> (sendBuffer), globalReducts, 1,
1065 char_block, op, *rawMpiComm_);
1066 if (err != MPI_SUCCESS) {
1071 (void) MPI_Type_free (&char_block);
1073 true, std::runtime_error,
"Teuchos::reduceAll (MPI, custom op): " 1074 "MPI_Allreduce failed with error \"" << mpiErrorCodeToString (err)
1077 err = MPI_Type_free (&char_block);
1079 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1080 "MPI_Type_free failed with error \"" << mpiErrorCodeToString (err)
1085 template<
typename Ordinal>
1087 const ValueTypeReductionOp<Ordinal,char> &reductOp
1088 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
1093 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1094 MPI_Op op = Details::setMpiReductionOp (opWrap);
1096 MPI_Scan (const_cast<char*> (sendBuffer), scanReducts, bytes, MPI_CHAR,
1099 "Teuchos::MpiComm::scan: MPI_Scan() failed with error \"" 1100 << mpiErrorCodeToString (err) <<
"\".");
1104 template<
typename Ordinal>
1107 const char sendBuffer[],
1108 const int destRank)
const 1112 #ifdef TEUCHOS_MPI_COMM_DUMP 1114 dumpBuffer<Ordinal,char>(
1115 "Teuchos::MpiComm<Ordinal>::send(...)" 1116 ,
"sendBuffer", bytes, sendBuffer
1119 #endif // TEUCHOS_MPI_COMM_DUMP 1121 const int err = MPI_Send (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1122 destRank, tag_, *rawMpiComm_);
1124 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1125 << mpiErrorCodeToString (err) <<
"\".");
1129 template<
typename Ordinal>
1132 const char sendBuffer[],
1134 const int tag)
const 1137 const int err = MPI_Send (const_cast<char*> (sendBuffer), bytes, MPI_CHAR,
1138 destRank, tag, *rawMpiComm_);
1140 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1141 << mpiErrorCodeToString (err) <<
"\".");
1145 template<
typename Ordinal>
1148 const char sendBuffer[],
1149 const int destRank)
const 1153 #ifdef TEUCHOS_MPI_COMM_DUMP 1155 dumpBuffer<Ordinal,char>(
1156 "Teuchos::MpiComm<Ordinal>::send(...)" 1157 ,
"sendBuffer", bytes, sendBuffer
1160 #endif // TEUCHOS_MPI_COMM_DUMP 1162 const int err = MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1163 destRank, tag_, *rawMpiComm_);
1165 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1166 << mpiErrorCodeToString (err) <<
"\".");
1169 template<
typename Ordinal>
1172 const char sendBuffer[],
1174 const int tag)
const 1178 MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1179 destRank, tag, *rawMpiComm_);
1181 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1182 << mpiErrorCodeToString (err) <<
"\".");
1185 template<
typename Ordinal>
1187 const ArrayView<const char> &sendBuffer,
1193 #ifdef TEUCHOS_MPI_COMM_DUMP 1195 dumpBuffer<Ordinal,char>(
1196 "Teuchos::MpiComm<Ordinal>::readySend(...)" 1197 ,
"sendBuffer", bytes, sendBuffer
1200 #endif // TEUCHOS_MPI_COMM_DUMP 1203 MPI_Rsend (const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(),
1204 MPI_CHAR, destRank, tag_, *rawMpiComm_);
1206 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1207 << mpiErrorCodeToString (err) <<
"\".");
1211 template<
typename Ordinal>
1214 const char sendBuffer[],
1216 const int tag)
const 1220 MPI_Rsend (const_cast<char*> (sendBuffer), bytes,
1221 MPI_CHAR, destRank, tag, *rawMpiComm_);
1223 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1224 << mpiErrorCodeToString (err) <<
"\".");
1228 template<
typename Ordinal>
1230 MpiComm<Ordinal>::receive (
const int sourceRank,
1231 const Ordinal bytes,
1232 char recvBuffer[])
const 1239 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1242 const int err = MPI_Recv (recvBuffer, bytes, MPI_CHAR, theSrcRank, tag_,
1243 *rawMpiComm_, &status);
1245 "Teuchos::MpiComm::receive: MPI_Recv() failed with error \"" 1246 << mpiErrorCodeToString (err) <<
"\".");
1248 #ifdef TEUCHOS_MPI_COMM_DUMP 1250 dumpBuffer<Ordinal,char> (
"Teuchos::MpiComm<Ordinal>::receive(...)",
1251 "recvBuffer", bytes, recvBuffer);
1253 #endif // TEUCHOS_MPI_COMM_DUMP 1256 return status.MPI_SOURCE;
1260 template<
typename Ordinal>
1261 RCP<CommRequest<Ordinal> >
1263 const int destRank)
const 1268 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1270 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1271 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1272 destRank, tag_, *rawMpiComm_, &rawMpiRequest);
1274 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1275 << mpiErrorCodeToString (err) <<
"\".");
1277 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1281 template<
typename Ordinal>
1282 RCP<CommRequest<Ordinal> >
1284 isend (
const ArrayView<const char> &sendBuffer,
1286 const int tag)
const 1291 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1293 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1294 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1295 destRank, tag, *rawMpiComm_, &rawMpiRequest);
1297 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1298 << mpiErrorCodeToString (err) <<
"\".");
1300 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1304 template<
typename Ordinal>
1305 RCP<CommRequest<Ordinal> >
1307 const int sourceRank)
const 1314 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1316 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1318 MPI_Irecv (const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(),
1319 MPI_CHAR, theSrcRank, tag_, *rawMpiComm_, &rawMpiRequest);
1321 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1322 << mpiErrorCodeToString (err) <<
"\".");
1324 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size());
1327 template<
typename Ordinal>
1328 RCP<CommRequest<Ordinal> >
1330 const int sourceRank,
1331 const int tag)
const 1338 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1340 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1342 MPI_Irecv (const_cast<char*> (recvBuffer.getRawPtr ()), recvBuffer.size (),
1343 MPI_CHAR, theSrcRank, tag, *rawMpiComm_, &rawMpiRequest);
1345 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1346 << mpiErrorCodeToString (err) <<
"\".");
1348 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size ());
1353 template<
typename Ordinal>
1355 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1356 const ArrayView<MPI_Status>& rawMpiStatuses)
1358 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1359 const size_type count = requests.size();
1363 std::logic_error,
"Teuchos::MpiComm's waitAllImpl: rawMpiStatus.size() = " 1364 << rawMpiStatuses.size() <<
" != requests.size() = " << requests.size()
1365 <<
". Please report this bug to the Tpetra developers.");
1378 bool someNullRequests =
false;
1379 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1380 for (
int i = 0; i < count; ++i) {
1381 RCP<CommRequest<Ordinal> > request = requests[i];
1383 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1384 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1392 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest();
1395 rawMpiRequests[i] = MPI_REQUEST_NULL;
1396 someNullRequests =
true;
1409 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1410 rawMpiStatuses.getRawPtr());
1420 if (err != MPI_SUCCESS) {
1421 if (err == MPI_ERR_IN_STATUS) {
1429 Array<std::pair<size_type, int> > errorLocationsAndCodes;
1430 for (size_type k = 0; k < rawMpiStatuses.size(); ++k) {
1431 const int curErr = rawMpiStatuses[k].MPI_ERROR;
1432 if (curErr != MPI_SUCCESS) {
1433 errorLocationsAndCodes.push_back (std::make_pair (k, curErr));
1436 const size_type numErrs = errorLocationsAndCodes.size();
1441 std::ostringstream os;
1442 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1443 << mpiErrorCodeToString (err) <<
"\". Of the " << count
1444 <<
" total request" << (count != 1 ?
"s" :
"") <<
", " << numErrs
1445 <<
" failed. Here are the indices of the failed requests, and the " 1446 "error codes extracted from their returned MPI_Status objects:" 1448 for (size_type k = 0; k < numErrs; ++k) {
1449 const size_type errInd = errorLocationsAndCodes[k].first;
1450 os <<
"Request " << errInd <<
": MPI_ERROR = " 1451 << mpiErrorCodeToString (rawMpiStatuses[errInd].MPI_ERROR)
1454 if (someNullRequests) {
1455 os <<
" On input to MPI_Waitall, there was at least one MPI_" 1456 "Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1457 "normally fail in that case, but we thought we should let you know " 1466 std::ostringstream os;
1467 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1468 << mpiErrorCodeToString (err) <<
"\".";
1469 if (someNullRequests) {
1470 os <<
" On input to MPI_Waitall, there was at least one MPI_Request " 1471 "that was MPI_REQUEST_NULL. MPI_Waitall should not normally fail in " 1472 "that case, but we thought we should let you know regardless.";
1480 std::fill (requests.begin(), requests.end(),
null);
1486 template<
typename Ordinal>
1488 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
1490 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1491 const size_type count = requests.size ();
1504 bool someNullRequests =
false;
1505 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1506 for (
int i = 0; i < count; ++i) {
1507 RCP<CommRequest<Ordinal> > request = requests[i];
1508 if (! request.is_null ()) {
1509 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1510 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1518 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest ();
1521 rawMpiRequests[i] = MPI_REQUEST_NULL;
1522 someNullRequests =
true;
1532 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1533 MPI_STATUSES_IGNORE);
1543 if (err != MPI_SUCCESS) {
1544 std::ostringstream os;
1545 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1546 << mpiErrorCodeToString (err) <<
"\".";
1547 if (someNullRequests) {
1548 os << std::endl <<
"On input to MPI_Waitall, there was at least one " 1549 "MPI_Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1550 "normally fail in that case, but we thought we should let you know " 1561 std::fill (requests.begin(), requests.end(),
null);
1568 template<
typename Ordinal>
1571 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
const 1576 waitAllImpl<Ordinal> (requests);
1580 template<
typename Ordinal>
1583 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1584 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const 1588 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1589 const size_type count = requests.size();
1592 std::invalid_argument,
"Teuchos::MpiComm::waitAll: requests.size() = " 1593 << count <<
" != statuses.size() = " << statuses.size() <<
".");
1595 Array<MPI_Status> rawMpiStatuses (count);
1596 waitAllImpl<Ordinal> (requests, rawMpiStatuses());
1599 for (size_type i = 0; i < count; ++i) {
1600 statuses[i] = mpiCommStatus<Ordinal> (rawMpiStatuses[i]);
1605 template<
typename Ordinal>
1606 RCP<CommStatus<Ordinal> >
1607 MpiComm<Ordinal>::wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const 1615 RCP<CommStatus<Ordinal> > status = (*request)->wait ();
1623 template<
typename Ordinal>
1624 RCP< Comm<Ordinal> >
1625 MpiComm<Ordinal>::duplicate()
const 1627 MPI_Comm origRawComm = *rawMpiComm_;
1628 MPI_Comm newRawComm = MPI_COMM_NULL;
1629 const int err = MPI_Comm_dup (origRawComm, &newRawComm);
1631 "::MpiComm::duplicate: MPI_Comm_dup failed with the following error: " 1632 << mpiErrorCodeToString (err));
1637 RCP<OpaqueWrapper<MPI_Comm> > wrapped =
1638 opaqueWrapper<MPI_Comm> (newRawComm, details::safeCommFree);
1642 RCP<MpiComm<Ordinal> > newComm =
1643 rcp (
new MpiComm<Ordinal> (wrapped.getConst (), minTag_));
1644 return rcp_implicit_cast<Comm<Ordinal> > (newComm);
1648 template<
typename Ordinal>
1649 RCP< Comm<Ordinal> >
1653 const int splitReturn =
1654 MPI_Comm_split (*rawMpiComm_,
1655 color < 0 ? MPI_UNDEFINED : color,
1659 splitReturn != MPI_SUCCESS,
1661 "Teuchos::MpiComm::split: Failed to create communicator with color " 1662 << color <<
"and key " << key <<
". MPI_Comm_split failed with error \"" 1663 << mpiErrorCodeToString (splitReturn) <<
"\".");
1664 if (newComm == MPI_COMM_NULL) {
1665 return RCP< Comm<Ordinal> >();
1667 RCP<const OpaqueWrapper<MPI_Comm> > wrapped =
1668 opaqueWrapper<MPI_Comm> (newComm, details::safeCommFree);
1673 return rcp (
new MpiComm<Ordinal> (wrapped, minTag_));
1678 template<
typename Ordinal>
1679 RCP< Comm<Ordinal> >
1680 MpiComm<Ordinal>::createSubcommunicator(
const ArrayView<const int> &ranks)
const 1682 int err = MPI_SUCCESS;
1685 MPI_Group thisGroup;
1686 err = MPI_Comm_group (*rawMpiComm_, &thisGroup);
1688 "Failed to obtain the current communicator's group. " 1689 "MPI_Comm_group failed with error \"" 1690 << mpiErrorCodeToString (err) <<
"\".");
1698 err = MPI_Group_incl (thisGroup, ranks.size(),
1699 const_cast<int*
> (ranks.getRawPtr ()), &newGroup);
1701 "Failed to create subgroup. MPI_Group_incl failed with error \"" 1702 << mpiErrorCodeToString (err) <<
"\".");
1707 err = MPI_Comm_create (*rawMpiComm_, newGroup, &newComm);
1709 "Failed to create subcommunicator. MPI_Comm_create failed with error \"" 1710 << mpiErrorCodeToString (err) <<
"\".");
1717 (void) MPI_Group_free (&newGroup);
1718 (void) MPI_Group_free (&thisGroup);
1723 err = MPI_Group_free (&newGroup);
1725 "Failed to free subgroup. MPI_Group_free failed with error \"" 1726 << mpiErrorCodeToString (err) <<
"\".");
1727 err = MPI_Group_free (&thisGroup);
1729 "Failed to free subgroup. MPI_Group_free failed with error \"" 1730 << mpiErrorCodeToString (err) <<
"\".");
1732 if (newComm == MPI_COMM_NULL) {
1733 return RCP<Comm<Ordinal> > ();
1735 using Teuchos::details::safeCommFree;
1736 typedef OpaqueWrapper<MPI_Comm> ow_type;
1737 RCP<const ow_type> wrapper =
1738 rcp_implicit_cast<
const ow_type> (opaqueWrapper (newComm, safeCommFree));
1743 return rcp (
new MpiComm<Ordinal> (wrapper, minTag_));
1751 template<
typename Ordinal>
1752 std::string MpiComm<Ordinal>::description()
const 1754 std::ostringstream oss;
1760 <<
",rawMpiComm="<<
static_cast<MPI_Comm
>(*rawMpiComm_)
1766 #ifdef TEUCHOS_MPI_COMM_DUMP 1767 template<
typename Ordinal>
1768 bool MpiComm<Ordinal>::show_dump =
false;
1775 template<
typename Ordinal>
1776 void MpiComm<Ordinal>::assertRank(
const int rank,
const std::string &rankName)
const 1779 ! ( 0 <= rank && rank < size_ ), std::logic_error
1780 ,
"Error, "<<rankName<<
" = " << rank <<
" is not < 0 or is not" 1781 " in the range [0,"<<size_-1<<
"]!" 1789 template<
typename Ordinal>
1791 Teuchos::createMpiComm(
1792 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
1795 if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
1796 return rcp(
new MpiComm<Ordinal>(rawMpiComm));
1801 template<
typename Ordinal>
1803 Teuchos::getRawMpiComm(
const Comm<Ordinal> &comm)
1806 dyn_cast<
const MpiComm<Ordinal> >(comm).getRawMpiComm()
1811 #endif // HAVE_TEUCHOS_MPI 1812 #endif // TEUCHOS_MPI_COMM_HPP
RCP< T > rcp(const boost::shared_ptr< T > &sptr)
Conversion function that takes in a boost::shared_ptr object and spits out a Teuchos::RCP object...
std::string typeName(const T &t)
Template function for returning the concrete type name of a passed-in object.
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Ordinal size_type
Type representing the number of elements in an ArrayRCP or view thereof.
Teuchos header file which uses auto-configuration information to include necessary C++ headers...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object...
#define TEUCHOS_COMM_TIME_MONITOR(FUNCNAME)
std::vector< std::string > split(const std::string &str, const std::string &delimiters, const size_t start)
Split the given string using the given set of delimiters.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
Teuchos implementation details.
TEUCHOS_DEPRECATED void scan(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *scanReduct)
Deprecated.
T_To & dyn_cast(T_From &from)
Dynamic casting utility function meant to replace dynamic_cast<T&> by throwing a better documented er...
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
TEUCHOS_DEPRECATED void reduceAll(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *globalReduct)
Deprecated .
void send(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of send() that takes a tag (and restores the correct order of arguments). ...
TypeTo as(const TypeFrom &t)
Convert from one value type to another.
RCP< CommRequest< Ordinal > > ireceive(const ArrayRCP< Packet > &recvBuffer, const int sourceRank, const int tag, const Comm< Ordinal > &comm)
Variant of ireceive that takes a tag argument (and restores the correct order of arguments).
Defines basic traits for the ordinal field type.
RCP< Teuchos::CommRequest< int > > isend(const ArrayRCP< const double > &sendBuffer, const int destRank, const int tag, const Comm< int > &comm)
void ssend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of ssend() that takes a tag (and restores the correct order of arguments).
static std::string name()
Returns name of this ordinal type.
Smart reference counting pointer class for automatic garbage collection.
RCP< MpiCommRequest > mpiCommRequest(MPI_Request rawMpiRequest, const ArrayRCP< const char > &buffer)
Implementation detail of Teuchos' MPI wrapper.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
Defines basic traits returning the name of a type in a portable and readable way. ...
Definition of Teuchos::as, for conversions between types.
void readySend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of readySend() that accepts a message tag.