42 #ifndef TEUCHOS_MPI_COMM_HPP 43 #define TEUCHOS_MPI_COMM_HPP 53 #ifdef HAVE_TEUCHOS_MPI 55 #include "Teuchos_Comm.hpp" 56 #include "Teuchos_CommUtilities.hpp" 58 #include "Teuchos_OpaqueWrapper.hpp" 60 #include "Teuchos_SerializationTraitsHelpers.hpp" 61 #include "Teuchos_Workspace.hpp" 64 #include "Teuchos_Assert.hpp" 71 #ifdef TEUCHOS_MPI_COMM_DUMP 72 # include "Teuchos_VerboseObject.hpp" 79 mpiErrorCodeToString (
const int err);
95 void safeCommFree (MPI_Comm* comm);
101 int setCommErrhandler (MPI_Comm comm, MPI_Errhandler handler);
105 #ifdef TEUCHOS_MPI_COMM_DUMP 106 template<
typename Ordinal,
typename T>
108 const std::string &funcName,
const std::string &buffName
109 ,
const Ordinal bytes,
const T buff[]
116 <<
"\n" << funcName <<
"::" << buffName <<
":\n";
118 for( Ordinal i = 0; i < bytes; ++i ) {
119 *out << buffName <<
"[" << i <<
"] = '" << buff[i] <<
"'\n";
123 #endif // TEUCHOS_MPI_COMM_DUMP 136 template<
class OrdinalType>
137 class MpiCommStatus :
public CommStatus<OrdinalType> {
139 MpiCommStatus (MPI_Status status) : status_ (status) {}
142 virtual ~MpiCommStatus() {}
145 OrdinalType getSourceRank () {
return status_.MPI_SOURCE; }
148 OrdinalType getTag () {
return status_.MPI_TAG; }
151 OrdinalType getError () {
return status_.MPI_ERROR; }
164 template<
class OrdinalType>
165 inline RCP<MpiCommStatus<OrdinalType> >
166 mpiCommStatus (MPI_Status rawMpiStatus)
168 return rcp (
new MpiCommStatus<OrdinalType> (rawMpiStatus));
186 template<
class OrdinalType>
187 class MpiCommRequestBase :
public CommRequest<OrdinalType> {
190 MpiCommRequestBase () :
191 rawMpiRequest_ (MPI_REQUEST_NULL)
195 MpiCommRequestBase (MPI_Request rawMpiRequest) :
196 rawMpiRequest_ (rawMpiRequest)
206 MPI_Request releaseRawMpiRequest()
208 MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
209 rawMpiRequest_ = MPI_REQUEST_NULL;
210 return tmp_rawMpiRequest;
214 bool isNull()
const {
215 return rawMpiRequest_ == MPI_REQUEST_NULL;
223 RCP<CommStatus<OrdinalType> > wait () {
224 MPI_Status rawMpiStatus;
227 const int err = MPI_Wait (&rawMpiRequest_, &rawMpiStatus);
229 err != MPI_SUCCESS, std::runtime_error,
230 "Teuchos: MPI_Wait() failed with error \"" 231 << mpiErrorCodeToString (err));
233 return mpiCommStatus<OrdinalType> (rawMpiStatus);
240 RCP<CommStatus<OrdinalType> > cancel () {
241 if (rawMpiRequest_ == MPI_REQUEST_NULL) {
245 int err = MPI_Cancel (&rawMpiRequest_);
247 err != MPI_SUCCESS, std::runtime_error,
248 "Teuchos: MPI_Cancel failed with the following error: " 249 << mpiErrorCodeToString (err));
256 err = MPI_Wait (&rawMpiRequest_, &status);
258 "Teuchos::MpiCommStatus::cancel: MPI_Wait failed with the following " 259 "error: " << mpiErrorCodeToString (err));
260 return mpiCommStatus<OrdinalType> (status);
265 virtual ~MpiCommRequestBase () {
266 if (rawMpiRequest_ != MPI_REQUEST_NULL) {
269 const int err = MPI_Cancel (&rawMpiRequest_);
270 if (err == MPI_SUCCESS) {
287 (void) MPI_Wait (&rawMpiRequest_, MPI_STATUS_IGNORE);
294 MPI_Request rawMpiRequest_;
312 template<
class OrdinalType>
313 class MpiCommRequest :
public MpiCommRequestBase<OrdinalType> {
317 MpiCommRequestBase<OrdinalType> (MPI_REQUEST_NULL),
322 MpiCommRequest (MPI_Request rawMpiRequest,
324 MpiCommRequestBase<OrdinalType> (rawMpiRequest),
325 numBytes_ (numBytesInMessage)
338 virtual ~MpiCommRequest () {}
353 template<
class OrdinalType>
354 inline RCP<MpiCommRequest<OrdinalType> >
355 mpiCommRequest (MPI_Request rawMpiRequest,
358 return rcp (
new MpiCommRequest<OrdinalType> (rawMpiRequest, numBytes));
376 template<
typename Ordinal>
377 class MpiComm :
public Comm<Ordinal> {
402 explicit MpiComm (MPI_Comm rawMpiComm);
418 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm);
437 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
438 const int defaultTag);
456 MpiComm (
const MpiComm<Ordinal>& other);
459 RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm ()
const {
527 void setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler);
534 virtual int getRank()
const;
537 virtual int getSize()
const;
540 virtual void barrier()
const;
543 virtual void broadcast(
544 const int rootRank,
const Ordinal bytes,
char buffer[]
549 gather (
const Ordinal sendBytes,
const char sendBuffer[],
550 const Ordinal recvBytes,
char recvBuffer[],
551 const int root)
const;
553 virtual void gatherAll(
554 const Ordinal sendBytes,
const char sendBuffer[]
555 ,
const Ordinal recvBytes,
char recvBuffer[]
559 const ValueTypeReductionOp<Ordinal,char> &reductOp
560 ,
const Ordinal bytes,
const char sendBuffer[],
char globalReducts[]
564 const ValueTypeReductionOp<Ordinal,char> &reductOp
565 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
569 const Ordinal bytes,
const char sendBuffer[],
const int destRank
573 send (
const Ordinal bytes,
574 const char sendBuffer[],
576 const int tag)
const;
579 const Ordinal bytes,
const char sendBuffer[],
const int destRank
583 ssend (
const Ordinal bytes,
584 const char sendBuffer[],
586 const int tag)
const;
589 const int sourceRank,
const Ordinal bytes,
char recvBuffer[]
593 const ArrayView<const char> &sendBuffer,
599 const char sendBuffer[],
601 const int tag)
const;
603 virtual RCP<CommRequest<Ordinal> > isend(
604 const ArrayView<const char> &sendBuffer,
608 virtual RCP<CommRequest<Ordinal> >
609 isend (
const ArrayView<const char> &sendBuffer,
611 const int tag)
const;
613 virtual RCP<CommRequest<Ordinal> >
ireceive(
614 const ArrayView<char> &Buffer,
618 virtual RCP<CommRequest<Ordinal> >
619 ireceive (
const ArrayView<char> &Buffer,
620 const int sourceRank,
621 const int tag)
const;
623 virtual void waitAll(
624 const ArrayView<RCP<CommRequest<Ordinal> > > &requests
628 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
629 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const;
631 virtual RCP<CommStatus<Ordinal> >
632 wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const;
634 virtual RCP< Comm<Ordinal> > duplicate()
const;
636 virtual RCP< Comm<Ordinal> > split(
const int color,
const int key)
const;
638 virtual RCP< Comm<Ordinal> > createSubcommunicator(
639 const ArrayView<const int>& ranks)
const;
646 std::string description()
const;
652 static int const minTag_ = 26000;
653 static int const maxTag_ = 26099;
660 int getTag ()
const {
return tag_; }
667 void setupMembersFromComm();
668 static int tagCounter_;
677 RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
695 RCP<const OpaqueWrapper<MPI_Errhandler> > customErrorHandler_;
697 void assertRank(
const int rank,
const std::string &rankName)
const;
702 #ifdef TEUCHOS_MPI_COMM_DUMP 704 static bool show_dump;
705 #endif // TEUCHOS_MPI_COMM_DUMP 723 template<
typename Ordinal>
724 RCP<MpiComm<Ordinal> >
726 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
757 template<
typename Ordinal>
759 getRawMpiComm(
const Comm<Ordinal> &comm);
769 template<
typename Ordinal>
770 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
776 template<
typename Ordinal>
778 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm)
781 rawMpiComm.get () == NULL, std::invalid_argument,
782 "Teuchos::MpiComm constructor: The input RCP is null.");
784 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
785 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
787 rawMpiComm_ = rawMpiComm;
801 setupMembersFromComm ();
805 template<
typename Ordinal>
807 MpiComm (
const RCP<
const OpaqueWrapper<MPI_Comm> >& rawMpiComm,
808 const int defaultTag)
811 rawMpiComm.get () == NULL, std::invalid_argument,
812 "Teuchos::MpiComm constructor: The input RCP is null.");
814 *rawMpiComm == MPI_COMM_NULL, std::invalid_argument,
815 "Teuchos::MpiComm constructor: The given MPI_Comm is MPI_COMM_NULL.");
817 rawMpiComm_ = rawMpiComm;
819 int err = MPI_Comm_size (*rawMpiComm_, &size_);
821 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 822 "error \"" << mpiErrorCodeToString (err) <<
"\".");
824 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
826 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 827 "error \"" << mpiErrorCodeToString (err) <<
"\".");
832 template<
typename Ordinal>
833 MpiComm<Ordinal>::MpiComm (MPI_Comm rawMpiComm)
836 std::invalid_argument,
"Teuchos::MpiComm constructor: The given MPI_Comm " 837 "is MPI_COMM_NULL.");
841 rawMpiComm_ = opaqueWrapper<MPI_Comm> (rawMpiComm);
855 setupMembersFromComm ();
859 template<
typename Ordinal>
860 MpiComm<Ordinal>::MpiComm (
const MpiComm<Ordinal>& other) :
861 rawMpiComm_ (opaqueWrapper<MPI_Comm> (MPI_COMM_NULL))
864 RCP<const OpaqueWrapper<MPI_Comm> > origCommPtr = other.getRawMpiComm ();
866 "Teuchos::MpiComm copy constructor: " 867 "The input's getRawMpiComm() method returns null.");
868 MPI_Comm origComm = *origCommPtr;
870 "Teuchos::MpiComm copy constructor: " 871 "The input's raw MPI_Comm is MPI_COMM_NULL.");
879 rawMpiComm_ = origCommPtr;
883 const int err = MPI_Comm_dup (origComm, &newComm);
885 "Teuchos::MpiComm copy constructor: MPI_Comm_dup failed with " 886 "the following error: " << mpiErrorCodeToString (err));
888 rawMpiComm_ = opaqueWrapper (newComm, details::safeCommFree);
891 setupMembersFromComm ();
895 template<
typename Ordinal>
896 void MpiComm<Ordinal>::setupMembersFromComm ()
898 int err = MPI_Comm_size (*rawMpiComm_, &size_);
900 "Teuchos::MpiComm constructor: MPI_Comm_size failed with " 901 "error \"" << mpiErrorCodeToString (err) <<
"\".");
902 err = MPI_Comm_rank (*rawMpiComm_, &rank_);
904 "Teuchos::MpiComm constructor: MPI_Comm_rank failed with " 905 "error \"" << mpiErrorCodeToString (err) <<
"\".");
908 if (tagCounter_ > maxTag_) {
909 tagCounter_ = minTag_;
911 tag_ = tagCounter_++;
921 MPI_Bcast (&tag_, 1, MPI_INT, 0, *rawMpiComm_);
925 template<
typename Ordinal>
928 setErrorHandler (
const RCP<
const OpaqueWrapper<MPI_Errhandler> >& errHandler)
931 const int err = details::setCommErrhandler (*getRawMpiComm (), *errHandler);
933 "Teuchos::MpiComm: Setting the MPI_Comm's error handler failed with " 934 "error \"" << mpiErrorCodeToString (err) <<
"\".");
938 customErrorHandler_ = errHandler;
945 template<
typename Ordinal>
946 int MpiComm<Ordinal>::getRank()
const 952 template<
typename Ordinal>
953 int MpiComm<Ordinal>::getSize()
const 959 template<
typename Ordinal>
960 void MpiComm<Ordinal>::barrier()
const 962 TEUCHOS_COMM_TIME_MONITOR(
965 const int err = MPI_Barrier (*rawMpiComm_);
967 "Teuchos::MpiComm::barrier: MPI_Barrier failed with error \"" 968 << mpiErrorCodeToString (err) <<
"\".");
972 template<
typename Ordinal>
973 void MpiComm<Ordinal>::broadcast(
974 const int rootRank,
const Ordinal bytes,
char buffer[]
977 TEUCHOS_COMM_TIME_MONITOR(
980 const int err = MPI_Bcast (buffer, bytes, MPI_CHAR, rootRank, *rawMpiComm_);
982 "Teuchos::MpiComm::broadcast: MPI_Bcast failed with error \"" 983 << mpiErrorCodeToString (err) <<
"\".");
987 template<
typename Ordinal>
988 void MpiComm<Ordinal>::gatherAll(
989 const Ordinal sendBytes,
const char sendBuffer[],
990 const Ordinal recvBytes,
char recvBuffer[]
993 TEUCHOS_COMM_TIME_MONITOR(
998 MPI_Allgather (const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
999 recvBuffer, sendBytes, MPI_CHAR, *rawMpiComm_);
1004 "Teuchos::MpiComm::gatherAll: MPI_Allgather failed with error \"" 1005 << mpiErrorCodeToString (err) <<
"\".");
1009 template<
typename Ordinal>
1011 MpiComm<Ordinal>::gather (
const Ordinal sendBytes,
1012 const char sendBuffer[],
1013 const Ordinal recvBytes,
1015 const int root)
const 1019 TEUCHOS_COMM_TIME_MONITOR(
1023 MPI_Gather (const_cast<char *> (sendBuffer), sendBytes, MPI_CHAR,
1024 recvBuffer, sendBytes, MPI_CHAR, root, *rawMpiComm_);
1026 "Teuchos::MpiComm::gather: MPI_Gather failed with error \"" 1027 << mpiErrorCodeToString (err) <<
"\".");
1031 template<
typename Ordinal>
1034 reduceAll (
const ValueTypeReductionOp<Ordinal,char> &reductOp,
1035 const Ordinal bytes,
1036 const char sendBuffer[],
1037 char globalReducts[])
const 1039 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::reduceAll(...)" );
1040 int err = MPI_SUCCESS;
1042 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1043 MPI_Op op = Details::setMpiReductionOp (opWrap);
1052 MPI_Datatype char_block;
1053 err = MPI_Type_contiguous (bytes, MPI_CHAR, &char_block);
1055 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1056 "MPI_Type_contiguous failed with error \"" << mpiErrorCodeToString (err)
1058 err = MPI_Type_commit (&char_block);
1060 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1061 "MPI_Type_commit failed with error \"" << mpiErrorCodeToString (err)
1064 err = MPI_Allreduce (const_cast<char*> (sendBuffer), globalReducts, 1,
1065 char_block, op, *rawMpiComm_);
1066 if (err != MPI_SUCCESS) {
1071 (void) MPI_Type_free (&char_block);
1073 true, std::runtime_error,
"Teuchos::reduceAll (MPI, custom op): " 1074 "MPI_Allreduce failed with error \"" << mpiErrorCodeToString (err)
1077 err = MPI_Type_free (&char_block);
1079 err != MPI_SUCCESS, std::runtime_error,
"Teuchos::reduceAll: " 1080 "MPI_Type_free failed with error \"" << mpiErrorCodeToString (err)
1085 template<
typename Ordinal>
1086 void MpiComm<Ordinal>::scan(
1087 const ValueTypeReductionOp<Ordinal,char> &reductOp
1088 ,
const Ordinal bytes,
const char sendBuffer[],
char scanReducts[]
1091 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::scan(...)" );
1093 Details::MpiReductionOp<Ordinal> opWrap (reductOp);
1094 MPI_Op op = Details::setMpiReductionOp (opWrap);
1096 MPI_Scan (const_cast<char*> (sendBuffer), scanReducts, bytes, MPI_CHAR,
1099 "Teuchos::MpiComm::scan: MPI_Scan() failed with error \"" 1100 << mpiErrorCodeToString (err) <<
"\".");
1104 template<
typename Ordinal>
1106 MpiComm<Ordinal>::send (
const Ordinal bytes,
1107 const char sendBuffer[],
1108 const int destRank)
const 1110 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::send(...)" );
1112 #ifdef TEUCHOS_MPI_COMM_DUMP 1114 dumpBuffer<Ordinal,char>(
1115 "Teuchos::MpiComm<Ordinal>::send(...)" 1116 ,
"sendBuffer", bytes, sendBuffer
1119 #endif // TEUCHOS_MPI_COMM_DUMP 1121 const int err = MPI_Send (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1122 destRank, tag_, *rawMpiComm_);
1124 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1125 << mpiErrorCodeToString (err) <<
"\".");
1129 template<
typename Ordinal>
1131 MpiComm<Ordinal>::send (
const Ordinal bytes,
1132 const char sendBuffer[],
1134 const int tag)
const 1136 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::send(...)" );
1137 const int err = MPI_Send (const_cast<char*> (sendBuffer), bytes, MPI_CHAR,
1138 destRank, tag, *rawMpiComm_);
1140 "Teuchos::MpiComm::send: MPI_Send() failed with error \"" 1141 << mpiErrorCodeToString (err) <<
"\".");
1145 template<
typename Ordinal>
1147 MpiComm<Ordinal>::ssend (
const Ordinal bytes,
1148 const char sendBuffer[],
1149 const int destRank)
const 1151 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ssend(...)" );
1153 #ifdef TEUCHOS_MPI_COMM_DUMP 1155 dumpBuffer<Ordinal,char>(
1156 "Teuchos::MpiComm<Ordinal>::send(...)" 1157 ,
"sendBuffer", bytes, sendBuffer
1160 #endif // TEUCHOS_MPI_COMM_DUMP 1162 const int err = MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1163 destRank, tag_, *rawMpiComm_);
1165 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1166 << mpiErrorCodeToString (err) <<
"\".");
1169 template<
typename Ordinal>
1171 MpiComm<Ordinal>::ssend (
const Ordinal bytes,
1172 const char sendBuffer[],
1174 const int tag)
const 1176 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ssend(...)" );
1178 MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
1179 destRank, tag, *rawMpiComm_);
1181 "Teuchos::MpiComm::send: MPI_Ssend() failed with error \"" 1182 << mpiErrorCodeToString (err) <<
"\".");
1185 template<
typename Ordinal>
1186 void MpiComm<Ordinal>::readySend(
1187 const ArrayView<const char> &sendBuffer,
1191 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::readySend" );
1193 #ifdef TEUCHOS_MPI_COMM_DUMP 1195 dumpBuffer<Ordinal,char>(
1196 "Teuchos::MpiComm<Ordinal>::readySend(...)" 1197 ,
"sendBuffer", bytes, sendBuffer
1200 #endif // TEUCHOS_MPI_COMM_DUMP 1203 MPI_Rsend (const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(),
1204 MPI_CHAR, destRank, tag_, *rawMpiComm_);
1206 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1207 << mpiErrorCodeToString (err) <<
"\".");
1211 template<
typename Ordinal>
1212 void MpiComm<Ordinal>::
1213 readySend (
const Ordinal bytes,
1214 const char sendBuffer[],
1216 const int tag)
const 1218 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::readySend" );
1220 MPI_Rsend (const_cast<char*> (sendBuffer), bytes,
1221 MPI_CHAR, destRank, tag, *rawMpiComm_);
1223 "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \"" 1224 << mpiErrorCodeToString (err) <<
"\".");
1228 template<
typename Ordinal>
1230 MpiComm<Ordinal>::receive (
const int sourceRank,
1231 const Ordinal bytes,
1232 char recvBuffer[])
const 1234 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::receive(...)" );
1239 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1242 const int err = MPI_Recv (recvBuffer, bytes, MPI_CHAR, theSrcRank, tag_,
1243 *rawMpiComm_, &status);
1245 "Teuchos::MpiComm::receive: MPI_Recv() failed with error \"" 1246 << mpiErrorCodeToString (err) <<
"\".");
1248 #ifdef TEUCHOS_MPI_COMM_DUMP 1250 dumpBuffer<Ordinal,char> (
"Teuchos::MpiComm<Ordinal>::receive(...)",
1251 "recvBuffer", bytes, recvBuffer);
1253 #endif // TEUCHOS_MPI_COMM_DUMP 1256 return status.MPI_SOURCE;
1260 template<
typename Ordinal>
1261 RCP<CommRequest<Ordinal> >
1262 MpiComm<Ordinal>::isend (
const ArrayView<const char> &sendBuffer,
1263 const int destRank)
const 1266 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::isend(...)" );
1268 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1270 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1271 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1272 destRank, tag_, *rawMpiComm_, &rawMpiRequest);
1274 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1275 << mpiErrorCodeToString (err) <<
"\".");
1277 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1281 template<
typename Ordinal>
1282 RCP<CommRequest<Ordinal> >
1284 isend (
const ArrayView<const char> &sendBuffer,
1286 const int tag)
const 1289 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::isend(...)" );
1291 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1293 MPI_Isend (const_cast<char*> (sendBuffer.getRawPtr ()),
1294 as<Ordinal> (sendBuffer.size ()), MPI_CHAR,
1295 destRank, tag, *rawMpiComm_, &rawMpiRequest);
1297 "Teuchos::MpiComm::isend: MPI_Isend() failed with error \"" 1298 << mpiErrorCodeToString (err) <<
"\".");
1300 return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size ());
1304 template<
typename Ordinal>
1305 RCP<CommRequest<Ordinal> >
1306 MpiComm<Ordinal>::ireceive (
const ArrayView<char> &recvBuffer,
1307 const int sourceRank)
const 1309 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ireceive(...)" );
1314 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1316 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1318 MPI_Irecv (const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(),
1319 MPI_CHAR, theSrcRank, tag_, *rawMpiComm_, &rawMpiRequest);
1321 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1322 << mpiErrorCodeToString (err) <<
"\".");
1324 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size());
1327 template<
typename Ordinal>
1328 RCP<CommRequest<Ordinal> >
1329 MpiComm<Ordinal>::ireceive (
const ArrayView<char> &recvBuffer,
1330 const int sourceRank,
1331 const int tag)
const 1333 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::ireceive(...)" );
1338 const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
1340 MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
1342 MPI_Irecv (const_cast<char*> (recvBuffer.getRawPtr ()), recvBuffer.size (),
1343 MPI_CHAR, theSrcRank, tag, *rawMpiComm_, &rawMpiRequest);
1345 "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \"" 1346 << mpiErrorCodeToString (err) <<
"\".");
1348 return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size ());
1353 template<
typename Ordinal>
1355 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1356 const ArrayView<MPI_Status>& rawMpiStatuses)
1358 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1359 const size_type count = requests.size();
1363 std::logic_error,
"Teuchos::MpiComm's waitAllImpl: rawMpiStatus.size() = " 1364 << rawMpiStatuses.size() <<
" != requests.size() = " << requests.size()
1365 <<
". Please report this bug to the Tpetra developers.");
1378 bool someNullRequests =
false;
1379 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1380 for (
int i = 0; i < count; ++i) {
1381 RCP<CommRequest<Ordinal> > request = requests[i];
1383 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1384 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1392 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest();
1395 rawMpiRequests[i] = MPI_REQUEST_NULL;
1396 someNullRequests =
true;
1409 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1410 rawMpiStatuses.getRawPtr());
1420 if (err != MPI_SUCCESS) {
1421 if (err == MPI_ERR_IN_STATUS) {
1429 Array<std::pair<size_type, int> > errorLocationsAndCodes;
1430 for (size_type k = 0; k < rawMpiStatuses.size(); ++k) {
1431 const int curErr = rawMpiStatuses[k].MPI_ERROR;
1432 if (curErr != MPI_SUCCESS) {
1433 errorLocationsAndCodes.push_back (std::make_pair (k, curErr));
1436 const size_type numErrs = errorLocationsAndCodes.size();
1441 std::ostringstream os;
1442 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1443 << mpiErrorCodeToString (err) <<
"\". Of the " << count
1444 <<
" total request" << (count != 1 ?
"s" :
"") <<
", " << numErrs
1445 <<
" failed. Here are the indices of the failed requests, and the " 1446 "error codes extracted from their returned MPI_Status objects:" 1448 for (size_type k = 0; k < numErrs; ++k) {
1449 const size_type errInd = errorLocationsAndCodes[k].first;
1450 os <<
"Request " << errInd <<
": MPI_ERROR = " 1451 << mpiErrorCodeToString (rawMpiStatuses[errInd].MPI_ERROR)
1454 if (someNullRequests) {
1455 os <<
" On input to MPI_Waitall, there was at least one MPI_" 1456 "Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1457 "normally fail in that case, but we thought we should let you know " 1466 std::ostringstream os;
1467 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1468 << mpiErrorCodeToString (err) <<
"\".";
1469 if (someNullRequests) {
1470 os <<
" On input to MPI_Waitall, there was at least one MPI_Request " 1471 "that was MPI_REQUEST_NULL. MPI_Waitall should not normally fail in " 1472 "that case, but we thought we should let you know regardless.";
1480 std::fill (requests.begin(), requests.end(), null);
1486 template<
typename Ordinal>
1488 waitAllImpl (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
1490 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1491 const size_type count = requests.size ();
1504 bool someNullRequests =
false;
1505 Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
1506 for (
int i = 0; i < count; ++i) {
1507 RCP<CommRequest<Ordinal> > request = requests[i];
1508 if (! request.is_null ()) {
1509 RCP<MpiCommRequestBase<Ordinal> > mpiRequest =
1510 rcp_dynamic_cast<MpiCommRequestBase<Ordinal> > (request);
1518 rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest ();
1521 rawMpiRequests[i] = MPI_REQUEST_NULL;
1522 someNullRequests =
true;
1532 const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
1533 MPI_STATUSES_IGNORE);
1543 if (err != MPI_SUCCESS) {
1544 std::ostringstream os;
1545 os <<
"Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \"" 1546 << mpiErrorCodeToString (err) <<
"\".";
1547 if (someNullRequests) {
1548 os << std::endl <<
"On input to MPI_Waitall, there was at least one " 1549 "MPI_Request that was MPI_REQUEST_NULL. MPI_Waitall should not " 1550 "normally fail in that case, but we thought we should let you know " 1561 std::fill (requests.begin(), requests.end(), null);
1568 template<
typename Ordinal>
1571 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests)
const 1573 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::waitAll(requests)" );
1576 waitAllImpl<Ordinal> (requests);
1580 template<
typename Ordinal>
1583 waitAll (
const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
1584 const ArrayView<RCP<CommStatus<Ordinal> > >& statuses)
const 1586 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::waitAll(requests, statuses)" );
1588 typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
1589 const size_type count = requests.size();
1592 std::invalid_argument,
"Teuchos::MpiComm::waitAll: requests.size() = " 1593 << count <<
" != statuses.size() = " << statuses.size() <<
".");
1595 Array<MPI_Status> rawMpiStatuses (count);
1596 waitAllImpl<Ordinal> (requests, rawMpiStatuses());
1599 for (size_type i = 0; i < count; ++i) {
1600 statuses[i] = mpiCommStatus<Ordinal> (rawMpiStatuses[i]);
1605 template<
typename Ordinal>
1606 RCP<CommStatus<Ordinal> >
1607 MpiComm<Ordinal>::wait (
const Ptr<RCP<CommRequest<Ordinal> > >& request)
const 1609 TEUCHOS_COMM_TIME_MONITOR(
"Teuchos::MpiComm::wait(...)" );
1615 RCP<CommStatus<Ordinal> > status = (*request)->wait ();
1623 template<
typename Ordinal>
1624 RCP< Comm<Ordinal> >
1625 MpiComm<Ordinal>::duplicate()
const 1627 MPI_Comm origRawComm = *rawMpiComm_;
1628 MPI_Comm newRawComm = MPI_COMM_NULL;
1629 const int err = MPI_Comm_dup (origRawComm, &newRawComm);
1631 "::MpiComm::duplicate: MPI_Comm_dup failed with the following error: " 1632 << mpiErrorCodeToString (err));
1637 RCP<OpaqueWrapper<MPI_Comm> > wrapped =
1638 opaqueWrapper<MPI_Comm> (newRawComm, details::safeCommFree);
1642 RCP<MpiComm<Ordinal> > newComm =
1643 rcp (
new MpiComm<Ordinal> (wrapped.getConst (), minTag_));
1644 return rcp_implicit_cast<Comm<Ordinal> > (newComm);
1648 template<
typename Ordinal>
1649 RCP< Comm<Ordinal> >
1650 MpiComm<Ordinal>::split(
const int color,
const int key)
const 1653 const int splitReturn =
1654 MPI_Comm_split (*rawMpiComm_,
1655 color < 0 ? MPI_UNDEFINED : color,
1659 splitReturn != MPI_SUCCESS,
1661 "Teuchos::MpiComm::split: Failed to create communicator with color " 1662 << color <<
"and key " << key <<
". MPI_Comm_split failed with error \"" 1663 << mpiErrorCodeToString (splitReturn) <<
"\".");
1664 if (newComm == MPI_COMM_NULL) {
1665 return RCP< Comm<Ordinal> >();
1667 RCP<const OpaqueWrapper<MPI_Comm> > wrapped =
1668 opaqueWrapper<MPI_Comm> (newComm, details::safeCommFree);
1673 return rcp (
new MpiComm<Ordinal> (wrapped, minTag_));
1678 template<
typename Ordinal>
1679 RCP< Comm<Ordinal> >
1680 MpiComm<Ordinal>::createSubcommunicator(
const ArrayView<const int> &ranks)
const 1682 int err = MPI_SUCCESS;
1685 MPI_Group thisGroup;
1686 err = MPI_Comm_group (*rawMpiComm_, &thisGroup);
1688 "Failed to obtain the current communicator's group. " 1689 "MPI_Comm_group failed with error \"" 1690 << mpiErrorCodeToString (err) <<
"\".");
1698 err = MPI_Group_incl (thisGroup, ranks.size(),
1699 const_cast<int*
> (ranks.getRawPtr ()), &newGroup);
1701 "Failed to create subgroup. MPI_Group_incl failed with error \"" 1702 << mpiErrorCodeToString (err) <<
"\".");
1707 err = MPI_Comm_create (*rawMpiComm_, newGroup, &newComm);
1709 "Failed to create subcommunicator. MPI_Comm_create failed with error \"" 1710 << mpiErrorCodeToString (err) <<
"\".");
1717 (void) MPI_Group_free (&newGroup);
1718 (void) MPI_Group_free (&thisGroup);
1723 err = MPI_Group_free (&newGroup);
1725 "Failed to free subgroup. MPI_Group_free failed with error \"" 1726 << mpiErrorCodeToString (err) <<
"\".");
1727 err = MPI_Group_free (&thisGroup);
1729 "Failed to free subgroup. MPI_Group_free failed with error \"" 1730 << mpiErrorCodeToString (err) <<
"\".");
1732 if (newComm == MPI_COMM_NULL) {
1733 return RCP<Comm<Ordinal> > ();
1735 using Teuchos::details::safeCommFree;
1736 typedef OpaqueWrapper<MPI_Comm> ow_type;
1737 RCP<const ow_type> wrapper =
1738 rcp_implicit_cast<
const ow_type> (opaqueWrapper (newComm, safeCommFree));
1743 return rcp (
new MpiComm<Ordinal> (wrapper, minTag_));
1751 template<
typename Ordinal>
1752 std::string MpiComm<Ordinal>::description()
const 1754 std::ostringstream oss;
1760 <<
",rawMpiComm="<<
static_cast<MPI_Comm
>(*rawMpiComm_)
1766 #ifdef TEUCHOS_MPI_COMM_DUMP 1767 template<
typename Ordinal>
1768 bool MpiComm<Ordinal>::show_dump =
false;
1775 template<
typename Ordinal>
1776 void MpiComm<Ordinal>::assertRank(
const int rank,
const std::string &rankName)
const 1779 ! ( 0 <= rank && rank < size_ ), std::logic_error
1780 ,
"Error, "<<rankName<<
" = " << rank <<
" is not < 0 or is not" 1781 " in the range [0,"<<size_-1<<
"]!" 1789 template<
typename Ordinal>
1791 Teuchos::createMpiComm(
1792 const RCP<
const OpaqueWrapper<MPI_Comm> > &rawMpiComm
1795 if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
1796 return rcp(
new MpiComm<Ordinal>(rawMpiComm));
1797 return Teuchos::null;
1801 template<
typename Ordinal>
1803 Teuchos::getRawMpiComm(
const Comm<Ordinal> &comm)
1806 dyn_cast<
const MpiComm<Ordinal> >(comm).getRawMpiComm()
1811 #endif // HAVE_TEUCHOS_MPI 1812 #endif // TEUCHOS_MPI_COMM_HPP RCP< T > rcp(const boost::shared_ptr< T > &sptr)
Conversion function that takes in a boost::shared_ptr object and spits out a Teuchos::RCP object...
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Ordinal size_type
Type representing the number of elements in an ArrayRCP or view thereof.
T_To & dyn_cast(T_From &from)
Dynamic casting utility function meant to replace dynamic_cast<T&> by throwing a better documented er...
Teuchos header file which uses auto-configuration information to include necessary C++ headers...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object...
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
Teuchos implementation details.
TEUCHOS_DEPRECATED void scan(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *scanReduct)
Deprecated.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
TEUCHOS_DEPRECATED void reduceAll(const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, Packet *globalReduct)
Deprecated .
void send(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of send() that takes a tag (and restores the correct order of arguments). ...
RCP< CommRequest< Ordinal > > ireceive(const ArrayRCP< Packet > &recvBuffer, const int sourceRank, const int tag, const Comm< Ordinal > &comm)
Variant of ireceive that takes a tag argument (and restores the correct order of arguments).
Defines basic traits for the ordinal field type.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos, as well as a number of utility routines.
TypeTo as(const TypeFrom &t)
Convert from one value type to another.
void ssend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of ssend() that takes a tag (and restores the correct order of arguments).
static std::string name()
Returns name of this ordinal type.
Smart reference counting pointer class for automatic garbage collection.
Implementation detail of Teuchos' MPI wrapper.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
Defines basic traits returning the name of a type in a portable and readable way. ...
Definition of Teuchos::as, for conversions between types.
void readySend(const Packet sendBuffer[], const Ordinal count, const int destRank, const int tag, const Comm< Ordinal > &comm)
Variant of readySend() that accepts a message tag.
std::string typeName(const T &t)
Template function for returning the concrete type name of a passed-in object.