42 #ifndef TPETRA_DISTRIBUTOR_HPP 43 #define TPETRA_DISTRIBUTOR_HPP 46 #include <Teuchos_as.hpp> 47 #include <Teuchos_Describable.hpp> 48 #include <Teuchos_ParameterListAcceptorDefaultBase.hpp> 49 #include <Teuchos_VerboseObject.hpp> 60 #ifdef TPETRA_DISTRIBUTOR_TIMERS 61 # undef TPETRA_DISTRIBUTOR_TIMERS 62 #endif // TPETRA_DISTRIBUTOR_TIMERS 64 #include "KokkosCompat_View.hpp" 65 #include "Kokkos_Core.hpp" 66 #include "Kokkos_TeuchosCommAdapters.hpp" 95 DISTRIBUTOR_NOT_INITIALIZED,
96 DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS,
97 DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS,
98 DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS,
99 DISTRIBUTOR_INITIALIZED_BY_REVERSE,
100 DISTRIBUTOR_INITIALIZED_BY_COPY,
188 public Teuchos::Describable,
189 public Teuchos::ParameterListAcceptorDefaultBase {
202 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
215 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
216 const Teuchos::RCP<Teuchos::FancyOStream>& out);
231 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
232 const Teuchos::RCP<Teuchos::ParameterList>& plist);
250 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
251 const Teuchos::RCP<Teuchos::FancyOStream>& out,
252 const Teuchos::RCP<Teuchos::ParameterList>& plist);
306 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
341 template <
class Ordinal>
344 const Teuchos::ArrayView<const int>& remoteProcIDs,
345 Teuchos::Array<Ordinal>& exportIDs,
346 Teuchos::Array<int>& exportProcIDs);
357 const Teuchos::ArrayView<const int>& remoteProcIDs);
392 Teuchos::ArrayView<const int>
getProcsTo()
const;
417 return howInitialized_;
460 template <
class Packet>
464 const Teuchos::ArrayView<Packet> &imports);
487 template <
class Packet>
490 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
491 const Teuchos::ArrayView<Packet> &imports,
492 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
518 template <
class Packet>
520 doPosts (
const Teuchos::ArrayRCP<const Packet> &exports,
522 const Teuchos::ArrayRCP<Packet> &imports);
542 template <
class Packet>
544 doPosts (
const Teuchos::ArrayRCP<const Packet> &exports,
545 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
546 const Teuchos::ArrayRCP<Packet> &imports,
547 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
561 template <
class Packet>
565 const Teuchos::ArrayView<Packet> &imports);
571 template <
class Packet>
574 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
575 const Teuchos::ArrayView<Packet> &imports,
576 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
582 template <
class Packet>
586 const Teuchos::ArrayRCP<Packet> &imports);
592 template <
class Packet>
595 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
596 const Teuchos::ArrayRCP<Packet> &imports,
597 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
627 template <
class ExpView,
class ImpView>
628 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
630 const ExpView &exports,
632 const ImpView &imports);
655 template <
class ExpView,
class ImpView>
656 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
658 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
659 const ImpView &imports,
660 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
686 template <
class ExpView,
class ImpView>
687 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
688 doPosts (
const ExpView &exports,
690 const ImpView &imports);
710 template <
class ExpView,
class ImpView>
711 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
712 doPosts (
const ExpView &exports,
713 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
714 const ImpView &imports,
715 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
721 template <
class ExpView,
class ImpView>
722 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
725 const ImpView &imports);
731 template <
class ExpView,
class ImpView>
732 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
734 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
735 const ImpView &imports,
736 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
742 template <
class ExpView,
class ImpView>
743 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
746 const ImpView &imports);
752 template <
class ExpView,
class ImpView>
753 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
755 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
756 const ImpView &imports,
757 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
763 bytes_sent = lastRoundBytesSend_;
764 bytes_recvd = lastRoundBytesRecv_;
796 describe (Teuchos::FancyOStream& out,
797 const Teuchos::EVerbosityLevel verbLevel =
798 Teuchos::Describable::verbLevel_default)
const;
803 Teuchos::RCP<const Teuchos::Comm<int> > comm_;
806 Teuchos::RCP<Teuchos::FancyOStream> out_;
818 bool barrierBetween_;
844 Teuchos::Array<int> procsTo_;
854 Teuchos::Array<size_t> startsTo_;
861 Teuchos::Array<size_t> lengthsTo_;
866 size_t maxSendLength_;
883 Teuchos::Array<size_t> indicesTo_;
902 size_t totalReceiveLength_;
909 Teuchos::Array<size_t> lengthsFrom_;
916 Teuchos::Array<int> procsFrom_;
923 Teuchos::Array<size_t> startsFrom_;
931 Teuchos::Array<size_t> indicesFrom_;
939 Teuchos::Array<Teuchos::RCP<Teuchos::CommRequest<int> > > requests_;
945 mutable Teuchos::RCP<Distributor> reverseDistributor_;
948 size_t lastRoundBytesSend_;
951 size_t lastRoundBytesRecv_;
953 #ifdef TPETRA_DISTRIBUTOR_TIMERS 954 Teuchos::RCP<Teuchos::Time> timer_doPosts3_;
955 Teuchos::RCP<Teuchos::Time> timer_doPosts4_;
956 Teuchos::RCP<Teuchos::Time> timer_doWaits_;
957 Teuchos::RCP<Teuchos::Time> timer_doPosts3_recvs_;
958 Teuchos::RCP<Teuchos::Time> timer_doPosts4_recvs_;
959 Teuchos::RCP<Teuchos::Time> timer_doPosts3_barrier_;
960 Teuchos::RCP<Teuchos::Time> timer_doPosts4_barrier_;
961 Teuchos::RCP<Teuchos::Time> timer_doPosts3_sends_;
962 Teuchos::RCP<Teuchos::Time> timer_doPosts4_sends_;
966 #endif // TPETRA_DISTRIBUTOR_TIMERS 979 bool useDistinctTags_;
985 int getTag (
const int pathTag)
const;
1005 init (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
1006 const Teuchos::RCP<Teuchos::FancyOStream>& out,
1007 const Teuchos::RCP<Teuchos::ParameterList>& plist);
1019 void computeReceives ();
1033 template <
class Ordinal>
1034 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
1035 const Teuchos::ArrayView<const int> &remoteProcIDs,
1036 Teuchos::Array<Ordinal> &exportGIDs,
1037 Teuchos::Array<int> &exportProcIDs);
1040 void createReverseDistributor()
const;
1048 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
1052 template <
class Packet>
1056 const Teuchos::ArrayView<Packet>& imports)
1058 using Teuchos::arcp;
1059 using Teuchos::ArrayRCP;
1060 typedef typename ArrayRCP<const Packet>::size_type size_type;
1062 TEUCHOS_TEST_FOR_EXCEPTION(
1063 requests_.size () != 0, std::runtime_error,
"Tpetra::Distributor::" 1064 "doPostsAndWaits(3 args): There are " << requests_.size () <<
1065 " outstanding nonblocking messages pending. It is incorrect to call " 1066 "this method with posts outstanding.");
1078 ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (),
1079 static_cast<size_type
> (0),
1080 exports.size(),
false);
1095 arcp<Packet> (imports.getRawPtr (), 0, imports.size (),
false));
1098 lastRoundBytesSend_ = exports.size () *
sizeof (Packet);
1099 lastRoundBytesRecv_ = imports.size () *
sizeof (Packet);
1102 template <
class Packet>
1105 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1106 const Teuchos::ArrayView<Packet> &imports,
1107 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1109 using Teuchos::arcp;
1110 using Teuchos::ArrayRCP;
1112 TEUCHOS_TEST_FOR_EXCEPTION(
1113 requests_.size () != 0, std::runtime_error,
"Tpetra::Distributor::" 1114 "doPostsAndWaits: There are " << requests_.size () <<
" outstanding " 1115 "nonblocking messages pending. It is incorrect to call doPostsAndWaits " 1116 "with posts outstanding.");
1129 typedef typename ArrayRCP<const Packet>::size_type size_type;
1130 ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (),
1131 static_cast<size_type
> (0),
1132 exports.size (),
false);
1138 numExportPacketsPerLID,
1139 arcp<Packet> (imports.getRawPtr (), 0, imports.size (),
false),
1140 numImportPacketsPerLID);
1143 lastRoundBytesSend_ = exports.size () *
sizeof (Packet);
1144 lastRoundBytesRecv_ = imports.size () *
sizeof (Packet);
1148 template <
class Packet>
1150 doPosts (
const Teuchos::ArrayRCP<const Packet>& exports,
1152 const Teuchos::ArrayRCP<Packet>& imports)
1154 using Teuchos::Array;
1155 using Teuchos::ArrayRCP;
1156 using Teuchos::ArrayView;
1158 using Teuchos::FancyOStream;
1159 using Teuchos::includesVerbLevel;
1160 using Teuchos::ireceive;
1161 using Teuchos::isend;
1162 using Teuchos::OSTab;
1163 using Teuchos::readySend;
1164 using Teuchos::send;
1165 using Teuchos::ssend;
1166 using Teuchos::TypeNameTraits;
1167 using Teuchos::typeName;
1169 typedef Array<size_t>::size_type size_type;
1171 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1172 Teuchos::TimeMonitor timeMon (*timer_doPosts3_);
1173 #endif // TPETRA_DISTRIBUTOR_TIMERS 1175 const int myRank = comm_->getRank ();
1179 const bool doBarrier = barrierBetween_;
1181 Teuchos::OSTab tab0 (out_);
1183 std::ostringstream os;
1184 os <<
"Proc " << myRank <<
": Distributor::doPosts(3 args, Teuchos::ArrayRCP)" << endl;
1187 Teuchos::OSTab tab1 (out_);
1189 TEUCHOS_TEST_FOR_EXCEPTION(
1190 sendType == Details::DISTRIBUTOR_RSEND && ! doBarrier, std::logic_error,
1191 "Tpetra::Distributor::doPosts(3 args): Ready-send version requires a " 1192 "barrier between posting receives and posting ready sends. This should " 1193 "have been checked before. " 1194 "Please report this bug to the Tpetra developers.");
1196 size_t selfReceiveOffset = 0;
1201 if (howInitialized_ != Details::DISTRIBUTOR_INITIALIZED_BY_REVERSE) {
1208 const size_t totalNumImportPackets = totalReceiveLength_ * numPackets;
1209 TEUCHOS_TEST_FOR_EXCEPTION
1210 (static_cast<size_t> (imports.size ()) < totalNumImportPackets,
1211 std::invalid_argument,
"Tpetra::Distributor::doPosts(3 args): The " 1212 "'imports' array must have enough entries to hold the expected number " 1213 "of import packets. imports.size() = " << imports.size () <<
" < " 1214 "totalNumImportPackets = " << totalNumImportPackets <<
".");
1222 const int pathTag = 0;
1223 const int tag = this->getTag (pathTag);
1225 #ifdef HAVE_TPETRA_DEBUG 1226 TEUCHOS_TEST_FOR_EXCEPTION
1227 (requests_.size () != 0, std::logic_error,
"Tpetra::Distributor::" 1228 "doPosts(3 args, Kokkos): Process " << myRank <<
": requests_.size() = " 1229 << requests_.size () <<
" != 0.");
1230 #endif // HAVE_TPETRA_DEBUG 1245 const size_type actualNumReceives = as<size_type> (numReceives_) +
1246 as<size_type> (selfMessage_ ? 1 : 0);
1247 requests_.resize (0);
1250 std::ostringstream os;
1251 os <<
"Proc " << myRank <<
": doPosts(3," 1252 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Post receives" 1263 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1264 Teuchos::TimeMonitor timeMonRecvs (*timer_doPosts3_recvs_);
1265 #endif // TPETRA_DISTRIBUTOR_TIMERS 1267 size_t curBufOffset = 0;
1268 for (size_type i = 0; i < actualNumReceives; ++i) {
1269 const size_t curBufLen = lengthsFrom_[i] * numPackets;
1270 if (procsFrom_[i] != myRank) {
1272 std::ostringstream os;
1273 os <<
"Proc " << myRank <<
": doPosts(3," 1274 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): " 1275 <<
"Post irecv: {source: " << procsFrom_[i]
1276 <<
", tag: " << tag <<
"}" << endl;
1286 TEUCHOS_TEST_FOR_EXCEPTION(
1287 curBufOffset + curBufLen > static_cast<size_t> (imports.size ()),
1288 std::logic_error,
"Tpetra::Distributor::doPosts(3 args, Teuchos): " 1289 "Exceeded size of 'imports' array in packing loop on Process " <<
1290 myRank <<
". imports.size() = " << imports.size () <<
" < " 1291 "curBufOffset(" << curBufOffset <<
") + curBufLen(" << curBufLen
1293 ArrayRCP<Packet> recvBuf =
1294 imports.persistingView (curBufOffset, curBufLen);
1295 requests_.push_back (ireceive<int, Packet> (recvBuf, procsFrom_[i],
1299 selfReceiveOffset = curBufOffset;
1301 curBufOffset += curBufLen;
1306 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1307 Teuchos::TimeMonitor timeMonBarrier (*timer_doPosts3_barrier_);
1308 #endif // TPETRA_DISTRIBUTOR_TIMERS 1311 std::ostringstream os;
1312 os <<
"Proc " << myRank <<
": doPosts(3," 1313 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Barrier" << endl;
1324 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1325 Teuchos::TimeMonitor timeMonSends (*timer_doPosts3_sends_);
1326 #endif // TPETRA_DISTRIBUTOR_TIMERS 1333 size_t numBlocks = numSends_ + selfMessage_;
1334 size_t procIndex = 0;
1335 while ((procIndex < numBlocks) && (procsTo_[procIndex] < myRank)) {
1338 if (procIndex == numBlocks) {
1343 size_t selfIndex = 0;
1346 std::ostringstream os;
1347 os <<
"Proc " << myRank <<
": doPosts(3," 1348 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Post sends" << endl;
1352 if (indicesTo_.empty()) {
1354 std::ostringstream os;
1355 os << myRank <<
": doPosts(3,fast): posting sends" << endl;
1361 for (
size_t i = 0; i < numBlocks; ++i) {
1362 size_t p = i + procIndex;
1363 if (p > (numBlocks - 1)) {
1367 if (procsTo_[p] != myRank) {
1369 std::ostringstream os;
1370 os <<
"Proc " << myRank <<
": doPosts(3,fast): Post send: " 1371 "{target: " << procsTo_[p] <<
", tag: " << tag <<
"}" << endl;
1375 ArrayView<const Packet> tmpSend =
1376 exports.view (startsTo_[p]*numPackets, lengthsTo_[p]*numPackets);
1378 if (sendType == Details::DISTRIBUTOR_SEND) {
1379 send<int, Packet> (tmpSend.getRawPtr (),
1380 as<int> (tmpSend.size ()),
1381 procsTo_[p], tag, *comm_);
1383 else if (sendType == Details::DISTRIBUTOR_ISEND) {
1384 ArrayRCP<const Packet> tmpSendBuf =
1385 exports.persistingView (startsTo_[p] * numPackets,
1386 lengthsTo_[p] * numPackets);
1387 requests_.push_back (isend<int, Packet> (tmpSendBuf, procsTo_[p],
1390 else if (sendType == Details::DISTRIBUTOR_RSEND) {
1391 readySend<int, Packet> (tmpSend.getRawPtr (),
1392 as<int> (tmpSend.size ()),
1393 procsTo_[p], tag, *comm_);
1395 else if (sendType == Details::DISTRIBUTOR_SSEND) {
1396 ssend<int, Packet> (tmpSend.getRawPtr (),
1397 as<int> (tmpSend.size ()),
1398 procsTo_[p], tag, *comm_);
1400 TEUCHOS_TEST_FOR_EXCEPTION(
1401 true, std::logic_error,
"Tpetra::Distributor::doPosts(3 args): " 1402 "Invalid send type. We should never get here. " 1403 "Please report this bug to the Tpetra developers.");
1413 std::ostringstream os;
1414 os <<
"Proc " << myRank <<
": doPosts(3,fast): Self-send" << endl;
1424 std::copy (exports.begin()+startsTo_[selfNum]*numPackets,
1425 exports.begin()+startsTo_[selfNum]*numPackets+lengthsTo_[selfNum]*numPackets,
1426 imports.begin()+selfReceiveOffset);
1429 std::ostringstream os;
1430 os << myRank <<
": doPosts(3,fast) done" << endl;
1436 std::ostringstream os;
1437 os << myRank <<
": doPosts(3,slow): posting sends" << endl;
1443 ArrayRCP<Packet> sendArray (maxSendLength_ * numPackets);
1445 TEUCHOS_TEST_FOR_EXCEPTION(
1446 sendType == Details::DISTRIBUTOR_ISEND, std::logic_error,
1447 "Tpetra::Distributor::doPosts(3 args): The \"send buffer\" code path " 1448 <<
"doesn't currently work with nonblocking sends.");
1450 for (
size_t i = 0; i < numBlocks; ++i) {
1451 size_t p = i + procIndex;
1452 if (p > (numBlocks - 1)) {
1456 if (procsTo_[p] != myRank) {
1458 std::ostringstream os;
1459 os <<
"Proc " << myRank <<
": doPosts(3,slow): Post send: " 1460 "{target: " << procsTo_[p] <<
", tag: " << tag <<
"}" << endl;
1464 typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
1465 size_t sendArrayOffset = 0;
1466 size_t j = startsTo_[p];
1467 for (
size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
1468 srcBegin = exports.begin() + indicesTo_[j]*numPackets;
1469 srcEnd = srcBegin + numPackets;
1470 std::copy (srcBegin, srcEnd, sendArray.begin()+sendArrayOffset);
1471 sendArrayOffset += numPackets;
1473 ArrayView<const Packet> tmpSend =
1474 sendArray.view (0, lengthsTo_[p]*numPackets);
1476 if (sendType == Details::DISTRIBUTOR_SEND) {
1477 send<int, Packet> (tmpSend.getRawPtr (),
1478 as<int> (tmpSend.size ()),
1479 procsTo_[p], tag, *comm_);
1481 else if (sendType == Details::DISTRIBUTOR_ISEND) {
1482 ArrayRCP<const Packet> tmpSendBuf =
1483 sendArray.persistingView (0, lengthsTo_[p] * numPackets);
1484 requests_.push_back (isend<int, Packet> (tmpSendBuf, procsTo_[p],
1487 else if (sendType == Details::DISTRIBUTOR_RSEND) {
1488 readySend<int, Packet> (tmpSend.getRawPtr (),
1489 as<int> (tmpSend.size ()),
1490 procsTo_[p], tag, *comm_);
1492 else if (sendType == Details::DISTRIBUTOR_SSEND) {
1493 ssend<int, Packet> (tmpSend.getRawPtr (),
1494 as<int> (tmpSend.size ()),
1495 procsTo_[p], tag, *comm_);
1498 TEUCHOS_TEST_FOR_EXCEPTION(
1499 true, std::logic_error,
"Tpetra::Distributor::doPosts(3 args): " 1500 "Invalid send type. We should never get here. " 1501 "Please report this bug to the Tpetra developers.");
1506 selfIndex = startsTo_[p];
1512 std::ostringstream os;
1513 os <<
"Proc " << myRank <<
": doPosts(3,slow): Self-send" << endl;
1516 for (
size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
1517 std::copy (exports.begin()+indicesTo_[selfIndex]*numPackets,
1518 exports.begin()+indicesTo_[selfIndex]*numPackets + numPackets,
1519 imports.begin() + selfReceiveOffset);
1521 selfReceiveOffset += numPackets;
1525 std::ostringstream os;
1526 os << myRank <<
": doPosts(3,slow) done" << endl;
1532 std::ostringstream os;
1533 os <<
"Proc " << myRank <<
": doPosts done" << endl;
1538 template <
class Packet>
1540 doPosts (
const Teuchos::ArrayRCP<const Packet>& exports,
1541 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1542 const Teuchos::ArrayRCP<Packet>& imports,
1543 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1545 using Teuchos::Array;
1546 using Teuchos::ArrayRCP;
1547 using Teuchos::ArrayView;
1549 using Teuchos::ireceive;
1550 using Teuchos::isend;
1551 using Teuchos::readySend;
1552 using Teuchos::send;
1553 using Teuchos::ssend;
1554 using Teuchos::TypeNameTraits;
1555 #ifdef HAVE_TEUCHOS_DEBUG 1556 using Teuchos::OSTab;
1557 #endif // HAVE_TEUCHOS_DEBUG 1559 typedef Array<size_t>::size_type size_type;
1561 Teuchos::OSTab tab (out_);
1563 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1564 Teuchos::TimeMonitor timeMon (*timer_doPosts4_);
1565 #endif // TPETRA_DISTRIBUTOR_TIMERS 1570 const bool doBarrier = barrierBetween_;
1596 TEUCHOS_TEST_FOR_EXCEPTION(
1597 sendType == Details::DISTRIBUTOR_RSEND && ! doBarrier, std::logic_error,
1598 "Tpetra::Distributor::doPosts(4 args): Ready-send version requires a " 1599 "barrier between posting receives and posting ready sends. This should " 1600 "have been checked before. " 1601 "Please report this bug to the Tpetra developers.");
1603 const int myProcID = comm_->getRank ();
1604 size_t selfReceiveOffset = 0;
1606 #ifdef HAVE_TEUCHOS_DEBUG 1608 size_t totalNumImportPackets = 0;
1609 for (
size_t ii = 0; ii < static_cast<size_t> (numImportPacketsPerLID.size ()); ++ii) {
1610 totalNumImportPackets += numImportPacketsPerLID[ii];
1612 TEUCHOS_TEST_FOR_EXCEPTION(
1613 static_cast<size_t> (imports.size ()) < totalNumImportPackets,
1614 std::runtime_error,
"Tpetra::Distributor::doPosts(4 args): The 'imports' " 1615 "array must have enough entries to hold the expected number of import " 1616 "packets. imports.size() = " << imports.size() <<
" < " 1617 "totalNumImportPackets = " << totalNumImportPackets <<
".");
1618 #endif // HAVE_TEUCHOS_DEBUG 1625 const int pathTag = 1;
1626 const int tag = this->getTag (pathTag);
1629 TEUCHOS_TEST_FOR_EXCEPTION(
1630 requests_.size () != 0, std::logic_error,
"Tpetra::Distributor::" 1631 "doPosts(4 args): Process " << myProcID <<
": requests_.size() = " 1632 << requests_.size () <<
" != 0.");
1633 std::ostringstream os;
1634 os << myProcID <<
": doPosts(4," 1635 << (indicesTo_.empty () ?
"fast" :
"slow") <<
")" << endl;
1652 const size_type actualNumReceives = as<size_type> (numReceives_) +
1653 as<size_type> (selfMessage_ ? 1 : 0);
1654 requests_.resize (0);
1662 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1663 Teuchos::TimeMonitor timeMonRecvs (*timer_doPosts4_recvs_);
1664 #endif // TPETRA_DISTRIBUTOR_TIMERS 1666 size_t curBufferOffset = 0;
1667 size_t curLIDoffset = 0;
1668 for (size_type i = 0; i < actualNumReceives; ++i) {
1669 size_t totalPacketsFrom_i = 0;
1670 for (
size_t j = 0; j < lengthsFrom_[i]; ++j) {
1671 totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
1673 curLIDoffset += lengthsFrom_[i];
1674 if (procsFrom_[i] != myProcID && totalPacketsFrom_i) {
1683 ArrayRCP<Packet> recvBuf =
1684 imports.persistingView (curBufferOffset, totalPacketsFrom_i);
1685 requests_.push_back (ireceive<int, Packet> (recvBuf, procsFrom_[i],
1689 selfReceiveOffset = curBufferOffset;
1691 curBufferOffset += totalPacketsFrom_i;
1696 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1697 Teuchos::TimeMonitor timeMonBarrier (*timer_doPosts4_barrier_);
1698 #endif // TPETRA_DISTRIBUTOR_TIMERS 1707 #ifdef TPETRA_DISTRIBUTOR_TIMERS 1708 Teuchos::TimeMonitor timeMonSends (*timer_doPosts4_sends_);
1709 #endif // TPETRA_DISTRIBUTOR_TIMERS 1713 Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
1714 size_t maxNumPackets = 0;
1715 size_t curPKToffset = 0;
1716 for (
size_t pp=0; pp<numSends_; ++pp) {
1717 sendPacketOffsets[pp] = curPKToffset;
1718 size_t numPackets = 0;
1719 for (
size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
1720 numPackets += numExportPacketsPerLID[j];
1722 if (numPackets > maxNumPackets) maxNumPackets = numPackets;
1723 packetsPerSend[pp] = numPackets;
1724 curPKToffset += numPackets;
1729 size_t numBlocks = numSends_+ selfMessage_;
1730 size_t procIndex = 0;
1731 while ((procIndex < numBlocks) && (procsTo_[procIndex] < myProcID)) {
1734 if (procIndex == numBlocks) {
1739 size_t selfIndex = 0;
1741 if (indicesTo_.empty()) {
1743 std::ostringstream os;
1744 os << myProcID <<
": doPosts(4,fast): posting sends" << endl;
1750 for (
size_t i = 0; i < numBlocks; ++i) {
1751 size_t p = i + procIndex;
1752 if (p > (numBlocks - 1)) {
1756 if (procsTo_[p] != myProcID && packetsPerSend[p] > 0) {
1757 ArrayView<const Packet> tmpSend =
1758 exports.view (sendPacketOffsets[p], packetsPerSend[p]);
1760 if (sendType == Details::DISTRIBUTOR_SEND) {
1761 send<int, Packet> (tmpSend.getRawPtr (),
1762 as<int> (tmpSend.size ()),
1763 procsTo_[p], tag, *comm_);
1765 else if (sendType == Details::DISTRIBUTOR_RSEND) {
1766 readySend<int, Packet> (tmpSend.getRawPtr (),
1767 as<int> (tmpSend.size ()),
1768 procsTo_[p], tag, *comm_);
1770 else if (sendType == Details::DISTRIBUTOR_ISEND) {
1771 ArrayRCP<const Packet> tmpSendBuf =
1772 exports.persistingView (sendPacketOffsets[p], packetsPerSend[p]);
1773 requests_.push_back (isend<int, Packet> (tmpSendBuf, procsTo_[p],
1776 else if (sendType == Details::DISTRIBUTOR_SSEND) {
1777 ssend<int, Packet> (tmpSend.getRawPtr (),
1778 as<int> (tmpSend.size ()),
1779 procsTo_[p], tag, *comm_);
1782 TEUCHOS_TEST_FOR_EXCEPTION(
1783 true, std::logic_error,
"Tpetra::Distributor::doPosts(4 args): " 1784 "Invalid send type. We should never get here. Please report " 1785 "this bug to the Tpetra developers.");
1794 std::copy (exports.begin()+sendPacketOffsets[selfNum],
1795 exports.begin()+sendPacketOffsets[selfNum]+packetsPerSend[selfNum],
1796 imports.begin()+selfReceiveOffset);
1799 std::ostringstream os;
1800 os << myProcID <<
": doPosts(4,fast) done" << endl;
1806 std::ostringstream os;
1807 os << myProcID <<
": doPosts(4,slow): posting sends" << endl;
1812 ArrayRCP<Packet> sendArray (maxNumPackets);
1814 TEUCHOS_TEST_FOR_EXCEPTION(
1815 sendType == Details::DISTRIBUTOR_ISEND, std::logic_error,
1816 "Tpetra::Distributor::doPosts(3 args): The \"send buffer\" " 1817 "code path may not necessarily work with nonblocking sends.");
1819 Array<size_t> indicesOffsets (numExportPacketsPerLID.size(), 0);
1821 for (
int j=0; j<numExportPacketsPerLID.size(); ++j) {
1822 indicesOffsets[j] = ioffset;
1823 ioffset += numExportPacketsPerLID[j];
1826 for (
size_t i = 0; i < numBlocks; ++i) {
1827 size_t p = i + procIndex;
1828 if (p > (numBlocks - 1)) {
1832 if (procsTo_[p] != myProcID) {
1833 typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
1834 size_t sendArrayOffset = 0;
1835 size_t j = startsTo_[p];
1836 size_t numPacketsTo_p = 0;
1837 for (
size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
1838 srcBegin = exports.begin() + indicesOffsets[j];
1839 srcEnd = srcBegin + numExportPacketsPerLID[j];
1840 numPacketsTo_p += numExportPacketsPerLID[j];
1841 std::copy (srcBegin, srcEnd, sendArray.begin()+sendArrayOffset);
1842 sendArrayOffset += numExportPacketsPerLID[j];
1844 if (numPacketsTo_p > 0) {
1845 ArrayView<const Packet> tmpSend =
1846 sendArray.view (0, numPacketsTo_p);
1848 if (sendType == Details::DISTRIBUTOR_RSEND) {
1849 readySend<int, Packet> (tmpSend.getRawPtr (),
1850 as<int> (tmpSend.size ()),
1851 procsTo_[p], tag, *comm_);
1853 else if (sendType == Details::DISTRIBUTOR_ISEND) {
1854 ArrayRCP<const Packet> tmpSendBuf =
1855 sendArray.persistingView (0, numPacketsTo_p);
1856 requests_.push_back (isend<int, Packet> (tmpSendBuf, procsTo_[p],
1859 else if (sendType == Details::DISTRIBUTOR_SSEND) {
1860 ssend<int, Packet> (tmpSend.getRawPtr (),
1861 as<int> (tmpSend.size ()),
1862 procsTo_[p], tag, *comm_);
1865 send<int, Packet> (tmpSend.getRawPtr (),
1866 as<int> (tmpSend.size ()),
1867 procsTo_[p], tag, *comm_);
1873 selfIndex = startsTo_[p];
1878 for (
size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
1879 std::copy (exports.begin()+indicesOffsets[selfIndex],
1880 exports.begin()+indicesOffsets[selfIndex]+numExportPacketsPerLID[selfIndex],
1881 imports.begin() + selfReceiveOffset);
1882 selfReceiveOffset += numExportPacketsPerLID[selfIndex];
1887 std::ostringstream os;
1888 os << myProcID <<
": doPosts(4,slow) done" << endl;
1894 template <
class Packet>
1898 const Teuchos::ArrayView<Packet>& imports)
1900 using Teuchos::arcp;
1901 using Teuchos::ArrayRCP;
1914 typedef typename ArrayRCP<const Packet>::size_type size_type;
1915 ArrayRCP<const Packet> exportsArcp (exports.getRawPtr(), as<size_type> (0),
1916 exports.size(),
false);
1923 arcp<Packet> (imports.getRawPtr (), 0, imports.size (),
false));
1926 lastRoundBytesSend_ = exports.size() *
sizeof(Packet);
1927 lastRoundBytesRecv_ = imports.size() *
sizeof(Packet);
1930 template <
class Packet>
1933 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1934 const Teuchos::ArrayView<Packet> &imports,
1935 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1938 using Teuchos::arcp;
1939 using Teuchos::ArrayRCP;
1941 TEUCHOS_TEST_FOR_EXCEPTION(
1942 requests_.size () != 0, std::runtime_error,
"Tpetra::Distributor::" 1943 "doReversePostsAndWaits(4 args): There are " << requests_.size ()
1944 <<
" outstanding nonblocking messages pending. It is incorrect to call " 1945 "this method with posts outstanding.");
1958 typedef typename ArrayRCP<const Packet>::size_type size_type;
1959 ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (), as<size_type> (0),
1960 exports.size (),
false);
1962 numExportPacketsPerLID,
1963 arcp<Packet> (imports.getRawPtr (), 0, imports.size (),
false),
1964 numImportPacketsPerLID);
1967 lastRoundBytesSend_ = exports.size() *
sizeof(Packet);
1968 lastRoundBytesRecv_ = imports.size() *
sizeof(Packet);
1971 template <
class Packet>
1975 const Teuchos::ArrayRCP<Packet>& imports)
1978 TEUCHOS_TEST_FOR_EXCEPTION(
1979 ! indicesTo_.empty (), std::runtime_error,
1980 "Tpetra::Distributor::doReversePosts(3 args): Can only do reverse " 1981 "communication when original data are blocked by process.");
1982 if (reverseDistributor_.is_null ()) {
1983 createReverseDistributor ();
1985 reverseDistributor_->doPosts (exports, numPackets, imports);
1988 template <
class Packet>
1991 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1992 const Teuchos::ArrayRCP<Packet>& imports,
1993 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1996 TEUCHOS_TEST_FOR_EXCEPTION(
1997 ! indicesTo_.empty (), std::runtime_error,
1998 "Tpetra::Distributor::doReversePosts(3 args): Can only do reverse " 1999 "communication when original data are blocked by process.");
2000 if (reverseDistributor_.is_null ()) {
2001 createReverseDistributor ();
2003 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
2004 imports, numImportPacketsPerLID);
2007 template <
class ExpView,
class ImpView>
2008 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2010 doPostsAndWaits (
const ExpView& exports,
2012 const ImpView& imports)
2018 RCP<Teuchos::OSTab> tab0, tab1;
2020 tab0 = rcp (
new Teuchos::OSTab (out_));
2021 const int myRank = comm_->getRank ();
2022 std::ostringstream os;
2023 os <<
"Proc " << myRank
2024 <<
": Distributor::doPostsAndWaits(3 args, Kokkos): " 2025 <<
"{sendType: " << DistributorSendTypeEnumToString (sendType_)
2026 <<
", barrierBetween: " << barrierBetween_ <<
"}" << endl;
2028 tab1 = rcp (
new Teuchos::OSTab (out_));
2031 TEUCHOS_TEST_FOR_EXCEPTION(
2032 requests_.size () != 0, std::runtime_error,
"Tpetra::Distributor::" 2033 "doPostsAndWaits(3 args): There are " << requests_.size () <<
2034 " outstanding nonblocking messages pending. It is incorrect to call " 2035 "this method with posts outstanding.");
2038 const int myRank = comm_->getRank ();
2039 std::ostringstream os;
2040 os <<
"Proc " << myRank
2041 <<
": Distributor::doPostsAndWaits: Call doPosts" << endl;
2044 doPosts (exports, numPackets, imports);
2046 const int myRank = comm_->getRank ();
2047 std::ostringstream os;
2048 os <<
"Proc " << myRank
2049 <<
": Distributor::doPostsAndWaits: Call doWaits" << endl;
2055 template <
class ExpView,
class ImpView>
2056 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2058 doPostsAndWaits (
const ExpView& exports,
2059 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
2060 const ImpView& imports,
2061 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
2063 TEUCHOS_TEST_FOR_EXCEPTION(
2064 requests_.size () != 0, std::runtime_error,
2065 "Tpetra::Distributor::doPostsAndWaits(4 args): There are " 2066 << requests_.size () <<
" outstanding nonblocking messages pending. " 2067 "It is incorrect to call this method with posts outstanding.");
2069 doPosts (exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
2074 template <
class ExpView,
class ImpView>
2075 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2077 doPosts (
const ExpView &exports,
2079 const ImpView &imports)
2081 using Teuchos::Array;
2083 using Teuchos::FancyOStream;
2084 using Teuchos::includesVerbLevel;
2085 using Teuchos::ireceive;
2086 using Teuchos::isend;
2087 using Teuchos::OSTab;
2088 using Teuchos::readySend;
2089 using Teuchos::send;
2090 using Teuchos::ssend;
2091 using Teuchos::TypeNameTraits;
2092 using Teuchos::typeName;
2094 using Kokkos::Compat::create_const_view;
2095 using Kokkos::Compat::create_view;
2096 using Kokkos::Compat::subview_offset;
2097 using Kokkos::Compat::deep_copy_offset;
2098 typedef Array<size_t>::size_type size_type;
2099 typedef ExpView exports_view_type;
2100 typedef ImpView imports_view_type;
2102 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2103 Teuchos::TimeMonitor timeMon (*timer_doPosts3_);
2104 #endif // TPETRA_DISTRIBUTOR_TIMERS 2106 const int myRank = comm_->getRank ();
2110 const bool doBarrier = barrierBetween_;
2112 Teuchos::OSTab tab0 (out_);
2114 std::ostringstream os;
2115 os <<
"Proc " << myRank <<
": Distributor::doPosts(3 args, Kokkos)" << endl;
2118 Teuchos::OSTab tab1 (out_);
2120 TEUCHOS_TEST_FOR_EXCEPTION(
2121 sendType == Details::DISTRIBUTOR_RSEND && ! doBarrier, std::logic_error,
2122 "Tpetra::Distributor::doPosts(3 args): Ready-send version requires a " 2123 "barrier between posting receives and posting ready sends. This should " 2124 "have been checked before. " 2125 "Please report this bug to the Tpetra developers.");
2127 size_t selfReceiveOffset = 0;
2134 const size_t totalNumImportPackets = totalReceiveLength_ * numPackets;
2137 std::ostringstream os;
2138 os <<
"Proc " << myRank <<
": doPosts: totalNumImportPackets = " <<
2139 totalNumImportPackets <<
" = " << totalReceiveLength_ <<
" * " <<
2140 numPackets <<
"; imports.dimension_0() = " << imports.dimension_0 ()
2145 #ifdef HAVE_TPETRA_DEBUG 2148 const size_t importBufSize =
static_cast<size_t> (imports.dimension_0 ());
2149 const int lclBad = (importBufSize < totalNumImportPackets) ? 1 : 0;
2151 using Teuchos::reduceAll;
2152 using Teuchos::REDUCE_MAX;
2153 using Teuchos::outArg;
2154 reduceAll (*comm_, REDUCE_MAX, lclBad, outArg (gblBad));
2155 TEUCHOS_TEST_FOR_EXCEPTION
2156 (gblBad != 0, std::runtime_error,
"Tpetra::Distributor::doPosts(3 " 2157 "args, Kokkos): On one or more MPI processes, the 'imports' array " 2158 "does not have enough entries to hold the expected number of " 2159 "import packets. ");
2162 TEUCHOS_TEST_FOR_EXCEPTION
2163 (static_cast<size_t> (imports.dimension_0 ()) < totalNumImportPackets,
2164 std::runtime_error,
"Tpetra::Distributor::doPosts(3 args): The 'imports' " 2165 "array must have enough entries to hold the expected number of import " 2166 "packets. imports.dimension_0() = " << imports.dimension_0 () <<
" < " 2167 "totalNumImportPackets = " << totalNumImportPackets <<
" = " 2168 "totalReceiveLength_ (" << totalReceiveLength_ <<
") * numPackets (" 2169 << numPackets <<
").");
2170 #endif // HAVE_TPETRA_DEBUG 2178 const int pathTag = 0;
2179 const int tag = this->getTag (pathTag);
2181 #ifdef HAVE_TPETRA_DEBUG 2182 TEUCHOS_TEST_FOR_EXCEPTION
2183 (requests_.size () != 0, std::logic_error,
"Tpetra::Distributor::" 2184 "doPosts(3 args, Kokkos): Process " << myRank <<
": requests_.size() = " 2185 << requests_.size () <<
" != 0.");
2186 #endif // HAVE_TPETRA_DEBUG 2201 const size_type actualNumReceives = as<size_type> (numReceives_) +
2202 as<size_type> (selfMessage_ ? 1 : 0);
2203 requests_.resize (0);
2206 std::ostringstream os;
2207 os <<
"Proc " << myRank <<
": doPosts(3," 2208 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Post receives" 2219 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2220 Teuchos::TimeMonitor timeMonRecvs (*timer_doPosts3_recvs_);
2221 #endif // TPETRA_DISTRIBUTOR_TIMERS 2223 size_t curBufferOffset = 0;
2224 for (size_type i = 0; i < actualNumReceives; ++i) {
2225 const size_t curBufLen = lengthsFrom_[i] * numPackets;
2226 if (procsFrom_[i] != myRank) {
2228 std::ostringstream os;
2229 os <<
"Proc " << myRank <<
": doPosts(3," 2230 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): " 2231 <<
"Post irecv: {source: " << procsFrom_[i]
2232 <<
", tag: " << tag <<
"}" << endl;
2242 TEUCHOS_TEST_FOR_EXCEPTION(
2243 curBufferOffset + curBufLen > static_cast<size_t> (imports.size ()),
2244 std::logic_error,
"Tpetra::Distributor::doPosts(3 args, Kokkos): " 2245 "Exceeded size of 'imports' array in packing loop on Process " <<
2246 myRank <<
". imports.size() = " << imports.size () <<
" < " 2247 "curBufferOffset(" << curBufferOffset <<
") + curBufLen(" <<
2249 imports_view_type recvBuf =
2250 subview_offset (imports, curBufferOffset, curBufLen);
2251 requests_.push_back (ireceive<int> (recvBuf, procsFrom_[i],
2255 selfReceiveOffset = curBufferOffset;
2257 curBufferOffset += curBufLen;
2262 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2263 Teuchos::TimeMonitor timeMonBarrier (*timer_doPosts3_barrier_);
2264 #endif // TPETRA_DISTRIBUTOR_TIMERS 2267 std::ostringstream os;
2268 os <<
"Proc " << myRank <<
": doPosts(3," 2269 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Barrier" << endl;
2280 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2281 Teuchos::TimeMonitor timeMonSends (*timer_doPosts3_sends_);
2282 #endif // TPETRA_DISTRIBUTOR_TIMERS 2289 size_t numBlocks = numSends_ + selfMessage_;
2290 size_t procIndex = 0;
2291 while ((procIndex < numBlocks) && (procsTo_[procIndex] < myRank)) {
2294 if (procIndex == numBlocks) {
2299 size_t selfIndex = 0;
2302 std::ostringstream os;
2303 os <<
"Proc " << myRank <<
": doPosts(3," 2304 << (indicesTo_.empty () ?
"fast" :
"slow") <<
"): Post sends" << endl;
2308 if (indicesTo_.empty()) {
2310 std::ostringstream os;
2311 os <<
"Proc " << myRank <<
": doPosts(3,fast): posting sends" << endl;
2317 for (
size_t i = 0; i < numBlocks; ++i) {
2318 size_t p = i + procIndex;
2319 if (p > (numBlocks - 1)) {
2323 if (procsTo_[p] != myRank) {
2325 std::ostringstream os;
2326 os <<
"Proc " << myRank <<
": doPosts(3,fast): Post send: " 2327 "{target: " << procsTo_[p] <<
", tag: " << tag <<
"}" << endl;
2340 exports_view_type tmpSend = subview_offset(
2341 exports, startsTo_[p]*numPackets, lengthsTo_[p]*numPackets);
2343 if (sendType == Details::DISTRIBUTOR_SEND) {
2345 as<int> (tmpSend.size ()),
2346 procsTo_[p], tag, *comm_);
2348 else if (sendType == Details::DISTRIBUTOR_ISEND) {
2349 exports_view_type tmpSendBuf =
2350 subview_offset (exports, startsTo_[p] * numPackets,
2351 lengthsTo_[p] * numPackets);
2352 requests_.push_back (isend<int> (tmpSendBuf, procsTo_[p],
2355 else if (sendType == Details::DISTRIBUTOR_RSEND) {
2356 readySend<int> (tmpSend,
2357 as<int> (tmpSend.size ()),
2358 procsTo_[p], tag, *comm_);
2360 else if (sendType == Details::DISTRIBUTOR_SSEND) {
2361 ssend<int> (tmpSend,
2362 as<int> (tmpSend.size ()),
2363 procsTo_[p], tag, *comm_);
2365 TEUCHOS_TEST_FOR_EXCEPTION(
2366 true, std::logic_error,
"Tpetra::Distributor::doPosts(3 args): " 2367 "Invalid send type. We should never get here. " 2368 "Please report this bug to the Tpetra developers.");
2378 std::ostringstream os;
2379 os <<
"Proc " << myRank <<
": doPosts(3,fast): Self-send" << endl;
2389 deep_copy_offset(imports, exports, selfReceiveOffset,
2390 startsTo_[selfNum]*numPackets,
2391 lengthsTo_[selfNum]*numPackets);
2394 std::ostringstream os;
2395 os <<
"Proc " << myRank <<
": doPosts(3,fast) done" << endl;
2401 std::ostringstream os;
2402 os <<
"Proc " << myRank <<
": doPosts(3,slow): posting sends" << endl;
2406 typedef typename ExpView::non_const_value_type Packet;
2407 typedef typename ExpView::array_layout Layout;
2408 typedef typename ExpView::device_type Device;
2409 typedef typename ExpView::memory_traits Mem;
2410 Kokkos::View<Packet*,Layout,Device,Mem> sendArray (
"sendArray",
2411 maxSendLength_ * numPackets);
2415 TEUCHOS_TEST_FOR_EXCEPTION(
2416 sendType == Details::DISTRIBUTOR_ISEND, std::logic_error,
2417 "Tpetra::Distributor::doPosts(3 args): The \"send buffer\" code path " 2418 "doesn't currently work with nonblocking sends.");
2420 for (
size_t i = 0; i < numBlocks; ++i) {
2421 size_t p = i + procIndex;
2422 if (p > (numBlocks - 1)) {
2426 if (procsTo_[p] != myRank) {
2428 std::ostringstream os;
2429 os <<
"Proc " << myRank <<
": doPosts(3,slow): Post send: " 2430 "{target: " << procsTo_[p] <<
", tag: " << tag <<
"}" << endl;
2434 size_t sendArrayOffset = 0;
2435 size_t j = startsTo_[p];
2436 for (
size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
2437 deep_copy_offset(sendArray, exports, sendArrayOffset,
2438 indicesTo_[j]*numPackets, numPackets);
2439 sendArrayOffset += numPackets;
2442 subview_offset(sendArray,
size_t(0), lengthsTo_[p]*numPackets);
2444 if (sendType == Details::DISTRIBUTOR_SEND) {
2446 as<int> (tmpSend.size ()),
2447 procsTo_[p], tag, *comm_);
2449 else if (sendType == Details::DISTRIBUTOR_ISEND) {
2450 exports_view_type tmpSendBuf =
2451 subview_offset (sendArray,
size_t(0), lengthsTo_[p] * numPackets);
2452 requests_.push_back (isend<int> (tmpSendBuf, procsTo_[p],
2455 else if (sendType == Details::DISTRIBUTOR_RSEND) {
2456 readySend<int> (tmpSend,
2457 as<int> (tmpSend.size ()),
2458 procsTo_[p], tag, *comm_);
2460 else if (sendType == Details::DISTRIBUTOR_SSEND) {
2461 ssend<int> (tmpSend,
2462 as<int> (tmpSend.size ()),
2463 procsTo_[p], tag, *comm_);
2466 TEUCHOS_TEST_FOR_EXCEPTION(
2467 true, std::logic_error,
"Tpetra::Distributor::doPosts(3 args): " 2468 "Invalid send type. We should never get here. " 2469 "Please report this bug to the Tpetra developers.");
2474 selfIndex = startsTo_[p];
2480 std::ostringstream os;
2481 os <<
"Proc " << myRank <<
": doPosts(3,slow): Self-send" << endl;
2484 for (
size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
2485 deep_copy_offset(imports, exports, selfReceiveOffset,
2486 indicesTo_[selfIndex]*numPackets, numPackets);
2488 selfReceiveOffset += numPackets;
2492 std::ostringstream os;
2493 os <<
"Proc " << myRank <<
": doPosts(3,slow) done" << endl;
2499 std::ostringstream os;
2500 os <<
"Proc " << myRank <<
": doPosts done" << endl;
2505 template <
class ExpView,
class ImpView>
2506 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2508 doPosts (
const ExpView &exports,
2509 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
2510 const ImpView &imports,
2511 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
2513 using Teuchos::Array;
2515 using Teuchos::ireceive;
2516 using Teuchos::isend;
2517 using Teuchos::readySend;
2518 using Teuchos::send;
2519 using Teuchos::ssend;
2520 using Teuchos::TypeNameTraits;
2521 #ifdef HAVE_TEUCHOS_DEBUG 2522 using Teuchos::OSTab;
2523 #endif // HAVE_TEUCHOS_DEBUG 2525 using Kokkos::Compat::create_const_view;
2526 using Kokkos::Compat::create_view;
2527 using Kokkos::Compat::subview_offset;
2528 using Kokkos::Compat::deep_copy_offset;
2529 typedef Array<size_t>::size_type size_type;
2530 typedef ExpView exports_view_type;
2531 typedef ImpView imports_view_type;
2533 Teuchos::OSTab tab (out_);
2535 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2536 Teuchos::TimeMonitor timeMon (*timer_doPosts4_);
2537 #endif // TPETRA_DISTRIBUTOR_TIMERS 2542 const bool doBarrier = barrierBetween_;
2568 TEUCHOS_TEST_FOR_EXCEPTION(
2569 sendType == Details::DISTRIBUTOR_RSEND && ! doBarrier,
2570 std::logic_error,
"Tpetra::Distributor::doPosts(4 args): Ready-send " 2571 "version requires a barrier between posting receives and posting ready " 2572 "sends. This should have been checked before. " 2573 "Please report this bug to the Tpetra developers.");
2575 const int myProcID = comm_->getRank ();
2576 size_t selfReceiveOffset = 0;
2578 #ifdef HAVE_TEUCHOS_DEBUG 2580 size_t totalNumImportPackets = 0;
2581 for (size_type ii = 0; ii < numImportPacketsPerLID.size (); ++ii) {
2582 totalNumImportPackets += numImportPacketsPerLID[ii];
2584 TEUCHOS_TEST_FOR_EXCEPTION(
2585 imports.dimension_0 () < totalNumImportPackets, std::runtime_error,
2586 "Tpetra::Distributor::doPosts(4 args): The 'imports' array must have " 2587 "enough entries to hold the expected number of import packets. " 2588 "imports.dimension_0() = " << imports.dimension_0 () <<
" < " 2589 "totalNumImportPackets = " << totalNumImportPackets <<
".");
2590 #endif // HAVE_TEUCHOS_DEBUG 2597 const int pathTag = 1;
2598 const int tag = this->getTag (pathTag);
2601 TEUCHOS_TEST_FOR_EXCEPTION(
2602 requests_.size () != 0, std::logic_error,
"Tpetra::Distributor::" 2603 "doPosts(4 args): Process " << myProcID <<
": requests_.size () = " 2604 << requests_.size () <<
" != 0.");
2605 std::ostringstream os;
2606 os << myProcID <<
": doPosts(4," 2607 << (indicesTo_.empty () ?
"fast" :
"slow") <<
")" << endl;
2624 const size_type actualNumReceives = as<size_type> (numReceives_) +
2625 as<size_type> (selfMessage_ ? 1 : 0);
2626 requests_.resize (0);
2634 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2635 Teuchos::TimeMonitor timeMonRecvs (*timer_doPosts4_recvs_);
2636 #endif // TPETRA_DISTRIBUTOR_TIMERS 2638 size_t curBufferOffset = 0;
2639 size_t curLIDoffset = 0;
2640 for (size_type i = 0; i < actualNumReceives; ++i) {
2641 size_t totalPacketsFrom_i = 0;
2642 for (
size_t j = 0; j < lengthsFrom_[i]; ++j) {
2643 totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
2645 curLIDoffset += lengthsFrom_[i];
2646 if (procsFrom_[i] != myProcID && totalPacketsFrom_i) {
2655 imports_view_type recvBuf =
2656 subview_offset (imports, curBufferOffset, totalPacketsFrom_i);
2657 requests_.push_back (ireceive<int> (recvBuf, procsFrom_[i],
2661 selfReceiveOffset = curBufferOffset;
2663 curBufferOffset += totalPacketsFrom_i;
2668 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2669 Teuchos::TimeMonitor timeMonBarrier (*timer_doPosts4_barrier_);
2670 #endif // TPETRA_DISTRIBUTOR_TIMERS 2679 #ifdef TPETRA_DISTRIBUTOR_TIMERS 2680 Teuchos::TimeMonitor timeMonSends (*timer_doPosts4_sends_);
2681 #endif // TPETRA_DISTRIBUTOR_TIMERS 2685 Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
2686 size_t maxNumPackets = 0;
2687 size_t curPKToffset = 0;
2688 for (
size_t pp=0; pp<numSends_; ++pp) {
2689 sendPacketOffsets[pp] = curPKToffset;
2690 size_t numPackets = 0;
2691 for (
size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
2692 numPackets += numExportPacketsPerLID[j];
2694 if (numPackets > maxNumPackets) maxNumPackets = numPackets;
2695 packetsPerSend[pp] = numPackets;
2696 curPKToffset += numPackets;
2701 size_t numBlocks = numSends_+ selfMessage_;
2702 size_t procIndex = 0;
2703 while ((procIndex < numBlocks) && (procsTo_[procIndex] < myProcID)) {
2706 if (procIndex == numBlocks) {
2711 size_t selfIndex = 0;
2713 if (indicesTo_.empty()) {
2715 std::ostringstream os;
2716 os << myProcID <<
": doPosts(4,fast): posting sends" << endl;
2722 for (
size_t i = 0; i < numBlocks; ++i) {
2723 size_t p = i + procIndex;
2724 if (p > (numBlocks - 1)) {
2728 if (procsTo_[p] != myProcID && packetsPerSend[p] > 0) {
2729 exports_view_type tmpSend =
2730 subview_offset(exports, sendPacketOffsets[p], packetsPerSend[p]);
2732 if (sendType == Details::DISTRIBUTOR_SEND) {
2734 as<int> (tmpSend.size ()),
2735 procsTo_[p], tag, *comm_);
2737 else if (sendType == Details::DISTRIBUTOR_RSEND) {
2738 readySend<int> (tmpSend,
2739 as<int> (tmpSend.size ()),
2740 procsTo_[p], tag, *comm_);
2742 else if (sendType == Details::DISTRIBUTOR_ISEND) {
2743 exports_view_type tmpSendBuf =
2744 subview_offset (exports, sendPacketOffsets[p], packetsPerSend[p]);
2745 requests_.push_back (isend<int> (tmpSendBuf, procsTo_[p],
2748 else if (sendType == Details::DISTRIBUTOR_SSEND) {
2749 ssend<int> (tmpSend,
2750 as<int> (tmpSend.size ()),
2751 procsTo_[p], tag, *comm_);
2754 TEUCHOS_TEST_FOR_EXCEPTION(
2755 true, std::logic_error,
"Tpetra::Distributor::doPosts(4 args): " 2756 "Invalid send type. We should never get here. " 2757 "Please report this bug to the Tpetra developers.");
2766 deep_copy_offset(imports, exports, selfReceiveOffset,
2767 sendPacketOffsets[selfNum], packetsPerSend[selfNum]);
2770 std::ostringstream os;
2771 os << myProcID <<
": doPosts(4,fast) done" << endl;
2777 std::ostringstream os;
2778 os << myProcID <<
": doPosts(4,slow): posting sends" << endl;
2783 typedef typename ExpView::non_const_value_type Packet;
2784 typedef typename ExpView::array_layout Layout;
2785 typedef typename ExpView::device_type Device;
2786 typedef typename ExpView::memory_traits Mem;
2787 Kokkos::View<Packet*,Layout,Device,Mem> sendArray (
"sendArray", maxNumPackets);
2789 TEUCHOS_TEST_FOR_EXCEPTION(
2790 sendType == Details::DISTRIBUTOR_ISEND, std::logic_error,
2791 "Tpetra::Distributor::doPosts(3 args): The \"send buffer\" code path " 2792 "may not necessarily work with nonblocking sends.");
2794 Array<size_t> indicesOffsets (numExportPacketsPerLID.size(), 0);
2796 for (
int j=0; j<numExportPacketsPerLID.size(); ++j) {
2797 indicesOffsets[j] = ioffset;
2798 ioffset += numExportPacketsPerLID[j];
2801 for (
size_t i = 0; i < numBlocks; ++i) {
2802 size_t p = i + procIndex;
2803 if (p > (numBlocks - 1)) {
2807 if (procsTo_[p] != myProcID) {
2808 size_t sendArrayOffset = 0;
2809 size_t j = startsTo_[p];
2810 size_t numPacketsTo_p = 0;
2811 for (
size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
2812 deep_copy_offset(sendArray, exports, sendArrayOffset,
2813 indicesOffsets[j], numExportPacketsPerLID[j]);
2814 sendArrayOffset += numExportPacketsPerLID[j];
2816 if (numPacketsTo_p > 0) {
2818 subview_offset(sendArray,
size_t(0), numPacketsTo_p);
2820 if (sendType == Details::DISTRIBUTOR_RSEND) {
2821 readySend<int> (tmpSend,
2822 as<int> (tmpSend.size ()),
2823 procsTo_[p], tag, *comm_);
2825 else if (sendType == Details::DISTRIBUTOR_ISEND) {
2826 exports_view_type tmpSendBuf =
2827 subview_offset (sendArray,
size_t(0), numPacketsTo_p);
2828 requests_.push_back (isend<int> (tmpSendBuf, procsTo_[p],
2831 else if (sendType == Details::DISTRIBUTOR_SSEND) {
2832 ssend<int> (tmpSend,
2833 as<int> (tmpSend.size ()),
2834 procsTo_[p], tag, *comm_);
2838 as<int> (tmpSend.size ()),
2839 procsTo_[p], tag, *comm_);
2845 selfIndex = startsTo_[p];
2850 for (
size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
2851 deep_copy_offset(imports, exports, selfReceiveOffset,
2852 indicesOffsets[selfIndex],
2853 numExportPacketsPerLID[selfIndex]);
2854 selfReceiveOffset += numExportPacketsPerLID[selfIndex];
2859 std::ostringstream os;
2860 os << myProcID <<
": doPosts(4,slow) done" << endl;
2866 template <
class ExpView,
class ImpView>
2867 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2869 doReversePostsAndWaits (
const ExpView& exports,
2871 const ImpView& imports)
2873 doReversePosts (exports, numPackets, imports);
2877 template <
class ExpView,
class ImpView>
2878 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2880 doReversePostsAndWaits (
const ExpView& exports,
2881 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
2882 const ImpView& imports,
2883 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
2885 TEUCHOS_TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
2886 "Tpetra::Distributor::doReversePostsAndWaits(4 args): There are " 2887 << requests_.size() <<
" outstanding nonblocking messages pending. It " 2888 "is incorrect to call this method with posts outstanding.");
2890 doReversePosts (exports, numExportPacketsPerLID, imports,
2891 numImportPacketsPerLID);
2895 template <
class ExpView,
class ImpView>
2896 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2898 doReversePosts (
const ExpView &exports,
2900 const ImpView &imports)
2903 TEUCHOS_TEST_FOR_EXCEPTION(
2904 ! indicesTo_.empty (), std::runtime_error,
2905 "Tpetra::Distributor::doReversePosts(3 args): Can only do " 2906 "reverse communication when original data are blocked by process.");
2907 if (reverseDistributor_.is_null ()) {
2908 createReverseDistributor ();
2910 reverseDistributor_->doPosts (exports, numPackets, imports);
2913 template <
class ExpView,
class ImpView>
2914 typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
2916 doReversePosts (
const ExpView &exports,
2917 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
2918 const ImpView &imports,
2919 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
2922 TEUCHOS_TEST_FOR_EXCEPTION(
2923 ! indicesTo_.empty (), std::runtime_error,
2924 "Tpetra::Distributor::doReversePosts(3 args): Can only do " 2925 "reverse communication when original data are blocked by process.");
2926 if (reverseDistributor_.is_null ()) {
2927 createReverseDistributor ();
2929 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
2930 imports, numImportPacketsPerLID);
2933 template <
class OrdinalType>
2935 computeSends (
const Teuchos::ArrayView<const OrdinalType> & importGIDs,
2936 const Teuchos::ArrayView<const int> & importProcIDs,
2937 Teuchos::Array<OrdinalType> & exportGIDs,
2938 Teuchos::Array<int> & exportProcIDs)
2947 using Teuchos::Array;
2948 using Teuchos::ArrayView;
2950 typedef typename ArrayView<const OrdinalType>::size_type size_type;
2952 Teuchos::OSTab tab (out_);
2953 const int myRank = comm_->getRank ();
2955 std::ostringstream os;
2956 os << myRank <<
": computeSends" << endl;
2960 TEUCHOS_TEST_FOR_EXCEPTION(
2961 importGIDs.size () != importProcIDs.size (), std::invalid_argument,
2962 "Tpetra::Distributor::computeSends: On Process " << myRank <<
": " 2963 "importProcIDs.size() = " << importProcIDs.size ()
2964 <<
" != importGIDs.size() = " << importGIDs.size () <<
".");
2966 const size_type numImports = importProcIDs.size ();
2967 Array<size_t> importObjs (2*numImports);
2969 for (size_type i = 0; i < numImports; ++i) {
2970 importObjs[2*i] =
static_cast<size_t> (importGIDs[i]);
2971 importObjs[2*i+1] =
static_cast<size_t> (myRank);
2979 std::ostringstream os;
2980 os << myRank <<
": computeSends: tempPlan.createFromSends" << endl;
2986 const size_t numExportsAsSizeT = tempPlan.createFromSends (importProcIDs);
2987 const size_type numExports =
static_cast<size_type
> (numExportsAsSizeT);
2988 TEUCHOS_TEST_FOR_EXCEPTION(
2989 numExports < 0, std::logic_error,
"Tpetra::Distributor::computeSends: " 2990 "tempPlan.createFromSends() returned numExports = " << numExportsAsSizeT
2991 <<
" as a size_t, which overflows to " << numExports <<
" when cast to " 2992 << Teuchos::TypeNameTraits<size_type>::name () <<
". " 2993 "Please report this bug to the Tpetra developers.");
2994 TEUCHOS_TEST_FOR_EXCEPTION(
2995 static_cast<size_type> (tempPlan.getTotalReceiveLength ()) != numExports,
2996 std::logic_error,
"Tpetra::Distributor::computeSends: tempPlan.getTotal" 2997 "ReceiveLength() = " << tempPlan.getTotalReceiveLength () <<
" != num" 2998 "Exports = " << numExports <<
". Please report this bug to the " 2999 "Tpetra developers.");
3001 if (numExports > 0) {
3002 exportGIDs.resize (numExports);
3003 exportProcIDs.resize (numExports);
3014 TEUCHOS_TEST_FOR_EXCEPTION(
3015 sizeof (
size_t) <
sizeof (OrdinalType), std::logic_error,
3016 "Tpetra::Distributor::computeSends: sizeof(size_t) = " <<
sizeof(
size_t)
3017 <<
" < sizeof(" << Teuchos::TypeNameTraits<OrdinalType>::name () <<
") = " 3018 <<
sizeof (OrdinalType) <<
". This violates an assumption of the " 3019 "method. It's not hard to work around (just use Array<OrdinalType> as " 3020 "the export buffer, not Array<size_t>), but we haven't done that yet. " 3021 "Please report this bug to the Tpetra developers.");
3023 TEUCHOS_TEST_FOR_EXCEPTION(
3024 tempPlan.getTotalReceiveLength () <
static_cast<size_t> (numExports),
3026 "Tpetra::Distributor::computeSends: tempPlan.getTotalReceiveLength() = " 3027 << tempPlan.getTotalReceiveLength() <<
" < numExports = " << numExports
3028 <<
". Please report this bug to the Tpetra developers.");
3030 Array<size_t> exportObjs (tempPlan.getTotalReceiveLength () * 2);
3032 std::ostringstream os;
3033 os << myRank <<
": computeSends: tempPlan.doPostsAndWaits" << endl;
3036 tempPlan.doPostsAndWaits<
size_t> (importObjs (), 2, exportObjs ());
3039 for (size_type i = 0; i < numExports; ++i) {
3040 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
3041 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
3045 std::ostringstream os;
3046 os << myRank <<
": computeSends done" << endl;
3051 template <
class OrdinalType>
3053 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
3054 const Teuchos::ArrayView<const int> &remoteProcIDs,
3055 Teuchos::Array<OrdinalType> &exportGIDs,
3056 Teuchos::Array<int> &exportProcIDs)
3060 Teuchos::OSTab tab (out_);
3061 const int myRank = comm_->getRank();
3064 *out_ << myRank <<
": createFromRecvs" << endl;
3067 #ifdef HAVE_TPETRA_DEBUG 3068 using Teuchos::outArg;
3069 using Teuchos::reduceAll;
3074 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
3075 int maxErrProc = -1;
3076 reduceAll<int, int> (*comm_, Teuchos::REDUCE_MAX, errProc, outArg (maxErrProc));
3077 TEUCHOS_TEST_FOR_EXCEPTION(maxErrProc != -1, std::runtime_error,
3078 Teuchos::typeName (*
this) <<
"::createFromRecvs(): lists of remote IDs " 3079 "and remote process IDs must have the same size on all participating " 3080 "processes. Maximum process ID with error: " << maxErrProc <<
".");
3081 #else // NOT HAVE_TPETRA_DEBUG 3084 TEUCHOS_TEST_FOR_EXCEPTION(
3085 remoteGIDs.size () != remoteProcIDs.size (), std::invalid_argument,
3086 Teuchos::typeName (*
this) <<
"::createFromRecvs<" <<
3087 Teuchos::TypeNameTraits<OrdinalType>::name () <<
">(): On Process " <<
3088 myRank <<
": remoteGIDs.size() = " << remoteGIDs.size () <<
" != " 3089 "remoteProcIDs.size() = " << remoteProcIDs.size () <<
".");
3090 #endif // HAVE_TPETRA_DEBUG 3092 computeSends (remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
3094 const size_t numProcsSendingToMe =
createFromSends (exportProcIDs ());
3101 std::ostringstream os;
3102 os <<
"Proc " << myRank <<
": {numProcsSendingToMe: " 3103 << numProcsSendingToMe <<
", remoteProcIDs.size(): " 3104 << remoteProcIDs.size () <<
", selfMessage_: " 3105 << (selfMessage_ ?
"true" :
"false") <<
"}" << std::endl;
3106 std::cerr << os.str ();
3110 *out_ << myRank <<
": createFromRecvs done" << endl;
3113 howInitialized_ = Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS;
3119 #endif // TPETRA_DISTRIBUTOR_HPP void doPostsAndWaits(const Teuchos::ArrayView< const Packet > &exports, size_t numPackets, const Teuchos::ArrayView< Packet > &imports)
Execute the (forward) communication plan.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::RCP< Distributor > getReverse() const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
std::string DistributorSendTypeEnumToString(EDistributorSendType sendType)
Convert an EDistributorSendType enum value to a string.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
void doReversePostsAndWaits(const Teuchos::ArrayView< const Packet > &exports, size_t numPackets, const Teuchos::ArrayView< Packet > &imports)
Execute the reverse communication plan.
size_t getNumReceives() const
The number of processes from which we will receive data.
size_t getNumSends() const
The number of processes to which we will send data.
Implementation details of Tpetra.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
void doPosts(const Teuchos::ArrayRCP< const Packet > &exports, size_t numPackets, const Teuchos::ArrayRCP< Packet > &imports)
Post the data for a forward plan, but do not execute the waits yet.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
Sets up and executes a communication plan for a Tpetra DistObject.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
void doReversePosts(const Teuchos::ArrayRCP< const Packet > &exports, size_t numPackets, const Teuchos::ArrayRCP< Packet > &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
virtual ~Distributor()
Destructor (virtual for memory safety).
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.
std::string DistributorHowInitializedEnumToString(EDistributorHowInitialized how)
Convert an EDistributorHowInitialized enum value to a string.
Stand-alone utility functions and macros.
void getLastDoStatistics(size_t &bytes_sent, size_t &bytes_recvd) const
Information on the last call to do/doReverse.
std::string description() const
Return a one-line description of this object.
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
EDistributorSendType
The type of MPI send that Distributor should use.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.