|
virtual int | getTag () const =0 |
| The current tag. More...
|
|
|
virtual | ~Comm () |
| Destructor, declared virtual for safety of derived classes. More...
|
|
|
virtual int | getRank () const =0 |
| Returns the rank of this process. More...
|
|
virtual int | getSize () const =0 |
| Returns the number of processes that make up this communicator. More...
|
|
|
virtual void | barrier () const =0 |
| Pause every process in *this communicator until all the processes reach this point. More...
|
|
virtual void | broadcast (const int rootRank, const Ordinal bytes, char buffer[]) const =0 |
| Broadcast values from the root process to the slave processes. More...
|
|
virtual void | gather (const Ordinal sendBytes, const char sendBuffer[], const Ordinal recvBytes, char recvBuffer[], const int root) const =0 |
| Gather values from all processes to the root process. More...
|
|
virtual void | gatherAll (const Ordinal sendBytes, const char sendBuffer[], const Ordinal recvBytes, char recvBuffer[]) const =0 |
| Gather values from each process to collect on all processes. More...
|
|
virtual void | reduceAll (const ValueTypeReductionOp< Ordinal, char > &reductOp, const Ordinal bytes, const char sendBuffer[], char globalReducts[]) const =0 |
| Global reduction. More...
|
|
virtual void | scan (const ValueTypeReductionOp< Ordinal, char > &reductOp, const Ordinal bytes, const char sendBuffer[], char scanReducts[]) const =0 |
| Scan reduction. More...
|
|
|
virtual void | send (const Ordinal bytes, const char sendBuffer[], const int destRank) const =0 |
| Possibly blocking send of data from this process to another process. More...
|
|
virtual void | send (const Ordinal bytes, const char sendBuffer[], const int destRank, const int tag) const =0 |
| Variant of send() that takes a tag. More...
|
|
virtual void | ssend (const Ordinal bytes, const char sendBuffer[], const int destRank) const =0 |
| Always blocking send of data from this process to another process. More...
|
|
virtual void | ssend (const Ordinal bytes, const char sendBuffer[], const int destRank, const int tag) const =0 |
| Variant of ssend() that takes a message tag. More...
|
|
virtual int | receive (const int sourceRank, const Ordinal bytes, char recvBuffer[]) const =0 |
| Blocking receive of data from this process to another process. More...
|
|
virtual void | readySend (const ArrayView< const char > &sendBuffer, const int destRank) const =0 |
| Ready send of data from this process to another process. More...
|
|
virtual void | readySend (const Ordinal bytes, const char sendBuffer[], const int destRank, const int tag) const =0 |
| Variant of readySend() that accepts a message tag. More...
|
|
|
virtual RCP< CommRequest< Ordinal > > | isend (const ArrayView< const char > &sendBuffer, const int destRank) const =0 |
| Non-blocking send. More...
|
|
virtual RCP< CommRequest< Ordinal > > | isend (const ArrayView< const char > &sendBuffer, const int destRank, const int tag) const =0 |
| Variant of isend() that takes a tag. More...
|
|
virtual RCP< CommRequest< Ordinal > > | ireceive (const ArrayView< char > &recvBuffer, const int sourceRank) const =0 |
| Non-blocking receive. More...
|
|
virtual RCP< CommRequest< Ordinal > > | ireceive (const ArrayView< char > &recvBuffer, const int sourceRank, const int tag) const =0 |
| Variant of ireceive that takes a tag. More...
|
|
virtual void | waitAll (const ArrayView< RCP< CommRequest< Ordinal > > > &requests) const =0 |
| Wait on a set of communication requests. More...
|
|
virtual void | waitAll (const ArrayView< RCP< CommRequest< Ordinal > > > &requests, const ArrayView< RCP< CommStatus< Ordinal > > > &statuses) const =0 |
| Wait on communication requests, and return their statuses. More...
|
|
virtual RCP< CommStatus< Ordinal > > | wait (const Ptr< RCP< CommRequest< Ordinal > > > &request) const =0 |
| Wait on a single communication request, and return its status. More...
|
|
|
virtual RCP< Comm > | duplicate () const =0 |
| Duplicate this communicator. More...
|
|
virtual RCP< Comm > | split (const int color, const int key) const =0 |
| Split a communicator into subcommunicators based on color and key. More...
|
|
virtual RCP< Comm > | createSubcommunicator (const ArrayView< const int > &ranks) const =0 |
| Create a subcommunicator containing the specified processes. More...
|
|
virtual std::string | description () const |
| Return a simple one-line description of this object. More...
|
|
virtual void | describe (FancyOStream &out, const EVerbosityLevel verbLevel=verbLevel_default) const |
| Print the object with some verbosity level to a FancyOStream. More...
|
|
void | describe (std::ostream &out, const EVerbosityLevel verbLevel=verbLevel_default) const |
| Version of describe() that takes an std::ostream instead of a FancyOStream. More...
|
|
virtual | ~Describable () |
| Destructor (marked virtual for memory safety of derived classes). More...
|
|
| LabeledObject () |
| Construct with an empty label. More...
|
|
virtual | ~LabeledObject () |
|
virtual void | setObjectLabel (const std::string &objectLabel) |
| Set the object label (see LabeledObject). More...
|
|
virtual std::string | getObjectLabel () const |
| Get the object label (see LabeledObject). More...
|
|
|
(Note that these are not member functions.)
|
enum | EReductionType |
| Enumeration for selecting from a set of pre-defined reduction operations. More...
|
|
template<typename Ordinal > |
int | rank (const Comm< Ordinal > &comm) |
| Get the process rank. More...
|
|
template<typename Ordinal > |
int | size (const Comm< Ordinal > &comm) |
| Get the number of processes in the communicator. More...
|
|
template<typename Ordinal > |
void | barrier (const Comm< Ordinal > &comm) |
| Barrier. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const int rootRank, const Ordinal count, Packet buffer[]) |
| Broadcast array of objects that use value semantics. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const int rootRank, const ArrayView< Packet > &buffer) |
| Broadcast array of objects that use value semantics. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const int rootRank, Packet *object) |
| Broadcast single object that use value semantics. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const int rootRank, const Ptr< Packet > &object) |
| Broadcast single object that use value semantics. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const int rootRank, const Ordinal count, Packet *const buffer[]) |
| Broadcast array of objects that use reference semantics. More...
|
|
template<typename Ordinal , typename Packet > |
void | broadcast (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const int rootRank, const ArrayView< const Ptr< Packet > > &buffer) |
| Broadcast array of objects that use reference semantics. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | broadcast (const Comm< Ordinal > &comm, const Serializer &serializer, const int rootRank, const Ordinal count, Packet buffer[]) |
| Broadcast array of objects that use value semantics using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
void | gather (const Packet sendBuf[], const Ordinal sendCount, Packet recvBuf[], const Ordinal recvCount, const int root, const Comm< Ordinal > &comm) |
| Gather values from each process to the root process. More...
|
|
template<typename Ordinal , typename Packet > |
void | gatherv (const Packet sendBuf[], const Ordinal sendCount, Packet recvBuf[], const Ordinal recvCounts[], const Ordinal displs[], const int root, const Comm< Ordinal > &comm) |
| Gather arrays of possibly different lengths from each process to the root process. More...
|
|
template<typename Ordinal , typename Packet > |
void | gatherAll (const Comm< Ordinal > &comm, const Ordinal sendCount, const Packet sendBuffer[], const Ordinal recvCount, Packet recvBuffer[]) |
| Gather array of objects that use value semantics from every process to every process. More...
|
|
template<typename Ordinal , typename Packet > |
void | gatherAll (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const Ordinal sendCount, const Packet *const sendBuffer[], const Ordinal recvCount, Packet *const recvBuffer[]) |
| Gather array of objects that use reference semantics from every process to every process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | gatherAll (const Comm< Ordinal > &comm, const Serializer &serializer, const Ordinal sendCount, const Packet sendBuffer[], const Ordinal recvCount, Packet recvBuffer[]) |
| Gather array of objects that use value semantics from every process to every process using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
void | scatter (const Packet sendBuf[], const Ordinal sendCount, Packet recvBuf[], const Ordinal recvCount, const Ordinal root, const Comm< Ordinal > &comm) |
| Wrapper for MPI_Scatter; scatter collective. More...
|
|
template<typename Ordinal , typename Packet > |
void | reduce (const Packet sendBuf[], Packet recvBuf[], const Ordinal count, const EReductionType reductType, const Ordinal root, const Comm< Ordinal > &comm) |
| Wrapper for MPI_Reduce; reduction to one process, using a built-in reduction operator selected by enum. More...
|
|
template<typename Ordinal , typename Packet > |
void | reduceAll (const Comm< Ordinal > &comm, const ValueTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet sendBuffer[], Packet globalReducts[]) |
| Wrapper for MPI_Allreduce that takes a custom reduction operator. More...
|
|
template<typename Ordinal , typename Packet > |
void | reduceAll (const Comm< Ordinal > &comm, const EReductionType reductType, const Ordinal count, const Packet sendBuffer[], Packet globalReducts[]) |
| Collective reduce all of array of objects using value semantics using a pre-defined reduction type. More...
|
|
template<typename Ordinal , typename Packet > |
void | reduceAll (const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, const Ptr< Packet > &globalReduct) |
| Collective reduce all for single object using value semantics using a pre-defined reduction type. More...
|
|
template<typename Ordinal , typename Packet > |
void | reduceAll (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const ReferenceTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet *const sendBuffer[], Packet *const globalReducts[]) |
| Collective reduce all for array of objects using reference semantics. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | reduceAll (const Comm< Ordinal > &comm, const Serializer &serializer, const ValueTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet sendBuffer[], Packet globalReducts[]) |
| Collective reduce all of array of objects using value semantics using a user-defined reduction operator and customized serializer. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | reduceAll (const Comm< Ordinal > &comm, const Serializer &serializer, const EReductionType reductType, const Ordinal count, const Packet sendBuffer[], Packet globalReducts[]) |
| Collective reduce all of array of objects using value semantics using a pre-defined reduction type and customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
void | scan (const Comm< Ordinal > &comm, const ValueTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet sendBuffer[], Packet scanReducts[]) |
| Scan/Reduce array of objects that use value semantics using a user-defined reduction operator. More...
|
|
template<typename Ordinal , typename Packet > |
void | scan (const Comm< Ordinal > &comm, const EReductionType reductType, const Ordinal count, const Packet sendBuffer[], Packet scanReducts[]) |
| Scan/Reduce array of objects using value semantics using a predefined reduction type. More...
|
|
template<typename Ordinal , typename Packet > |
void | scan (const Comm< Ordinal > &comm, const EReductionType reductType, const Packet &send, const Ptr< Packet > &scanReduct) |
| Scan/Reduce single object using value semantics using a predefined reduction type. More...
|
|
template<typename Ordinal , typename Packet > |
void | scan (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const ReferenceTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet *const sendBuffer[], Packet *const scanReducts[]) |
| Scan/Reduce array of objects that use reference semantics using a user-defined reduction operator. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | scan (const Comm< Ordinal > &comm, const Serializer &serializer, const ValueTypeReductionOp< Ordinal, Packet > &reductOp, const Ordinal count, const Packet sendBuffer[], Packet scanReducts[]) |
| Scan/Reduce array of objects that use value semantics using a user-defined reduction operator and customized serializer. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | scan (const Comm< Ordinal > &comm, const Serializer &serializer, const EReductionType reductType, const Ordinal count, const Packet sendBuffer[], Packet scanReducts[]) |
| Scan/Reduce array of objects using value semantics using a predefined reduction type and customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
void | send (const Comm< Ordinal > &comm, const Ordinal count, const Packet sendBuffer[], const int destRank) |
| Send objects that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
void | ssend (const Comm< Ordinal > &comm, const Ordinal count, const Packet sendBuffer[], const int destRank) |
| Synchronously send objects that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
void | send (const Comm< Ordinal > &comm, const Packet &send, const int destRank) |
| Send a single object that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
void | ssend (const Comm< Ordinal > &comm, const Packet &send, const int destRank) |
| Synchronously send a single object that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
void | send (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const Ordinal count, const Packet *const sendBuffer[], const int destRank) |
| Send objects that use reference semantics to another process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | send (const Comm< Ordinal > &comm, const Serializer &serializer, const Ordinal count, const Packet sendBuffer[], const int destRank) |
| Send objects that use values semantics to another process using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
int | receive (const Comm< Ordinal > &comm, const int sourceRank, const Ordinal count, Packet recvBuffer[]) |
| Receive objects that use values semantics from another process. More...
|
|
template<typename Ordinal , typename Packet > |
int | receive (const Comm< Ordinal > &comm, const int sourceRank, Packet *recv) |
| Receive a single object that use values semantics from another process. More...
|
|
template<typename Ordinal , typename Packet > |
int | receive (const Comm< Ordinal > &comm, const Serializer< Ordinal, Packet > &serializer, const int sourceRank, const Ordinal count, Packet *const recvBuffer[]) |
| Receive objects that use reference semantics from another process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
int | receive (const Comm< Ordinal > &comm, const Serializer &serializer, const int sourceRank, const Ordinal count, Packet recvBuffer[]) |
| Receive objects that use values semantics from another process using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
void | readySend (const Comm< Ordinal > &comm, const ArrayView< const Packet > &sendBuffer, const int destRank) |
| Ready-Send an array of objects that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
void | readySend (const Comm< Ordinal > &comm, const Packet &send, const int destRank) |
| Ready-Send a single object that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
void | readySend (const Comm< Ordinal > &comm, const Serializer &serializer, const ArrayView< const Packet > &sendBuffer, const int destRank) |
| Ready-Send an array of objects that use values semantics to another process using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
RCP< CommRequest< Ordinal > > | isend (const Comm< Ordinal > &comm, const ArrayRCP< const Packet > &sendBuffer, const int destRank) |
| Send objects that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet > |
RCP< CommRequest< Ordinal > > | isend (const Comm< Ordinal > &comm, const RCP< const Packet > &send, const int destRank) |
| Send a single object that use values semantics to another process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
RCP< CommRequest< Ordinal > > | isend (const Comm< Ordinal > &comm, const Serializer &serializer, const ArrayRCP< const Packet > &sendBuffer, const int destRank) |
| Send objects that use values semantics to another process using customized serializer. More...
|
|
template<typename Ordinal , typename Packet > |
RCP< CommRequest< Ordinal > > | ireceive (const Comm< Ordinal > &comm, const ArrayRCP< Packet > &recvBuffer, const int sourceRank) |
| Receive one or more objects (that use values semantics) from another process. More...
|
|
template<typename Ordinal , typename Packet > |
RCP< CommRequest< Ordinal > > | ireceive (const Comm< Ordinal > &comm, const RCP< Packet > &recv, const int sourceRank) |
| Receive one object (that uses values semantics) from another process. More...
|
|
template<typename Ordinal , typename Packet , typename Serializer > |
RCP< CommRequest< Ordinal > > | ireceive (const Comm< Ordinal > &comm, const Serializer &serializer, const ArrayRCP< Packet > &recvBuffer, const int sourceRank) |
| Send objects that use values semantics to another process using customized serializer. More...
|
|
template<typename Ordinal > |
void | waitAll (const Comm< Ordinal > &comm, const ArrayView< RCP< CommRequest< Ordinal > > > &requests) |
| Wait for an array of Teuchos::CommRequest objects. More...
|
|
template<typename Ordinal > |
void | waitAll (const Comm< Ordinal > &comm, const ArrayView< RCP< CommRequest< Ordinal > > > &requests, const ArrayView< RCP< CommStatus< Ordinal > > > &statuses) |
| Wait on one or more communication requests, and return their statuses. More...
|
|
template<typename Ordinal > |
RCP< CommStatus< Ordinal > > | wait (const Comm< Ordinal > &comm, const Ptr< RCP< CommRequest< Ordinal > > > &request) |
| Wait on a single communication request, and return its status. More...
|
|
DescribableStreamManipulatorState | describe (const Describable &describable, const EVerbosityLevel verbLevel=Describable::verbLevel_default) |
| Describable output stream manipulator. More...
|
|
std::ostream & | operator<< (std::ostream &os, const DescribableStreamManipulatorState &d) |
| Output stream operator for Describable manipulator. More...
|
|
template<typename Ordinal>
class Teuchos::Comm< Ordinal >
Abstract interface for distributed-memory communication.
- Template Parameters
-
Ordinal | Type of indices used for communication. |
What is Comm?
This class is Teuchos' interface to distributed-memory communication between one or more parallel processes. It presents an interface very much like that of MPI (the Message Passing Interface). Teuchos provides two implementations of Comm:
- An MPI (Message Passing Interface) implementation, MpiComm
- A "serial" implementation, SerialComm, that only has one process
Comm is an abstract interface. You cannot create a Comm directly. You have to create one of the subclasses. The normal way to handle a Comm is to pass it around using RCP (a reference-counted "smart" pointer). For example:
RCP<const Comm<int> > comm =
rcp (
new MpiComm (MPI_COMM_WORLD));
const int myRank = comm->getRank ();
const int numProcs = comm->getSize ();
comm->barrier ();
Comm's communication methods that actually send or receive data accept that data as an array of char
. You should never call these methods directly. Instead, you should use the nonmember "helper" functions in Teuchos_CommHelpers.hpp. These methods are templated on the Packet
type, that is, the type of data you want to send or receive. See the example below.
Treat <tt>RCP<const Comm<int> ></tt> like an opaque handle
You should consider an RCP<const Comm<int> >
as equivalent to the MPI_Comm opaque handle, except that the RCP also does reference counting to ensure memory safety when using the same communicator in different parts of the code. That is, copying the RCP does not create a new communicator; the following two codes do about the same thing, except with a different syntax (and reference counting in the second code).
Raw MPI_Comm handles:
MPI_Comm comm = ...;
MPI_Comm sameComm = comm;
Reference-counted pointers to Comm:
RCP<const Comm<int> > comm = ...;
RCP<const Comm<int> > sameComm = comm;
If you want to make a "new communicator" rather than just "copy
the handle," you should call the duplicate() method. This has the same behavior as MPI_Comm_dup (which see).
The "reference counting" feature means that the subclass of Comm will take care of freeing the underlying MPI_Comm (and any other data structures it may use) by calling MPI_Comm_free if necessary, once the reference count of the RCP goes to zero.
- Warning
- Do not pass around subclasses of Comm by value! Comm or its subclasses by themselves do not have handle semantics. Their copy constructors likely do not behave as you would expect if the classes had handle semantics.
How do I make a Comm?
Comm works whether or not you have build Trilinos with MPI support. If you want to make a "default" Comm that is the equivalent of MPI_COMM_WORLD, but you don't know if your Trilinos with MPI enabled, you may use GlobalMPISession to call MPI_Init if necessary, and DefaultComm to "get a default communicator." For example: int main (int argc, char* argv[]) { using Teuchos::Comm; using Teuchos::DefaultComm; using Teuchos::RCP;
// This replaces the call to MPI_Init. If you didn't // build with MPI, this doesn't call MPI functions. Teuchos::GlobalMPISesssion session (&argc, &argv, NULL); // comm is the equivalent of MPI_COMM_WORLD. RCP<const Comm<int> > comm = DefaultComm<int>::getComm ();
// ... use comm in your code as you would use MPI_COMM_WORLD ...
// We don't need to call MPI_Finalize, since the // destructor of GlobalMPISession does that for us. return EXIT_SUCCESS; } This code works whether or not you built Trilinos with MPI support. It is not necessary to use GlobalMPISession, but it's useful so you don't have to remember to call MPI_Finalize. If you don't want to use GlobalMPISession, you can still call DefaultComm<int>::getComm()
, though you must have called MPI_Init first if you build Trilinos with MPI support. Furthermore, if you know MPI is present, you don't need to use DefaultComm. You may simply pass MPI_COMM_WORLD directly to MpiComm, like this:
RCP<const Comm<int> > comm =
rcp (
new MpiComm (MPI_COMM_WORLD));
You may also pass an arbitrary MPI_Comm directly into MpiComm's constructor, though you are responsible for freeing it after use (via MPI_Comm_free) if necessary. You may automate the freeing of your MPI_Comm by using OpaqueWrapper (which see).
How do I use Comm?
As we mentioned above, for communication of data with Comm, you you should use the nonmember "helper" functions in Teuchos_CommHelpers.hpp. These methods are templated on the Packet
type, that is, the type of data you want to send or receive. For example, suppose you have two processes (with ranks 0 and 1, respectively), and you want to send an array of 10 double
from Process 0 to Process 1. Both processes have defined RCP<const Comm<int> > comm
as above. Here is the code on Process 0:
const int count = 10;
double values[10] = ...;
const int destinationRank = 1;
Teuchos::send<int, double> (*comm, 10, values, destinationRank);
Here is the code on Process 1:
const int count = 10;
double values[10];
const int sourceRank = 0;
Teuchos::receive<int, double> (*comm, sourceRank, 10, values);
Please refer to the documentation in Teuchos_CommHelpers.hpp for more details.
Former documentation
This interface is templated on the ordinal type but only deals with buffers of untyped data represented as arrays char
type. All reduction operations that are initiated by the concreate communicator object are performed by user-defined ReductOpBase
objects. It is the responsibility of the ReductOpBase
object to know what the currect data type is, to perform casts or serializations/unserializations to and from char[]
buffers, and to know how to reduce the objects correctly. It is strictly up to the client to correctly convert data types to char[]
arrays but there is a great deal of helper code to make this easy and safe.
Definition at line 85 of file Teuchos_Comm.hpp.