Tpetra parallel linear algebra  Version of the Day
Tpetra_DistObject_decl.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_DISTOBJECT_DECL_HPP
43 #define TPETRA_DISTOBJECT_DECL_HPP
44 
52 
53 #include "Tpetra_Map.hpp"
54 #include "Tpetra_Import.hpp"
55 #include "Tpetra_Export.hpp"
56 #include "Tpetra_SrcDistObject.hpp"
57 #include "Kokkos_NodeAPIConfigDefs.hpp" // enum KokkosClassic::ReadWriteOption
58 #include "Kokkos_ArithTraits.hpp"
59 
60 // #ifndef HAVE_TPETRA_TRANSFER_TIMERS
61 // # define HAVE_TPETRA_TRANSFER_TIMERS 1
62 // #endif // HAVE_TPETRA_TRANSFER_TIMERS
63 
64 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
65 # undef HAVE_TPETRA_TRANSFER_TIMERS
66 #endif // HAVE_TPETRA_TRANSFER_TIMERS
67 
68 
69 namespace Tpetra {
70 
207  template <class Packet,
209  class GlobalOrdinal = ::Tpetra::Details::DefaultTypes::global_ordinal_type,
211  const bool classic = Node::classic>
212  class DistObject :
213  virtual public SrcDistObject,
214  virtual public Teuchos::Describable
215  {
216  public:
218 
219 
224  typedef typename Kokkos::Details::ArithTraits<Packet>::val_type packet_type;
226  typedef LocalOrdinal local_ordinal_type;
228  typedef GlobalOrdinal global_ordinal_type;
230  typedef Node node_type;
231 
233  typedef typename Node::device_type device_type;
235  typedef typename device_type::execution_space execution_space;
236 
237  private:
238  typedef typename Kokkos::View<packet_type*, device_type>::size_type view_size_type;
240 
241  public:
244 
246 
248 
250  explicit DistObject (const Teuchos::RCP<const map_type>& map);
251 
254 
256  virtual ~DistObject ();
257 
259 
261 
280  void
281  doImport (const SrcDistObject& source,
283  CombineMode CM);
284 
303  void
304  doExport (const SrcDistObject& source,
306  CombineMode CM);
307 
327  void
328  doImport (const SrcDistObject& source,
330  CombineMode CM);
331 
351  void
352  doExport (const SrcDistObject& source,
354  CombineMode CM);
355 
357 
359 
365  bool isDistributed () const;
366 
373  virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
374 
376 
378 
383  void print (std::ostream &os) const;
384 
386 
388 
393  virtual std::string description () const;
394 
399  virtual void
400  describe (Teuchos::FancyOStream &out,
401  const Teuchos::EVerbosityLevel verbLevel =
402  Teuchos::Describable::verbLevel_default) const;
403 
405 
407 
454  virtual void
455  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
456 
457  // Forward declaration of nonmember function.
458  template<class PT, class LO, class GO, class NT>
459  friend void
461  const Teuchos::RCP<const Map<LO, GO, NT> >& newMap);
462  // Forward declaration of nonmember function.
463  template<class PT, class LO, class GO, class NT>
464  friend void
467 
468  protected:
478  DoForward, //*!< Perform the transfer in forward mode.
479  DoReverse //*!< Perform the transfer in reverse mode.
480  };
481 
498  virtual size_t constantNumberOfPackets () const;
499 
522  virtual void
523  doTransfer (const SrcDistObject& src,
524  CombineMode CM,
525  size_t numSameIDs,
526  const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
527  const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
528  const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
529  const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
530  Distributor &distor,
531  ReverseOption revOp);
532 
533  virtual void
534  doTransferOld (const SrcDistObject& src,
535  CombineMode CM,
536  size_t numSameIDs,
537  const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
538  const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
539  const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
540  const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
541  Distributor &distor,
542  ReverseOption revOp);
543 
544  virtual void
545  doTransferNew (const SrcDistObject& src,
546  const CombineMode CM,
547  const size_t numSameIDs,
548  const Kokkos::DualView<const local_ordinal_type*,
549  device_type>& permuteToLIDs,
550  const Kokkos::DualView<const local_ordinal_type*,
551  device_type>& permuteFromLIDs,
552  const Kokkos::DualView<const local_ordinal_type*,
553  device_type>& remoteLIDs,
554  const Kokkos::DualView<const local_ordinal_type*,
555  device_type>& exportLIDs,
556  Distributor& distor,
557  const ReverseOption revOp,
558  const bool commOnHost);
559 
570 
571 
575  virtual bool
576  checkSizes (const SrcDistObject& source) = 0;
577 
591  virtual bool useNewInterface () { return false; }
592 
610  virtual void
612  size_t numSameIDs,
613  const Teuchos::ArrayView<const local_ordinal_type>& permuteToLIDs,
614  const Teuchos::ArrayView<const local_ordinal_type>& permuteFromLIDs)
615  {}
616  virtual void
617  copyAndPermuteNew (const SrcDistObject& source,
618  const size_t numSameIDs,
619  const Kokkos::DualView<const local_ordinal_type*, device_type>& permuteToLIDs,
620  const Kokkos::DualView<const local_ordinal_type*, device_type>& permuteFromLIDs)
621  {}
622 
645  virtual void
647  const Teuchos::ArrayView<const local_ordinal_type>& exportLIDs,
648  Teuchos::Array<packet_type>& exports,
649  const Teuchos::ArrayView<size_t>& numPacketsPerLID,
650  size_t& constantNumPackets,
651  Distributor &distor)
652  {}
653 
654  virtual void
655  packAndPrepareNew (const SrcDistObject& source,
656  const Kokkos::DualView<const local_ordinal_type*, device_type>& exportLIDs,
657  Kokkos::DualView<packet_type*, device_type>& exports,
658  const Kokkos::DualView<size_t*, device_type>& numPacketsPerLID,
659  size_t& constantNumPackets,
660  Distributor& distor)
661  {}
662 
686  virtual void
687  unpackAndCombine (const Teuchos::ArrayView<const local_ordinal_type>& importLIDs,
688  const Teuchos::ArrayView<const packet_type>& imports,
689  const Teuchos::ArrayView<size_t>& numPacketsPerLID,
690  size_t constantNumPackets,
691  Distributor &distor,
692  CombineMode CM)
693  {}
694 
720  virtual void
721  unpackAndCombineNew (const Kokkos::DualView<const local_ordinal_type*, device_type>& importLIDs,
722  const Kokkos::DualView<const packet_type*, device_type>& imports,
723  const Kokkos::DualView<const size_t*, device_type>& numPacketsPerLID,
724  const size_t constantNumPackets,
725  Distributor& distor,
726  const CombineMode CM)
727  {}
729 
736  virtual void createViews () const;
737 
753  virtual void createViewsNonConst (KokkosClassic::ReadWriteOption rwo);
754 
769  virtual void releaseViews () const;
770 
772  Teuchos::RCP<const map_type> map_;
773 
774  protected:
781  Kokkos::DualView<packet_type*, execution_space> imports_;
782 
790  void
791  reallocImportsIfNeeded (const size_t newSize, const bool debug = false);
792 
806  Kokkos::DualView<size_t*, execution_space> numImportPacketsPerLID_;
807 
813  Kokkos::DualView<packet_type*, device_type> exports_;
814 
828  Kokkos::DualView<size_t*, execution_space> numExportPacketsPerLID_;
829 
830 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
831  private:
832  Teuchos::RCP<Teuchos::Time> doXferTimer_;
833  Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
834  Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
835  Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
836  Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
837 #endif // HAVE_TPETRA_TRANSFER_TIMERS
838 
839  }; // class DistObject
840 
923  template<class DistObjectType>
924  void
925  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
926  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
927  typename DistObjectType::global_ordinal_type,
928  typename DistObjectType::node_type> >& newMap);
929 
966  template<class DistObjectType>
967  void
968  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
969 
970 } // namespace Tpetra
971 
972 #endif // TPETRA_DISTOBJECT_DECL_HPP
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Namespace Tpetra contains the class and methods constituting the Tpetra library.
virtual void createViews() const
Hook for creating a const view.
device_type::execution_space execution_space
The Kokkos execution space.
Node node_type
The Kokkos Node type.
KokkosClassic::DefaultNode::DefaultNodeType node_type
Default value of Node template parameter.
Kokkos::Details::ArithTraits< Packet >::val_type packet_type
The type of each datum being sent or received in an Import or Export.
Kokkos::DualView< size_t *, execution_space > numExportPacketsPerLID_
Number of packets to send for each send operation.
Abstract base class for sources of an Import or Export.
virtual void doTransfer(const SrcDistObject &src, CombineMode CM, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs, const Teuchos::ArrayView< const local_ordinal_type > &remoteLIDs, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Distributor &distor, ReverseOption revOp)
Redistribute data across memory images.
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no entries in this object&#39;s Map.
void removeEmptyProcessesInPlace(Teuchos::RCP< DistObjectType > &input, const Teuchos::RCP< const Map< typename DistObjectType::local_ordinal_type, typename DistObjectType::global_ordinal_type, typename DistObjectType::node_type > > &newMap)
Remove processes which contain no elements in this object&#39;s Map.
virtual std::string description() const
One-line descriptiion of this object.
int local_ordinal_type
Default value of LocalOrdinal template parameter.
Kokkos::DualView< packet_type *, execution_space > imports_
Buffer into which packed data are imported (received from other processes).
Node::device_type device_type
The Kokkos Device type.
void print(std::ostream &os) const
Print this object to the given output stream.
virtual void packAndPrepare(const SrcDistObject &source, const Teuchos::ArrayView< const local_ordinal_type > &exportLIDs, Teuchos::Array< packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t &constantNumPackets, Distributor &distor)
Perform any packing or preparation required for communication.
GlobalOrdinal global_ordinal_type
The type of global indices.
LocalOrdinal local_ordinal_type
The type of local indices.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.
virtual void releaseViews() const
Hook for releasing views.
virtual void unpackAndCombine(const Teuchos::ArrayView< const local_ordinal_type > &importLIDs, const Teuchos::ArrayView< const packet_type > &imports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, size_t constantNumPackets, Distributor &distor, CombineMode CM)
Perform any unpacking and combining after communication (old version that uses Teuchos memory managem...
virtual void unpackAndCombineNew(const Kokkos::DualView< const local_ordinal_type *, device_type > &importLIDs, const Kokkos::DualView< const packet_type *, device_type > &imports, const Kokkos::DualView< const size_t *, device_type > &numPacketsPerLID, const size_t constantNumPackets, Distributor &distor, const CombineMode CM)
Perform any unpacking and combining after communication (new version that uses Kokkos data structures...
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Map< local_ordinal_type, global_ordinal_type, node_type > map_type
The type of the Map specialization to use with this class.
Sets up and executes a communication plan for a Tpetra DistObject.
CombineMode
Rule for combining data in an Import or Export.
virtual ~DistObject()
Destructor (virtual for memory safety of derived classes).
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
virtual void copyAndPermute(const SrcDistObject &source, size_t numSameIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteToLIDs, const Teuchos::ArrayView< const local_ordinal_type > &permuteFromLIDs)
Perform copies and permutations that are local to this process.
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
Abstract base class for objects that can be the source of an Import or Export operation.
Kokkos::DualView< packet_type *, device_type > exports_
Buffer from which packed data are exported (sent to other processes).
ReverseOption
Whether the data transfer should be performed in forward or reverse mode.
DistObject(const Teuchos::RCP< const map_type > &map)
Constructor.
Kokkos::DualView< size_t *, execution_space > numImportPacketsPerLID_
Number of packets to receive for each receive operation.
virtual void createViewsNonConst(KokkosClassic::ReadWriteOption rwo)
Hook for creating a nonconst view.
Describes a parallel distribution of objects over processes.
virtual bool useNewInterface()
Whether the subclass implements the "old" or "new" (Kokkos-friendly) interface.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, CombineMode CM)
Export data into this object using an Export object ("forward mode").
bool isDistributed() const
Whether this is a globally distributed object.
Base class for distributed Tpetra objects that support data redistribution.
void reallocImportsIfNeeded(const size_t newSize, const bool debug=false)
Reallocate imports_ if needed.
virtual size_t constantNumberOfPackets() const
Whether the implementation&#39;s instance promises always to have a constant number of packets per LID (l...
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, CombineMode CM)
Import data into this object using an Import object ("forward mode").