Teuchos - Trilinos Tools Package  Version of the Day
Teuchos_DefaultComm.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
42 #ifndef TEUCHOS_DEFAULT_COMM_HPP
43 #define TEUCHOS_DEFAULT_COMM_HPP
44 
45 #include "Teuchos_RCP.hpp"
46 #include "Teuchos_DefaultSerialComm.hpp"
47 #ifdef HAVE_MPI
49 #endif
50 
51 namespace Teuchos {
52 
53 #ifdef HAVE_MPI
54 namespace Details {
55 
56 template<class OrdinalType>
57 int
58 mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
59 
60 template<class OrdinalType>
61 int
62 mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
63 
64 } // namespace Details
65 #endif // HAVE_MPI
66 
93 template<typename OrdinalType>
94 class DefaultComm {
95 public:
109 
118 
119 private:
125  static const Comm<OrdinalType>* comm_;
126 
128  static const Comm<OrdinalType>* defaultSerialComm_;
129 
130 #ifdef HAVE_MPI
131  template<class OT>
133  friend int
134  Details::mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
135 
137  template<class OT>
138  friend int
139  Details::mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
140 #endif // HAVE_MPI
141 
143  static void freeDefaultComm () {
144  if (comm_ != NULL) {
145  delete comm_;
146  comm_ = NULL;
147  }
148  }
149 
151  static void freeDefaultSerialComm () {
152  if (defaultSerialComm_ != NULL) {
153  delete defaultSerialComm_;
154  defaultSerialComm_ = NULL;
155  }
156  }
157 };
158 
159 #ifdef HAVE_MPI
160 namespace Details {
161 
162 template<class OrdinalType>
163 int
164 mpiFreeDefaultComm (MPI_Comm, int, void*, void*)
165 {
166  try {
168  } catch (...) {
169  // Destructors must not throw exceptions, so we must accept the
170  // possible memory leak and move on.
171  std::cerr << "Teuchos::DefaultComm: Failed to free default Comm! We can't "
172  "throw an exception here because this is a singleton destructor that "
173  "should only be called at MPI_Finalize or (if not building with MPI) at "
174  "exit from main()." << std::endl;
175  // FIXME (mfh 16 Nov 2014) There might be some way to create a
176  // custom return code with MPI error reporting. For now, we just
177  // pick some error code not equal to MPI_SUCCESS. It could
178  // perhaps overlap with some existing error code.
179  return (MPI_SUCCESS == 0) ? -1 : 0;
180  }
181  return MPI_SUCCESS;
182 }
183 
184 template<class OrdinalType>
185 int
186 mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*)
187 {
188  try {
190  } catch (...) {
191  // Destructors must not throw exceptions, so we must accept the
192  // possible memory leak and move on.
193  std::cerr << "Teuchos::DefaultComm: Failed to free default serial Comm! "
194  "We can't throw an exception here because this is a singleton destructor "
195  "that should only be called at MPI_Finalize or (if not building with MPI)"
196  " at exit from main()." << std::endl;
197  // FIXME (mfh 16 Nov 2014) There might be some way to create a
198  // custom return code with MPI error reporting. For now, we just
199  // pick some error code not equal to MPI_SUCCESS. It could
200  // perhaps overlap with some existing error code.
201  return (MPI_SUCCESS == 0) ? -1 : 0;
202  }
203  return MPI_SUCCESS;
204 }
205 
206 } // namespace Details
207 #endif // HAVE_MPI
208 
209 
210 template<typename OrdinalType>
213 {
214  if (comm_ == NULL) {
215 #ifdef HAVE_MPI
216  comm_ = new MpiComm<OrdinalType> (MPI_COMM_WORLD);
217 
218  // We want comm_ to be deallocated when MPI_Finalize is called.
219  // The standard idiom for this (look in the MPI standard) is to
220  // register an attribute ((key,value) pair) with MPI_COMM_SELF,
221  // with a custom "destructor" to be called at MPI_Finalize.
222 
223  // 'key' is an output argument of MPI_Comm_create_keyval.
224  int key = MPI_KEYVAL_INVALID;
225  int err =
226  MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
227  Details::mpiFreeDefaultComm<OrdinalType>,
228  &key,
229  NULL); // no extra state
230  if (err != MPI_SUCCESS) {
231  if (comm_ != NULL) { // clean up if MPI call fails
232  delete comm_;
233  comm_ = NULL;
234  }
235  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
236  "Teuchos::DefaultComm::getComm: MPI_Comm_create_keyval failed!");
237  }
238  int val = key; // doesn't matter
239 
240  // Attach the attribute to MPI_COMM_SELF.
241  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
242  if (err != MPI_SUCCESS) {
243  // MPI (versions up to and including 3.0) doesn't promise
244  // correct behavior after any function returns something other
245  // than MPI_SUCCESS. Thus, it's not required to try to free the
246  // new key via MPI_Comm_free_keyval. Furthermore, if something
247  // went wrong with MPI_Comm_set_attr, it's likely that the
248  // attribute mechanism is broken. Thus, it would be unwise to
249  // call MPI_Comm_free_keyval. However, we can still clean up
250  // other data.
251  if (comm_ != NULL) { // clean up if MPI call fails
252  delete comm_;
253  comm_ = NULL;
254  }
255  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
256  "Teuchos::DefaultComm::getComm: MPI_Comm_set_attr failed!");
257  }
258 
259  // It looks weird to "free" the key right away. However, this
260  // does not actually cause the "destructor" to be called. It only
261  // gets called at MPI_FINALIZE. See MPI 3.0 standard, Section
262  // 6.7.2, MPI_COMM_FREE_KEYVAL:
263  //
264  // "Note that it is not erroneous to free an attribute key that is
265  // in use, because the actual free does not transpire until after
266  // all references (in other communicators on the process) to the
267  // key have been freed. These references need to be explicitly
268  // freed by the program, either via calls to MPI_COMM_DELETE_ATTR
269  // that free one attribute instance, or by calls to MPI_COMM_FREE
270  // that free all attribute instances associated with the freed
271  // communicator."
272  //
273  // We rely here on the latter mechanism. MPI_FINALIZE calls
274  // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
275  // explicitly.
276  //
277  // It's not clear what to do if the MPI_* calls above succeeded,
278  // but this call fails (i.e., returns != MPI_SUCCESS). We could
279  // throw; this would make sense to do, because MPI (versions up to
280  // and including 3.0) doesn't promise correct behavior after any
281  // MPI function returns something other than MPI_SUCCESS. We
282  // could also be optimistic and just ignore the return value,
283  // hoping that if the above calls succeeded, then the communicator
284  // will get freed at MPI_FINALIZE, even though the unfreed key may
285  // leak memory (see Bug 6338). I've chosen the latter.
286  (void) MPI_Comm_free_keyval (&key);
287 
288 #else // NOT HAVE_MPI
289  comm_ = new SerialComm<OrdinalType> ();
290  // We want comm_ to be deallocated when main exits, so register
291  // its deallocation function as an atexit handler.
292  //
293  // The POSIX standard allows atexit to fail, in particular if it
294  // lacks space for registering more functions. "[T]he application
295  // should call sysconf() to obtain the value of {ATEXIT_MAX}, the
296  // [maximum] number of functions that can be registered. There is
297  // no way for an application to tell how many functions have
298  // already been registered with atexit()."
299  //
300  // We don't do this here. Instead, we just check atexit's return
301  // code. If it fails, we throw.
302  int err = atexit (freeDefaultComm);
303  if (err != 0) {
304  if (comm_ != NULL) { // clean up if atexit fails
305  delete comm_;
306  comm_ = NULL;
307  }
308  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
309  "Teuchos::DefaultComm::getComm: atexit failed!");
310  }
311 #endif // HAVE_MPI
312  }
313 
315  (comm_ == NULL, std::logic_error, "Teuchos::DefaultComm::getComm: "
316  "comm_ == NULL before return. This should never happen. "
317  "Please report this bug to the Teuchos developers.");
318 
319  // Return a nonowning RCP, because we need to ensure that
320  // destruction happens at MPI_Finalize (or at exit of main(), if not
321  // building with MPI).
322  return rcp (comm_, false);
323 }
324 
325 template<typename OrdinalType>
329 {
330  if (! comm.is_null ()) {
331  return comm;
332  } else {
333  if (defaultSerialComm_ == NULL) {
334 #ifdef HAVE_MPI
335  //defaultSerialComm_ = new MpiComm<OrdinalType> (MPI_COMM_SELF);
336  defaultSerialComm_ = new SerialComm<OrdinalType> ();
337 
338  // Register an MPI_Finalize hook to free defaultSerialComm_.
339  // (See getComm implementation above in this file for details.)
340 
341  int key = MPI_KEYVAL_INVALID;
342  int err =
343  MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
344  Details::mpiFreeDefaultSerialComm<OrdinalType>,
345  &key,
346  NULL); // no extra state
347  if (err != MPI_SUCCESS) {
348  if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
349  delete defaultSerialComm_;
350  defaultSerialComm_ = NULL;
351  }
353  true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
354  ": MPI_Comm_create_keyval failed!");
355  }
356  int val = key; // doesn't matter
357 
358  // Attach the attribute to MPI_COMM_SELF.
359  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
360  if (err != MPI_SUCCESS) {
361  // See comments in getComm implementation above to see why we
362  // don't call MPI_Comm_free_keyval here.
363  if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
364  delete defaultSerialComm_;
365  defaultSerialComm_ = NULL;
366  }
368  true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
369  ": MPI_Comm_set_attr failed!");
370  }
371 
372  // See comments in getComm implementation above to see why we
373  // _do_ call MPI_Comm_free_keyval here, and why we don't check
374  // the return code.
375  (void) MPI_Comm_free_keyval (&key);
376 
377 #else // NOT HAVE_MPI
378  defaultSerialComm_ = new SerialComm<OrdinalType> ();
379  // We want defaultSerialComm_ to be deallocated when main exits,
380  // so register its deallocation function as an atexit handler.
381  //
382  // The POSIX standard allows atexit to fail, in particular if it
383  // lacks space for registering more functions. "[T]he
384  // application should call sysconf() to obtain the value of
385  // {ATEXIT_MAX}, the [maximum] number of functions that can be
386  // registered. There is no way for an application to tell how
387  // many functions have already been registered with atexit()."
388  //
389  // We don't do this here. Instead, we just check atexit's
390  // return code. If it fails, we throw.
391  int err = atexit (freeDefaultComm);
392  if (err != 0) {
393  if (defaultSerialComm_ != NULL) { // clean up if atexit fails
394  delete defaultSerialComm_;
395  defaultSerialComm_ = NULL;
396  }
397  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
398  "Teuchos::DefaultComm::getDefaultSerialComm: atexit failed!");
399  }
400 #endif // HAVE_MPI
401  }
402 
404  (defaultSerialComm_ == NULL, std::logic_error, "Teuchos::DefaultComm::"
405  "getDefaultSerialComm: defaultSerialComm_ == NULL before return. This sh"
406  "ould never happen. Please report this bug to the Teuchos developers.");
407 
408  // Return a nonowning RCP, because we need to ensure that
409  // destruction happens at MPI_Finalize (or at exit of main(), if not
410  // building with MPI).
411  return rcp (defaultSerialComm_, false);
412  }
413 }
414 
415 template<typename OrdinalType>
418 
419 template<typename OrdinalType>
422 
423 } // namespace Teuchos
424 
425 #endif // TEUCHOS_DEFAULT_COMM_HPP
static Teuchos::RCP< const Comm< OrdinalType > > getDefaultSerialComm(const Teuchos::RCP< const Comm< OrdinalType > > &comm)
Return a serial Comm if the input Comm is null.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Concrete serial communicator subclass.
static Teuchos::RCP< const Comm< OrdinalType > > getComm()
Return the default global communicator.
Namespace of implementation details.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
Implementation of Teuchos wrappers for MPI.
Return a default global communicator appropriate for the build.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos, as well as a number of utility routines.
Smart reference counting pointer class for automatic garbage collection.
Reference-counted pointer class and non-member templated function implementations.