Teuchos Package Browser (Single Doxygen Collection)  Version of the Day
Teuchos_MpiReductionOpSetter.cpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
43 
44 #ifdef HAVE_MPI
45 # ifdef MPIAPI
46 # define CALL_API MPIAPI
47 # else
48 # define CALL_API
49 # endif
50 
51 //
52 // mfh 23 Nov 2014: My commits over the past day or two attempt to
53 // address Bug 6263. In particular, the code as I found it had the
54 // following issues:
55 //
56 // 1. Static RCP instances (that persist past return of main())
57 // 2. Static MPI_Op datum (that persists past MPI_Finalize())
58 // 3. Code won't work with MPI_THREAD_{SERIALIZED,MULTIPLE},
59 // because it assumes that only one MPI_Op for reductions
60 // is needed at any one time
61 //
62 // I'm neglecting Issue #3 for now and focusing on the first two
63 // issues. #1 goes away if one doesn't use RCPs and handles
64 // deallocation manually (we could also use std::shared_ptr, but that
65 // would require C++11). #2 goes away with the standard idiom of an
66 // MPI_Finalize() hook (attach a (key,value) pair to MPI_COMM_SELF).
67 //
68 
69 extern "C" {
70 
71 // The MPI_Op that implements the reduction or scan operation will
72 // call this function. We only need to create the MPI_Op once
73 // (lazily, on demand). This function in turn will invoke
74 // theReductOp_ (see below), which gets set to the current reduction
75 // operation. Thus, we only never need to create one MPI_Op, but we
76 // swap out the function. This is meant to save overhead in creating
77 // and freeing MPI_Op for each reduction or scan.
78 void CALL_API
79 Teuchos_MPI_reduction_op (void* invec, void* inoutvec,
80  int* len, MPI_Datatype* datatype);
81 } // extern "C"
82 
83 namespace { // anonymous
84 
85 //
86 // theMpiOp_: The MPI_Op singleton that implements the Teuchos
87 // reduction or scan operation. We only need to create the MPI_Op
88 // once (lazily, on demand). When we create the MPI_Op, we stash its
89 // "destructor" in MPI_COMM_SELF so that it gets freed at
90 // MPI_Finalize. (This is a standard MPI idiom.)
91 //
92 // This variable is global, persistent (until MPI_Finalize is called),
93 // and initialized lazily.
94 MPI_Op theMpiOp_ = MPI_OP_NULL;
95 
96 // The current reduction or scan "function." (It's actually a class
97 // instance.)
98 //
99 // This static variable is _NOT_ persistent. It does not need
100 // deallocation.
101 const Teuchos::Details::MpiReductionOpBase* theReductOp_ = NULL;
102 
103 // Free the given MPI_Op, and return the error code returned by MPI_Op_free.
104 int
105 freeMpiOp (MPI_Op* op)
106 {
107  // If this function is called as an MPI_Finalize hook, MPI should
108  // still be initialized at this point, and it should be OK to call
109  // MPI functions. Thus, we don't need to check if MPI is
110  // initialized.
111  int err = MPI_SUCCESS;
112  if (op != NULL) {
113  err = MPI_Op_free (op);
114  if (err == MPI_SUCCESS) {
115  // No externally visible side effects unless the above function succeeded.
116  *op = MPI_OP_NULL;
117  }
118  }
119  return err;
120 }
121 
122 // Free the MPI_Op singleton (theMpiOp_), and return the error code
123 // returned by freeMpiOp(). As a side effect, if freeing succeeds,
124 // set theMpiOp_ to MPI_OP_NULL.
125 //
126 // This is the singleton's "destructor" that we attach to
127 // MPI_COMM_SELF as an MPI_Finalize hook.
128 int
129 freeMpiOpCallback (MPI_Comm, int, void*, void*)
130 {
131  // We don't need any of the arguments to this function, since we're
132  // just freeing the singleton.
133  if (theMpiOp_ == MPI_OP_NULL) {
134  return MPI_SUCCESS;
135  } else {
136  return freeMpiOp (&theMpiOp_);
137  }
138 }
139 
140 // Create the MPI_Op singleton that invokes the
141 // Teuchos_MPI_reduction_op callback. Assign the MPI_Op to theMpiOp_,
142 // and set it up with an MPI_Finalize hook so it gets freed
143 // automatically.
144 void createReductOp ()
145 {
146  // This function has side effects on the global singleton theMpiOp_.
147  // This check ensures that the function is idempotent. We only need
148  // to create the MPI_Op singleton once.
149  if (theMpiOp_ != MPI_OP_NULL) {
150  return; // We've already called this function; we don't have to again.
151  }
152 
153  MPI_Op mpi_op = MPI_OP_NULL;
154 
155  // FIXME (mfh 23 Nov 2014) I found the following comment here:
156  // "Assume op is commutative". That's what it means to pass 1 as
157  // the second argument. I don't know whether it's a good idea to
158  // keep that assumption.
159  int err = MPI_Op_create (&Teuchos_MPI_reduction_op, 1, &mpi_op);
161  err != MPI_SUCCESS, std::runtime_error, "Teuchos::createReductOp: "
162  "MPI_Op_create (for custom reduction operator) failed!");
163 
164  // Use the standard MPI idiom (attach a (key,value) pair to
165  // MPI_COMM_SELF with a "destructor" function) in order that
166  // theMpiOp_ gets freed at MPI_Finalize, if necessary.
167 
168  // 'key' is an output argument of MPI_Comm_create_keyval.
169  int key = MPI_KEYVAL_INVALID;
170  err = MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, freeMpiOpCallback,
171  &key, NULL);
172  if (err != MPI_SUCCESS) {
173  // Attempt to clean up by freeing the newly created MPI_Op. If
174  // cleaning up fails, just let it slide, since we're already in
175  // trouble if MPI can't create a (key,value) pair.
176  (void) MPI_Op_free (&mpi_op);
178  true, std::runtime_error, "Teuchos::createReductOp: "
179  "MPI_Comm_create_keyval (for custom reduction operator) failed!");
180  }
181  int val = key; // doesn't matter
182 
183  // Attach the attribute to MPI_COMM_SELF.
184  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
185  if (err != MPI_SUCCESS) {
186  // MPI (versions up to and including 3.0) doesn't promise correct
187  // behavior after any function returns something other than
188  // MPI_SUCCESS. Thus, it's not required to try to free the new
189  // key via MPI_Comm_free_keyval. Furthermore, if something went
190  // wrong with MPI_Comm_set_attr, it's likely that the attribute
191  // mechanism is broken. Thus, it would be unwise to call
192  // MPI_Comm_free_keyval.
193  //
194  // I optimistically assume that the "rest" of MPI is still
195  // working, and attempt to clean up by freeing the newly created
196  // MPI_Op. If cleaning up fails, just let it slide, since we're
197  // already in trouble if MPI can't create a (key,value) pair.
198  (void) MPI_Op_free (&mpi_op);
200  true, std::runtime_error, "Teuchos::createReductOp: "
201  "MPI_Comm_set_attr (for custom reduction operator) failed!");
202  }
203 
204  // It looks weird to "free" the key right away. However, this does
205  // not actually cause the "destructor" to be called. It only gets
206  // called at MPI_FINALIZE. See MPI 3.0 standard, Section 6.7.2,
207  // MPI_COMM_FREE_KEYVAL:
208  //
209  // "Note that it is not erroneous to free an attribute key that is
210  // in use, because the actual free does not transpire until after
211  // all references (in other communicators on the process) to the key
212  // have been freed. These references need to be explicitly freed by
213  // the program, either via calls to MPI_COMM_DELETE_ATTR that free
214  // one attribute instance, or by calls to MPI_COMM_FREE that free
215  // all attribute instances associated with the freed communicator."
216  //
217  // We rely here on the latter mechanism. MPI_FINALIZE calls
218  // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
219  // explicitly.
220  //
221  // It's not clear what to do if the MPI_* calls above succeeded, but
222  // this call fails (i.e., returns != MPI_SUCCESS). We could throw;
223  // this would make sense to do, because MPI (versions up to and
224  // including 3.0) doesn't promise correct behavior after any MPI
225  // function returns something other than MPI_SUCCESS. We could also
226  // be optimistic and just ignore the return value, hoping that if
227  // the above calls succeeded, then the communicator will get freed
228  // at MPI_FINALIZE, even though the unfreed key may leak memory (see
229  // Bug 6338). I've chosen the latter.
230  (void) MPI_Comm_free_keyval (&key);
231 
232  // The "transaction" succeeded; save the result.
233  theMpiOp_ = mpi_op;
234 }
235 
236 void
237 setReductOp (const Teuchos::Details::MpiReductionOpBase* reductOp)
238 {
239  if (theMpiOp_ == MPI_OP_NULL) {
240  createReductOp ();
241  }
242  theReductOp_ = reductOp;
243 }
244 
245 } // namespace (anonymous)
246 
247 extern "C" {
248 
249 void CALL_API
250 Teuchos_MPI_reduction_op (void* invec,
251  void* inoutvec,
252  int* len,
253  MPI_Datatype* datatype)
254 {
255  if (theReductOp_ != NULL) {
256  theReductOp_->reduce (invec, inoutvec, len, datatype);
257  }
258 }
259 
260 } // extern "C"
261 
262 namespace Teuchos {
263 namespace Details {
264 
265 MPI_Op setMpiReductionOp (const MpiReductionOpBase& reductOp)
266 {
267  setReductOp (&reductOp);
269  (theMpiOp_ == MPI_OP_NULL, std::logic_error, "Teuchos::Details::"
270  "setMpiReductionOp: Failed to create reduction MPI_Op theMpiOp_. "
271  "This should never happen. "
272  "Please report this bug to the Teuchos developers.");
273  return theMpiOp_;
274 }
275 
276 } // namespace Details
277 } // namespace Teuchos
278 
279 #endif // HAVE_MPI
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Namespace of implementation details.
Implementation detail of Teuchos' MPI wrapper.