Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Loading...
Searching...
No Matches
Teuchos_MpiReductionOpSetter.cpp
Go to the documentation of this file.
1// @HEADER
2// ***********************************************************************
3//
4// Teuchos: Common Tools Package
5// Copyright (2004) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38//
39// ***********************************************************************
40// @HEADER
41
43
44#ifdef HAVE_MPI
45# ifdef MPIAPI
46# define CALL_API MPIAPI
47# else
48# define CALL_API
49# endif
50
51//
52// mfh 23 Nov 2014: My commits over the past day or two attempt to
53// address Bug 6263. In particular, the code as I found it had the
54// following issues:
55//
56// 1. Static RCP instances (that persist past return of main())
57// 2. Static MPI_Op datum (that persists past MPI_Finalize())
58// 3. Code won't work with MPI_THREAD_{SERIALIZED,MULTIPLE},
59// because it assumes that only one MPI_Op for reductions
60// is needed at any one time
61//
62// I'm neglecting Issue #3 for now and focusing on the first two
63// issues. #1 goes away if one doesn't use RCPs and handles
64// deallocation manually (we could also use std::shared_ptr, but that
65// would require C++11). #2 goes away with the standard idiom of an
66// MPI_Finalize() hook (attach a (key,value) pair to MPI_COMM_SELF).
67//
68
69extern "C" {
70
71// The MPI_Op that implements the reduction or scan operation will
72// call this function. We only need to create the MPI_Op once
73// (lazily, on demand). This function in turn will invoke
74// theReductOp_ (see below), which gets set to the current reduction
75// operation. Thus, we only never need to create one MPI_Op, but we
76// swap out the function. This is meant to save overhead in creating
77// and freeing MPI_Op for each reduction or scan.
78void CALL_API
79Teuchos_MPI_reduction_op (void* invec, void* inoutvec,
80 int* len, MPI_Datatype* datatype);
81} // extern "C"
82
83namespace { // anonymous
84
85//
86// theMpiOp_: The MPI_Op singleton that implements the Teuchos
87// reduction or scan operation. We only need to create the MPI_Op
88// once (lazily, on demand). When we create the MPI_Op, we stash its
89// "destructor" in MPI_COMM_SELF so that it gets freed at
90// MPI_Finalize. (This is a standard MPI idiom.)
91//
92// This variable is global, persistent (until MPI_Finalize is called),
93// and initialized lazily.
94MPI_Op theMpiOp_ = MPI_OP_NULL;
95
96// The current reduction or scan "function." (It's actually a class
97// instance.)
98//
99// This static variable is _NOT_ persistent. It does not need
100// deallocation.
101const Teuchos::Details::MpiReductionOpBase* theReductOp_ = NULL;
102
103// Free the given MPI_Op, and return the error code returned by MPI_Op_free.
104int
105freeMpiOp (MPI_Op* op)
106{
107 // If this function is called as an MPI_Finalize hook, MPI should
108 // still be initialized at this point, and it should be OK to call
109 // MPI functions. Thus, we don't need to check if MPI is
110 // initialized.
111 int err = MPI_SUCCESS;
112 if (op != NULL) {
113 err = MPI_Op_free (op);
114 if (err == MPI_SUCCESS) {
115 // No externally visible side effects unless the above function succeeded.
116 *op = MPI_OP_NULL;
117 }
118 }
119 return err;
120}
121
122// Free the MPI_Op singleton (theMpiOp_), and return the error code
123// returned by freeMpiOp(). As a side effect, if freeing succeeds,
124// set theMpiOp_ to MPI_OP_NULL.
125//
126// This is the singleton's "destructor" that we attach to
127// MPI_COMM_SELF as an MPI_Finalize hook.
128int
129freeMpiOpCallback (MPI_Comm, int, void*, void*)
130{
131 // We don't need any of the arguments to this function, since we're
132 // just freeing the singleton.
133 if (theMpiOp_ == MPI_OP_NULL) {
134 return MPI_SUCCESS;
135 } else {
136 return freeMpiOp (&theMpiOp_);
137 }
138}
139
140// Create the MPI_Op singleton that invokes the
141// Teuchos_MPI_reduction_op callback. Assign the MPI_Op to theMpiOp_,
142// and set it up with an MPI_Finalize hook so it gets freed
143// automatically.
144void createReductOp ()
145{
146#if MPI_VERSION >= 2
147
148 // This function has side effects on the global singleton theMpiOp_.
149 // This check ensures that the function is idempotent. We only need
150 // to create the MPI_Op singleton once.
151 if (theMpiOp_ != MPI_OP_NULL) {
152 return; // We've already called this function; we don't have to again.
153 }
154
155 MPI_Op mpi_op = MPI_OP_NULL;
156
157 // FIXME (mfh 23 Nov 2014) I found the following comment here:
158 // "Assume op is commutative". That's what it means to pass 1 as
159 // the second argument. I don't know whether it's a good idea to
160 // keep that assumption.
161 int err = MPI_Op_create (&Teuchos_MPI_reduction_op, 1, &mpi_op);
163 err != MPI_SUCCESS, std::runtime_error, "Teuchos::createReductOp: "
164 "MPI_Op_create (for custom reduction operator) failed!");
165
166 // Use the standard MPI idiom (attach a (key,value) pair to
167 // MPI_COMM_SELF with a "destructor" function) in order that
168 // theMpiOp_ gets freed at MPI_Finalize, if necessary.
169
170 // 'key' is an output argument of MPI_Comm_create_keyval.
171 int key = MPI_KEYVAL_INVALID;
172 err = MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, freeMpiOpCallback,
173 &key, NULL);
174 if (err != MPI_SUCCESS) {
175 // Attempt to clean up by freeing the newly created MPI_Op. If
176 // cleaning up fails, just let it slide, since we're already in
177 // trouble if MPI can't create a (key,value) pair.
178 (void) MPI_Op_free (&mpi_op);
180 true, std::runtime_error, "Teuchos::createReductOp: "
181 "MPI_Comm_create_keyval (for custom reduction operator) failed!");
182 }
183 int val = key; // doesn't matter
184
185 // Attach the attribute to MPI_COMM_SELF.
186 err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
187 if (err != MPI_SUCCESS) {
188 // MPI (versions up to and including 3.0) doesn't promise correct
189 // behavior after any function returns something other than
190 // MPI_SUCCESS. Thus, it's not required to try to free the new
191 // key via MPI_Comm_free_keyval. Furthermore, if something went
192 // wrong with MPI_Comm_set_attr, it's likely that the attribute
193 // mechanism is broken. Thus, it would be unwise to call
194 // MPI_Comm_free_keyval.
195 //
196 // I optimistically assume that the "rest" of MPI is still
197 // working, and attempt to clean up by freeing the newly created
198 // MPI_Op. If cleaning up fails, just let it slide, since we're
199 // already in trouble if MPI can't create a (key,value) pair.
200 (void) MPI_Op_free (&mpi_op);
202 true, std::runtime_error, "Teuchos::createReductOp: "
203 "MPI_Comm_set_attr (for custom reduction operator) failed!");
204 }
205
206 // It looks weird to "free" the key right away. However, this does
207 // not actually cause the "destructor" to be called. It only gets
208 // called at MPI_FINALIZE. See MPI 3.0 standard, Section 6.7.2,
209 // MPI_COMM_FREE_KEYVAL:
210 //
211 // "Note that it is not erroneous to free an attribute key that is
212 // in use, because the actual free does not transpire until after
213 // all references (in other communicators on the process) to the key
214 // have been freed. These references need to be explicitly freed by
215 // the program, either via calls to MPI_COMM_DELETE_ATTR that free
216 // one attribute instance, or by calls to MPI_COMM_FREE that free
217 // all attribute instances associated with the freed communicator."
218 //
219 // We rely here on the latter mechanism. MPI_FINALIZE calls
220 // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
221 // explicitly.
222 //
223 // It's not clear what to do if the MPI_* calls above succeeded, but
224 // this call fails (i.e., returns != MPI_SUCCESS). We could throw;
225 // this would make sense to do, because MPI (versions up to and
226 // including 3.0) doesn't promise correct behavior after any MPI
227 // function returns something other than MPI_SUCCESS. We could also
228 // be optimistic and just ignore the return value, hoping that if
229 // the above calls succeeded, then the communicator will get freed
230 // at MPI_FINALIZE, even though the unfreed key may leak memory (see
231 // Bug 6338). I've chosen the latter.
232 (void) MPI_Comm_free_keyval (&key);
233
234 // The "transaction" succeeded; save the result.
235 theMpiOp_ = mpi_op;
236
237#else // MPI_VERSION < 2
238# error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
239#endif // MPI_VERSION >= 2
240}
241
242void
243setReductOp (const Teuchos::Details::MpiReductionOpBase* reductOp)
244{
245 if (theMpiOp_ == MPI_OP_NULL) {
246 createReductOp ();
247 }
248 theReductOp_ = reductOp;
249}
250
251} // namespace (anonymous)
252
253extern "C" {
254
255void CALL_API
256Teuchos_MPI_reduction_op (void* invec,
257 void* inoutvec,
258 int* len,
259 MPI_Datatype* datatype)
260{
261 if (theReductOp_ != NULL) {
262 theReductOp_->reduce (invec, inoutvec, len, datatype);
263 }
264}
265
266} // extern "C"
267
268namespace Teuchos {
269namespace Details {
270
271MPI_Op setMpiReductionOp (const MpiReductionOpBase& reductOp)
272{
273 setReductOp (&reductOp);
275 (theMpiOp_ == MPI_OP_NULL, std::logic_error, "Teuchos::Details::"
276 "setMpiReductionOp: Failed to create reduction MPI_Op theMpiOp_. "
277 "This should never happen. "
278 "Please report this bug to the Teuchos developers.");
279 return theMpiOp_;
280}
281
282} // namespace Details
283} // namespace Teuchos
284
285#endif // HAVE_MPI
Implementation detail of Teuchos' MPI wrapper.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Namespace of implementation details.