Stokhos Package Browser (Single Doxygen Collection) Version of the Day
Loading...
Searching...
No Matches
MPAssembly/BoxElemPart.cpp
Go to the documentation of this file.
1/*
2//@HEADER
3// ************************************************************************
4//
5// Kokkos: Manycore Performance-Portable Multidimensional Arrays
6// Copyright (2012) Sandia Corporation
7//
8// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9// the U.S. Government retains certain rights in this software.
10//
11// Redistribution and use in source and binary forms, with or without
12// modification, are permitted provided that the following conditions are
13// met:
14//
15// 1. Redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer.
17//
18// 2. Redistributions in binary form must reproduce the above copyright
19// notice, this list of conditions and the following disclaimer in the
20// documentation and/or other materials provided with the distribution.
21//
22// 3. Neither the name of the Corporation nor the names of the
23// contributors may be used to endorse or promote products derived from
24// this software without specific prior written permission.
25//
26// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37//
38// Questions? Contact H. Carter Edwards (hcedwar@sandia.gov)
39//
40// ************************************************************************
41//@HEADER
42*/
43
44#include <utility>
45#include <iostream>
46#include <sstream>
47#include <stdexcept>
48#include <limits>
49#include <BoxElemPart.hpp>
50
51//----------------------------------------------------------------------------
52
53namespace Kokkos {
54namespace Example {
55
56void box_partition( const size_t global_size ,
57 const size_t global_rank ,
58 const size_t global_box[][2] ,
59 size_t box[][2] )
60{
61 box[0][0] = global_box[0][0] ; box[0][1] = global_box[0][1] ;
62 box[1][0] = global_box[1][0] ; box[1][1] = global_box[1][1] ;
63 box[2][0] = global_box[2][0] ; box[2][1] = global_box[2][1] ;
64
65 size_t ip = 0 ;
66 size_t np = global_size ;
67
68 while ( 1 < np ) {
69
70 // P = [ ip + j * portion , ip + ( j + 1 ) * portion )
71
72 size_t jip , jup ;
73
74 {
75 const size_t part = ( 0 == ( np % 5 ) ) ? 5 : (
76 ( 0 == ( np % 3 ) ) ? 3 : 2 );
77
78 const size_t portion = np / part ;
79
80 if ( 2 < part || global_rank < ip + portion ) {
81 jip = portion * size_t( double( global_rank - ip ) / double(portion) );
82 jup = jip + portion ;
83 }
84 else {
85 jip = portion ;
86 jup = np ;
87 }
88 }
89
90 // Choose axis with largest count:
91
92 const size_t nb[3] = {
93 box[0][1] - box[0][0] ,
94 box[1][1] - box[1][0] ,
95 box[2][1] - box[2][0] };
96
97 const int axis = nb[2] > nb[1] ? ( nb[2] > nb[0] ? 2 : 0 )
98 : ( nb[1] > nb[0] ? 1 : 0 );
99
100 box[ axis ][1] = box[ axis ][0] + size_t( double(nb[axis]) * ( double(jup) / double(np) ));
101 box[ axis ][0] = box[ axis ][0] + size_t( double(nb[axis]) * ( double(jip) / double(np) ));
102
103 np = jup - jip ;
104 ip = ip + jip ;
105 }
106}
107
108} /* namespace Example */
109} /* namespace Kokkos */
110
111//----------------------------------------------------------------------------
112
113namespace Kokkos {
114namespace Example {
115
116void BoxElemPart::local( const size_t rank ,
117 size_t uses_elem[][2] ,
118 size_t owns_node[][2] ,
119 size_t uses_node[][2] ) const
120{
122
124
125 for ( int i = 0 ; i < 3 ; ++i ) {
126 owns_node[i][0] = uses_elem[i][0] ;
127 owns_node[i][1] = uses_elem[i][1] + ( m_global_elem_box[i][1] == uses_elem[i][1] ? 1 : 0 );
128 }
129 }
130 else {
131
132 const size_t global_vert[3][2] =
133 { { 0 , m_global_elem_box[0][1] + 1 },
134 { 0 , m_global_elem_box[1][1] + 1 },
135 { 0 , m_global_elem_box[2][1] + 1 } };
136
137 Kokkos::Example::box_partition( m_global_size , rank , global_vert , owns_node );
138
139 for ( int i = 0 ; i < 3 ; ++i ) {
140 uses_elem[i][0] = global_vert[i][0] == owns_node[i][0] ? owns_node[i][0] : owns_node[i][0] - 1 ;
141 uses_elem[i][1] = global_vert[i][1] == owns_node[i][1] ? owns_node[i][1] - 1 : owns_node[i][1] ;
142 }
143 }
144
145 for ( int i = 0 ; i < 3 ; ++i ) {
146 uses_node[i][0] = uses_elem[i][0] ;
147 uses_node[i][1] = uses_elem[i][1] + 1 ;
148 }
149
151 for ( int i = 0 ; i < 3 ; ++i ) {
152 owns_node[i][0] = 2 * owns_node[i][0] ;
153 uses_node[i][0] = 2 * uses_node[i][0] ;
154 owns_node[i][1] = 2 * owns_node[i][1] - 1 ;
155 uses_node[i][1] = 2 * uses_node[i][1] - 1 ;
156 }
157 }
158}
159
161 const BoxElemPart::ElemOrder elem_order ,
162 const BoxElemPart::Decompose decompose ,
163 const size_t global_size ,
164 const size_t global_rank ,
165 const size_t elem_nx ,
166 const size_t elem_ny ,
167 const size_t elem_nz )
168{
169 m_global_size = global_size ;
170 m_global_rank = global_rank ;
171 m_decompose = decompose ;
172 m_elem_order = elem_order ;
173
174 m_global_elem_box[0][0] = 0 ; m_global_elem_box[0][1] = elem_nx ;
175 m_global_elem_box[1][0] = 0 ; m_global_elem_box[1][1] = elem_ny ;
176 m_global_elem_box[2][0] = 0 ; m_global_elem_box[2][1] = elem_nz ;
177
178 m_global_node_box[0][0] = 0 ; m_global_node_box[0][1] = 0 ;
179 m_global_node_box[1][0] = 0 ; m_global_node_box[1][1] = 0 ;
180 m_global_node_box[2][0] = 0 ; m_global_node_box[2][1] = 0 ;
181
184
185 m_ok = true ;
186
187 //----------------------------------------
188
189 if ( ElemLinear == elem_order ) {
190 m_global_node_box[0][1] = elem_nx + 1 ;
191 m_global_node_box[1][1] = elem_ny + 1 ;
192 m_global_node_box[2][1] = elem_nz + 1 ;
193 }
194 else if ( ElemQuadratic == elem_order ) {
195 m_global_node_box[0][1] = 2 * elem_nx + 1 ;
196 m_global_node_box[1][1] = 2 * elem_ny + 1 ;
197 m_global_node_box[2][1] = 2 * elem_nz + 1 ;
198 }
199
200 //----------------------------------------
201
203
206
207 //----------------------------------------
208
209 size_t elem_count = Kokkos::Example::box_count( m_uses_elem_box );
210 size_t node_count = Kokkos::Example::box_count( m_owns_node_box[0] );
211
212 m_owns_node[0][0] = global_rank ;
213 m_owns_node[0][1] = node_count ;
216
217 for ( size_t rr = 1 ; rr < m_global_size && m_ok ; ++rr ) {
218
219 const size_t rank = ( m_global_rank + rr ) % m_global_size ;
220
221 size_t elem_box[3][2] , o_node_box[3][2] , u_node_box[3][2] ;
222
223 // Boxes for process 'rank'
224 local( rank , elem_box , o_node_box , u_node_box );
225
226 // Box that this process uses but is owned by process 'rank'
228
230
231 if ( m_owns_node[ m_owns_node_count ][1] ) {
232
233 if ( ( PROC_NEIGH_MAX - 1 ) <= m_owns_node_count ) {
234 std::cout << "BoxElemPart exceeded maximum neighbor count" << std::endl ;
235 m_ok = false ;
236 break ;
237 }
238
239 m_owns_node[ m_owns_node_count ][0] = rank ;
240
242 }
243
244 // Box that this process owns and is used by process 'rank'
246
248
249 if ( m_send_node[ m_send_node_count ][1] ) {
250
251 if ( ( PROC_NEIGH_MAX - 1 ) <= m_send_node_count ) {
252 std::cout << "BoxElemPart exceeded maximum neighbor count" << std::endl ;
253 m_ok = false ;
254 break ;
255 }
256
257 m_send_node[ m_send_node_count ][0] = rank ;
259 }
260
261 // Error checking:
262
263 size_t test_box[3][2] ;
264
265 elem_count += Kokkos::Example::box_count( elem_box );
266 node_count += Kokkos::Example::box_count( o_node_box );
267
268 {
269 Kokkos::Example::box_intersect( test_box , m_owns_node_box[0] , o_node_box );
270
271 if ( Kokkos::Example::box_count( test_box ) ) {
272 std::cout << "Box partitioning error" << std::endl ;
273 std::cout << "owns_node[" << m_global_rank << "]{"
274 << " [" << m_owns_node_box[0][0][0] << "," << m_owns_node_box[0][0][1] << ")"
275 << " [" << m_owns_node_box[0][1][0] << "," << m_owns_node_box[0][1][1] << ")"
276 << " [" << m_owns_node_box[0][2][0] << "," << m_owns_node_box[0][2][1] << ")"
277 << "} intersects"
278 << " owns_node[" << rank << "]{"
279 << " [" << o_node_box[0][0] << "," << o_node_box[0][1] << ")"
280 << " [" << o_node_box[1][0] << "," << o_node_box[1][1] << ")"
281 << " [" << o_node_box[2][0] << "," << o_node_box[2][1] << ")"
282 << "}" << std::endl ;
283 m_ok = false ;
284 break ;
285 }
286 }
287
288 if ( DecomposeElem == decompose ) {
289
290 Kokkos::Example::box_intersect( test_box , m_uses_elem_box , elem_box );
291
292 if ( Kokkos::Example::box_count( test_box ) ) {
293 std::cout << "Box partitioning error" << std::endl ;
294 std::cout << "ElemBox[" << m_global_rank << "]{"
295 << " [" << m_uses_elem_box[0][0] << "," << m_uses_elem_box[0][1] << ")"
296 << " [" << m_uses_elem_box[1][0] << "," << m_uses_elem_box[1][1] << ")"
297 << " [" << m_uses_elem_box[2][0] << "," << m_uses_elem_box[2][1] << ")"
298 << "} intersects"
299 << " ElemBox[" << rank << "]{"
300 << " [" << elem_box[0][0] << "," << elem_box[0][1] << ")"
301 << " [" << elem_box[1][0] << "," << elem_box[1][1] << ")"
302 << " [" << elem_box[2][0] << "," << elem_box[2][1] << ")"
303 << "}" << std::endl ;
304 m_ok = false ;
305 break ;
306 }
307 }
308 }
309
310 // Sentinal values at the end of the owns and send lists:
311
312 m_owns_node[ m_owns_node_count ][0] = ~0u ;
313 m_owns_node[ m_owns_node_count ][1] = ~0u ;
317
318 m_send_node[ m_send_node_count ][0] = ~0u ;
319 m_send_node[ m_send_node_count ][1] = ~0u ;
323
324 {
325 size_t count = 0 ;
326 for ( size_t i = 0 ; i < m_owns_node_count ; ++i ) {
327 count += m_owns_node[i][1] ;
328 }
330 std::cout << "Node uses count = " << Kokkos::Example::box_count( m_uses_node_box )
331 << " error count = " << count << std::endl ;
332 m_ok = false ;
333 }
334 }
335
336 if ( global_node_count != node_count ) {
337 std::cout << "Node count = " << global_node_count << " overlap error count = " << node_count << std::endl ;
338 m_ok = false ;
339 }
340
341 if ( DecomposeElem == decompose && global_elem_count != elem_count ) {
342 std::cout << "Elem count = " << global_elem_count << " overlap error count = " << elem_count << std::endl ;
343 m_ok = false ;
344 }
345
346 if ( ! m_ok ) {
347 for ( int i = 0 ; i < 3 ; ++i ) { for ( int j = 0 ; j < 2 ; ++j ) {
348 m_global_elem_box[i][j] = 0 ;
349 m_global_node_box[i][j] = 0 ;
350 m_uses_elem_box[i][j] = 0 ;
351 m_uses_node_box[i][j] = 0 ;
352 }}
355 }
356}
357
358void BoxElemPart::print( std::ostream & s ) const
359{
360 s << "BoxElemPart P[" << m_global_rank << ":" << m_global_size << "]"
361 << std::endl
362 << " elem_box {"
363 << " [" << m_uses_elem_box[0][0] << "," << m_uses_elem_box[0][1] << ")"
364 << " [" << m_uses_elem_box[1][0] << "," << m_uses_elem_box[1][1] << ")"
365 << " [" << m_uses_elem_box[2][0] << "," << m_uses_elem_box[2][1] << ")"
366 << " } / {"
367 << " [" << m_global_elem_box[0][0] << "," << m_global_elem_box[0][1] << ")"
368 << " [" << m_global_elem_box[1][0] << "," << m_global_elem_box[1][1] << ")"
369 << " [" << m_global_elem_box[2][0] << "," << m_global_elem_box[2][1] << ")"
370 << " }"
371 << std::endl
372 << " node_box {"
373 << " [" << m_owns_node_box[0][0][0] << "," << m_owns_node_box[0][0][1] << ")"
374 << " [" << m_owns_node_box[0][1][0] << "," << m_owns_node_box[0][1][1] << ")"
375 << " [" << m_owns_node_box[0][2][0] << "," << m_owns_node_box[0][2][1] << ")"
376 << " } / {"
377 << " [" << m_uses_node_box[0][0] << "," << m_uses_node_box[0][1] << ")"
378 << " [" << m_uses_node_box[1][0] << "," << m_uses_node_box[1][1] << ")"
379 << " [" << m_uses_node_box[2][0] << "," << m_uses_node_box[2][1] << ")"
380 << " } / {"
381 << " [" << m_global_node_box[0][0] << "," << m_global_node_box[0][1] << ")"
382 << " [" << m_global_node_box[1][0] << "," << m_global_node_box[1][1] << ")"
383 << " [" << m_global_node_box[2][0] << "," << m_global_node_box[2][1] << ")"
384 << " }"
385 << std::endl ;
386
387 for ( size_t i = 1 ; i < m_owns_node_count ; ++i ) {
388 s << " P[" << m_owns_node[i][0] << "]"
389 << " recv node_box {"
390 << " [" << m_owns_node_box[i][0][0] << "," << m_owns_node_box[i][0][1] << ")"
391 << " [" << m_owns_node_box[i][1][0] << "," << m_owns_node_box[i][1][1] << ")"
392 << " [" << m_owns_node_box[i][2][0] << "," << m_owns_node_box[i][2][1] << ")"
393 << " }"
394 << std::endl ;
395 }
396
397 for ( size_t i = 0 ; i < m_send_node_count ; ++i ) {
398 s << " P[" << m_send_node[i][0] << "]"
399 << " send node_box {"
400 << " [" << m_send_node_box[i][0][0] << "," << m_send_node_box[i][0][1] << ")"
401 << " [" << m_send_node_box[i][1][0] << "," << m_send_node_box[i][1][1] << ")"
402 << " [" << m_send_node_box[i][2][0] << "," << m_send_node_box[i][2][1] << ")"
403 << " }"
404 << std::endl ;
405 }
406}
407
408} /* namespace Example */
409} /* namespace Kokkos */
410
411//----------------------------------------------------------------------------
412
413
unsigned m_owns_node_box[PROC_NEIGH_MAX][3][2]
BoxElemPart(const ElemOrder elem_order, const Decompose decompose, const unsigned global_size, const unsigned global_rank, const unsigned elem_nx, const unsigned elem_ny, const unsigned elem_nz)
KOKKOS_INLINE_FUNCTION size_t global_elem_count() const
unsigned m_send_node_box[PROC_NEIGH_MAX][3][2]
unsigned m_send_node[PROC_NEIGH_MAX][2]
unsigned m_owns_node[PROC_NEIGH_MAX][2]
void local(const unsigned rank, unsigned uses_elem[][2], unsigned owns_node[][2], unsigned uses_node[][2]) const
KOKKOS_INLINE_FUNCTION size_t global_node_count() const
KOKKOS_INLINE_FUNCTION void box_intersect(unsigned box[][2], const unsigned boxA[][2], const unsigned boxB[][2])
KOKKOS_INLINE_FUNCTION size_t box_count(const unsigned box[][2])
void box_partition(const unsigned global_size, const unsigned global_rank, const unsigned global_box[][2], unsigned box[][2])