Zoltan2
Loading...
Searching...
No Matches
zoltanCompare.cpp
Go to the documentation of this file.
1// @HEADER
2//
3// ***********************************************************************
4//
5// Zoltan2: A package of combinatorial algorithms for scientific computing
6// Copyright 2012 Sandia Corporation
7//
8// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9// the U.S. Government retains certain rights in this software.
10//
11// Redistribution and use in source and binary forms, with or without
12// modification, are permitted provided that the following conditions are
13// met:
14//
15// 1. Redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer.
17//
18// 2. Redistributions in binary form must reproduce the above copyright
19// notice, this list of conditions and the following disclaimer in the
20// documentation and/or other materials provided with the distribution.
21//
22// 3. Neither the name of the Corporation nor the names of the
23// contributors may be used to endorse or promote products derived from
24// this software without specific prior written permission.
25//
26// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37//
38// Questions? Contact Karen Devine (kddevin@sandia.gov)
39// Erik Boman (egboman@sandia.gov)
40// Siva Rajamanickam (srajama@sandia.gov)
41//
42// ***********************************************************************
43//
44// @HEADER
45
55
56#include <Tpetra_MultiVector.hpp>
57#include <zoltan.h>
58
59using Teuchos::RCP;
60using Teuchos::rcp;
61using Teuchos::Comm;
62
63//
64// A few of the tests done by Zoltan in nightly testing.
65//
66
73};
74
75
76#define NUMTESTS 30
77static string testArgs[] = {
78// Filename LB_Method ObjWeightDim NumProcs
79"simple", "phg", "0", "2",
80"simple", "rcb", "0", "2",
81"vwgt2", "rcb", "2", "2",
82
83"bug", "rcb", "1", "3",
84"drake", "rcb", "0", "3",
85"onedbug", "rcb", "0", "3",
86"simple", "rcb", "0", "3",
87"vwgt", "rcb", "1", "3",
88"vwgt", "phg", "1", "3",
89"vwgt2", "rcb", "2", "3",
90
91"simple", "default", "0", "4",
92"ewgt", "hsfc", "0", "4",
93"grid20x19", "hsfc", "0", "4",
94"grid20x19", "hsfc", "0", "4",
95"grid20x19", "hsfc", "0", "4",
96"nograph", "rib", "0", "4",
97"nograph", "phg", "0", "4",
98"simple", "rib", "0", "4",
99"simple", "rib", "0", "4",
100"vwgt2", "rib", "2", "4",
101"simple", "rib", "0", "4",
102"simple", "phg", "0", "4",
103
104"brack2_3", "rcb", "3", "5",
105
106"hammond2", "rcb", "2", "6",
107"degenerateAA", "rcb", "0", "6",
108"degenerate", "rcb", "0", "6",
109"degenerate", "rcb", "0", "6",
110
111"hammond", "rcb", "0", "8",
112"hammond", "phg", "0", "8",
113"vwgt2", "rcb", "2", "8"
114};
115
116typedef Tpetra::CrsGraph<zlno_t, zgno_t, znode_t> tGraph_t;
117typedef Tpetra::CrsMatrix<zscalar_t, zlno_t, zgno_t, znode_t> tMatrix_t;
118typedef Tpetra::MultiVector<zscalar_t, zlno_t, zgno_t, znode_t> tMVector_t;
122
123// Number of ZOLTAN_ID in a zgno_t (for NUM_GID_ENTRIES)
124static constexpr int znGidEnt = sizeof(zgno_t) / sizeof(ZOLTAN_ID_TYPE);
125#define SET_ZID(n,a,b) \
126 {int ZOLTAN_ID_LOOP; \
127 for (ZOLTAN_ID_LOOP = 0; ZOLTAN_ID_LOOP < (n); ZOLTAN_ID_LOOP++) \
128 (a)[ZOLTAN_ID_LOOP] = (b)[ZOLTAN_ID_LOOP]; \
129 }
130
131
133// Zoltan callbacks
134
135static int znumobj(void *data, int *ierr)
136{
137 *ierr = ZOLTAN_OK;
138 tMVector_t *vec = (tMVector_t *) data;
139 return vec->getLocalLength();
140}
141
142static void zobjlist(void *data, int ngid, int nlid,
143 ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
144 int nwgts, float *wgts, int *ierr)
145{
146 *ierr = ZOLTAN_OK;
147 tMVector_t *vec = (tMVector_t *) data;
148 size_t n = vec->getLocalLength();
149
150 for (size_t i = 0; i < n; i++) {
151 zgno_t vgid = vec->getMap()->getGlobalElement(i);
152 ZOLTAN_ID_PTR vgidptr = (ZOLTAN_ID_PTR) &vgid;
153 SET_ZID(znGidEnt, &(gids[i*znGidEnt]), vgidptr);
154 lids[i] = i;
155 }
156
157 for (int w = 0; w < nwgts; w++) {
158 ArrayRCP<const zscalar_t> wvec = vec->getData(w);
159 for (size_t i = 0; i < n; i++)
160 wgts[i*nwgts+w] = wvec[i];
161 }
162}
163
164static int znumgeom(void *data, int *ierr)
165{
166 *ierr = ZOLTAN_OK;
167 tMVector_t *cvec = (tMVector_t *) data;
168 return cvec->getNumVectors();
169}
170
171static void zgeom(void *data, int ngid, int nlid, int nobj,
172 ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
173 int ndim, double *coords, int *ierr)
174{
175 *ierr = ZOLTAN_OK;
176 tMVector_t *vec = (tMVector_t *) data;
177 for (int d = 0; d < ndim; d++) {
178 ArrayRCP<const zscalar_t> cvec = vec->getData(d);
179 for (int i = 0; i < nobj; i++) {
180 coords[lids[i]*ndim+d] = cvec[lids[i]];
181 }
182 }
183}
184
185static void zhgsize(void *data, int *nLists, int *nPins, int *format, int *ierr)
186{
187 tMatrix_t *matrix = (tMatrix_t *) data;
188 *nLists = matrix->getLocalNumRows();
189 *nPins = matrix->getLocalNumEntries();
190 *format = ZOLTAN_COMPRESSED_VERTEX;
191 *ierr = ZOLTAN_OK;
192}
193
194static void zhg(void *data, int ngid, int nLists, int nPins, int format,
195 ZOLTAN_ID_PTR listGids, int *offsets, ZOLTAN_ID_PTR pinGids,
196 int *ierr)
197{
198 tMatrix_t *matrix = (tMatrix_t *) data;
199 RCP<const tGraph_t> graph = matrix->getCrsGraph();
200 zlno_t nrows = graph->getLocalNumRows();
201
202 offsets[0] = 0;
203 for (zlno_t i = 0; i < nrows; i++) {
204
205 zgno_t tmp = graph->getRowMap()->getGlobalElement(i);
206 ZOLTAN_ID_PTR ztmp = (ZOLTAN_ID_PTR) &tmp;
207 SET_ZID(znGidEnt, &(listGids[i*znGidEnt]), ztmp);
208
209 size_t nEntries = graph->getNumEntriesInLocalRow(i);
210 offsets[i+1] = offsets[i] + nEntries;
211
212
213 typename tMatrix_t::local_inds_host_view_type colind;
214 graph->getLocalRowView(i, colind);
215
216 for (size_t j = 0; j < nEntries; j++) {
217 tmp = graph->getColMap()->getGlobalElement(colind[j]);
218 ztmp = (ZOLTAN_ID_PTR) &tmp;
219 SET_ZID(znGidEnt, &(pinGids[(offsets[i]+j)*znGidEnt]), ztmp);
220 }
221 }
222
223 *ierr = ZOLTAN_OK;
224}
225
227// Function to compute both Zoltan2 and Zoltan partitions and print metrics
228
229int run(
230 const RCP<const Comm<int> > &comm,
231 int numGlobalParts,
232 int testCnt,
233 std::string *thisTest
234)
235{
236#ifdef HAVE_ZOLTAN2_MPI
237 // Zoltan needs an MPI comm
238 const Teuchos::MpiComm<int> *tmpicomm =
239 dynamic_cast<const Teuchos::MpiComm<int> *>(comm.getRawPtr());
240 MPI_Comm mpiComm = *(tmpicomm->getRawMpiComm());
241#endif
242
243 int me = comm->getRank();
244 int np = comm->getSize();
245 double tolerance = 1.05;
246
247
249 // Read test data from Zoltan's test directory
251
252 UserInputForTests *uinput;
253 try{
255 thisTest[TESTNAMEOFFSET],
256 comm, true);
257 }
258 catch(std::exception &e){
259 if (me == 0)
260 std::cout << "Test " << testCnt << ": FAIL: UserInputForTests "
261 << e.what() << std::endl;
262 return 1;
263 }
264
265 RCP<tMatrix_t> matrix;
266 try{
267 matrix = uinput->getUITpetraCrsMatrix();
268 }
269 catch(std::exception &e){
270 if (me == 0)
271 std::cout << "Test " << testCnt << ": FAIL: get matrix "
272 << e.what() << std::endl;
273 return 1;
274 }
275
276 RCP<const tMatrix_t> matrixConst = rcp_const_cast<const tMatrix_t>(matrix);
277
278 RCP<tMVector_t> coords;
279 try{
280 coords = uinput->getUICoordinates();
281 }
282 catch(std::exception &e){
283 if (me == 0)
284 std::cout << "Test " << testCnt << ": FAIL: get coordinates "
285 << e.what() << std::endl;
286 return 1;
287 }
288
289 RCP<tMVector_t> weights;
290 try{
291 weights = uinput->getUIWeights();
292 }
293 catch(std::exception &e){
294 if (me == 0)
295 std::cout << "Test " << testCnt << ": FAIL: get weights "
296 << e.what() << std::endl;
297 return 1;
298 }
299 int nWeights = atoi(thisTest[TESTOBJWGTOFFSET].c_str());
300
301 if (me == 0) {
302 std::cout << "Test " << testCnt << " filename = "
303 << thisTest[TESTNAMEOFFSET] << std::endl;
304 std::cout << "Test " << testCnt << " num processors = "
305 << np << std::endl;
306 std::cout << "Test " << testCnt << " zoltan method = "
307 << thisTest[TESTMETHODOFFSET] << std::endl;
308 std::cout << "Test " << testCnt << " num_global_parts = "
309 << numGlobalParts << std::endl;
310 std::cout << "Test " << testCnt << " imbalance_tolerance = "
311 << tolerance << std::endl;
312 std::cout << "Test " << testCnt << " num weights per ID = "
313 << nWeights << std::endl;
314 }
315
317 // PARTITION USING ZOLTAN DIRECTLY
319
320 if (me == 0) std::cout << "Calling Zoltan directly" << std::endl;
321
322# ifdef HAVE_ZOLTAN2_MPI
323 Zoltan zz(mpiComm);
324# else
325 Zoltan zz;
326# endif
327
328 char tmp[56];
329 zz.Set_Param("LB_METHOD", thisTest[TESTMETHODOFFSET]);
330
331 sprintf(tmp, "%d", znGidEnt);
332 zz.Set_Param("NUM_GID_ENTRIES", tmp);
333 sprintf(tmp, "%d", numGlobalParts);
334 zz.Set_Param("NUM_GLOBAL_PARTS", tmp);
335 sprintf(tmp, "%d", nWeights);
336 zz.Set_Param("OBJ_WEIGHT_DIM", tmp);
337 sprintf(tmp, "%f", tolerance);
338 zz.Set_Param("IMBALANCE_TOL", tmp);
339 zz.Set_Param("RETURN_LISTS", "PART");
340 zz.Set_Param("FINAL_OUTPUT", "1");
341 zz.Set_Param("SEED", "1111");
342 zz.Set_Param("LB_APPROACH", "PARTITION");
343
344 zz.Set_Num_Obj_Fn(znumobj, (void *) coords.getRawPtr());
345 if (nWeights)
346 zz.Set_Obj_List_Fn(zobjlist, (void *) weights.getRawPtr());
347 else
348 zz.Set_Obj_List_Fn(zobjlist, (void *) coords.getRawPtr());
349 zz.Set_Num_Geom_Fn(znumgeom, (void *) coords.getRawPtr());
350 zz.Set_Geom_Multi_Fn(zgeom, (void *) coords.getRawPtr());
351 zz.Set_HG_Size_CS_Fn(zhgsize, (void *) &(*matrix));
352 zz.Set_HG_CS_Fn(zhg, (void *) &(*matrix));
353
354 int changes, ngid, nlid;
355 int numd, nump;
356 ZOLTAN_ID_PTR dgid = NULL, dlid = NULL, pgid = NULL, plid = NULL;
357 int *dproc = NULL, *dpart = NULL, *pproc = NULL, *ppart = NULL;
358
359 int ierr = zz.LB_Partition(changes, ngid, nlid,
360 numd, dgid, dlid, dproc, dpart,
361 nump, pgid, plid, pproc, ppart);
362 if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
363 if (me == 0)
364 std::cout << "Test " << testCnt << ": FAIL: direct Zoltan call" << std::endl;
365 zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
366 return 1;
367 }
368
369for(int i = 0; i < nump; i++) {
370 std::cout << me << " KDD Z1 " << pgid[i] << " " << plid[i] << " " << ppart[i] << std::endl;
371}
372
374 // PARTITION USING ZOLTAN THROUGH ZOLTAN2
376
377 if (me == 0) std::cout << "Calling Zoltan through Zoltan2" << std::endl;
378
379 matrixAdapter_t *ia;
380 try{
381 ia = new matrixAdapter_t(matrixConst, nWeights);
382 }
383 catch(std::exception &e){
384 if (me == 0)
385 std::cout << "Test " << testCnt << ": FAIL: matrix adapter "
386 << e.what() << std::endl;
387 return 1;
388 }
389 for (int idx=0; idx < nWeights; idx++)
390 ia->setRowWeights(weights->getData(idx).getRawPtr(), 1, idx);
391
392 vectorAdapter_t *ca = NULL;
393 try{
394 ca = new vectorAdapter_t(coords);
395 }
396 catch(std::exception &e){
397 if (me == 0)
398 std::cout << "Test " << testCnt << ": FAIL: vector adapter "
399 << e.what() << std::endl;
400 return 1;
401 }
402 ia->setCoordinateInput(ca);
403
404 Teuchos::ParameterList params;
405 params.set("timer_output_stream" , "std::cout");
406 // params.set("debug_level" , "verbose_detailed_status");
407
408 params.set("algorithm", "zoltan");
409 params.set("imbalance_tolerance", tolerance );
410 params.set("num_global_parts", numGlobalParts);
411
412 if (thisTest[TESTMETHODOFFSET] != "default") {
413 // "default" tests case of no Zoltan parameter sublist
414 Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
415 zparams.set("LB_METHOD",thisTest[TESTMETHODOFFSET]);
416 zparams.set("LB_APPROACH", "PARTITION");
417 zparams.set("SEED", "1111");
418 }
419
421# ifdef HAVE_ZOLTAN2_MPI
422 try{
423 problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, &params,
424 mpiComm);
425 }
426# else
427 try{
428 problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, &params);
429 }
430# endif
431 catch(std::exception &e){
432 std::cout << "Test " << testCnt << " FAIL: problem " << e.what() << std::endl;
433 return 1;
434 }
435
436 try {
437 problem->solve();
438 }
439 catch(std::exception &e){
440 std::cout << "Test " << testCnt << " FAIL: solve " << e.what() << std::endl;
441 return 1;
442 }
443
444for(int i = 0; i < nump; i++) {
445 std::cout << me << " KDD Z2 " << coords->getMap()->getGlobalElement(i) << " " << i << " " << problem->getSolution().getPartListView()[i] << std::endl;
446}
447
448 // create metric object
449
450 RCP<quality_t>metricObject = rcp(new quality_t(ia,&params,problem->getComm(),
451 &problem->getSolution()));
452 if (me == 0){
453 metricObject->printMetrics(std::cout);
454 }
455 problem->printTimers();
456
458 // COMPARE RESULTS
460 size_t nObj = coords->getLocalLength();
461 const int *z2parts = problem->getSolution().getPartListView();
462 int diffcnt = 0, gdiffcnt = 0;
463 for (size_t i = 0; i < nObj; i++) {
464 if (z2parts[plid[i]] != ppart[i]) {
465 diffcnt++;
466 std::cout << me << " DIFF for " << i << " ("
467 << coords->getMap()->getGlobalElement(i) << "): "
468 << "Z2 = " << z2parts[i] << "; Z1 = " << ppart[plid[i]] << std::endl;
469 }
470 }
471
473 // CLEAN UP
475 zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
476 delete ia;
477 delete ca;
478 delete problem;
479 delete uinput;
480
481 Teuchos::reduceAll(*comm, Teuchos::REDUCE_SUM, 1, &diffcnt, &gdiffcnt);
482 if (gdiffcnt > 0) {
483 if (me == 0)
484 std::cout << "Test " << testCnt << " "
485 << thisTest[TESTNAMEOFFSET] << " "
486 << thisTest[TESTMETHODOFFSET] << " "
487 << thisTest[TESTOBJWGTOFFSET] << " "
488 << " FAIL: comparison " << std::endl;
489 return 1;
490 }
491
492 return 0;
493}
494
496
497int main(int narg, char *arg[])
498{
499 Tpetra::ScopeGuard tscope(&narg, &arg);
500 Teuchos::RCP<const Teuchos::Comm<int> > comm = Tpetra::getDefaultComm();
501
502 int me = comm->getRank();
503 int np = comm->getSize();
504
505 int fail=0;
506
507 Array<int> ranks(np);
508 for (int i = 0; i < np; i++) ranks[i] = i;
509
510 for (int i=0; i < NUMTESTS; i++) {
511 std::string *thisTest = &(testArgs[i*TESTNUMARGS]);
512 int nTestProcs = atoi(thisTest[TESTNUMPROCS].c_str());
513 if (nTestProcs > np) {
514 if (me == 0) {
515 std::cout << "Skipping test " << i << " on "
516 << thisTest[TESTNAMEOFFSET]
517 << "; required number of procs " << nTestProcs
518 << " is greater than available procs " << np << std::endl;
519 }
520 continue;
521 }
522
523 // Make a communicator of appropriate size for the test
524 RCP<const Comm<int> > testcomm;
525 if (nTestProcs == np)
526 testcomm = comm;
527 else
528 testcomm = comm->createSubcommunicator(ranks.view(0,nTestProcs));
529
530 // Run the test if in the communicator
531 if (me < nTestProcs) {
532 fail += run(testcomm, nTestProcs, i, thisTest);
533 }
534 }
535
536 if (me == 0 && !fail)
537 std::cout << "PASS" << std::endl;
538
539 return 0;
540}
541
Defines the PartitioningProblem class.
Defines the PartitioningSolution class.
common code used by tests
std::string zoltanTestDirectory(".")
Tpetra::Map ::local_ordinal_type zlno_t
Tpetra::Map ::global_ordinal_type zgno_t
Defines the XpetraCrsMatrixAdapter class.
Defines the XpetraMultiVectorAdapter.
int main()
RCP< tMVector_t > getUICoordinates()
RCP< tcrsMatrix_t > getUITpetraCrsMatrix()
RCP< tMVector_t > getUIWeights()
A class that computes and returns quality metrics.
void setCoordinateInput(VectorAdapter< UserCoord > *coordData) override
Allow user to provide additional data that contains coordinate info associated with the MatrixAdapter...
PartitioningProblem sets up partitioning problems for the user.
const PartitioningSolution< Adapter > & getSolution()
Get the solution to the problem.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
RCP< const Comm< int > > getComm()
Return the communicator used by the problem.
void printTimers() const
Return the communicator passed to the problem.
Provides access for Zoltan2 to Xpetra::CrsMatrix data.
void setRowWeights(const scalar_t *weightVal, int stride, int idx=0)
Specify a weight for each row.
static const std::string fail
static ArrayRCP< ArrayRCP< zscalar_t > > weights
static void zobjlist(void *data, int ngid, int nlid, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int nwgts, float *wgts, int *ierr)
static int znumobj(void *data, int *ierr)
Tpetra::MultiVector< zscalar_t, zlno_t, zgno_t, znode_t > tMVector_t
Tpetra::CrsMatrix< zscalar_t, zlno_t, zgno_t, znode_t > tMatrix_t
#define NUMTESTS
static void zhgsize(void *data, int *nLists, int *nPins, int *format, int *ierr)
static int znumgeom(void *data, int *ierr)
Tpetra::CrsGraph< zlno_t, zgno_t, znode_t > tGraph_t
Zoltan2::EvaluatePartition< matrixAdapter_t > quality_t
testFields
@ TESTNUMARGS
@ TESTNAMEOFFSET
@ TESTMETHODOFFSET
@ TESTOBJWGTOFFSET
@ TESTNUMPROCS
Zoltan2::XpetraCrsMatrixAdapter< tMatrix_t, tMVector_t > matrixAdapter_t
static string testArgs[]
static void zgeom(void *data, int ngid, int nlid, int nobj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int ndim, double *coords, int *ierr)
static constexpr int znGidEnt
static void zhg(void *data, int ngid, int nLists, int nPins, int format, ZOLTAN_ID_PTR listGids, int *offsets, ZOLTAN_ID_PTR pinGids, int *ierr)
int run(const RCP< const Comm< int > > &comm, int numGlobalParts, int testCnt, std::string *thisTest)
#define SET_ZID(n, a, b)
Zoltan2::XpetraMultiVectorAdapter< tMVector_t > vectorAdapter_t