46#ifndef MUELU_UNCOUPLEDAGGREGATIONFACTORY_KOKKOS_DEF_HPP_
47#define MUELU_UNCOUPLEDAGGREGATIONFACTORY_KOKKOS_DEF_HPP_
51#include <Xpetra_Map.hpp>
52#include <Xpetra_Vector.hpp>
53#include <Xpetra_MultiVectorFactory.hpp>
54#include <Xpetra_VectorFactory.hpp>
58#include "MueLu_OnePtAggregationAlgorithm_kokkos.hpp"
59#include "MueLu_PreserveDirichletAggregationAlgorithm_kokkos.hpp"
60#include "MueLu_IsolatedNodeAggregationAlgorithm_kokkos.hpp"
62#include "MueLu_AggregationPhase1Algorithm_kokkos.hpp"
63#include "MueLu_AggregationPhase2aAlgorithm_kokkos.hpp"
64#include "MueLu_AggregationPhase2bAlgorithm_kokkos.hpp"
65#include "MueLu_AggregationPhase3Algorithm_kokkos.hpp"
68#include "MueLu_LWGraph_kokkos.hpp"
69#include "MueLu_Aggregates_kokkos.hpp"
72#include "MueLu_AmalgamationInfo.hpp"
73#include "MueLu_Utilities.hpp"
75#include "KokkosGraph_Distance2ColorHandle.hpp"
76#include "KokkosGraph_Distance2Color.hpp"
77#include "KokkosGraph_MIS2.hpp"
81 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
83 : bDefinitionPhase_(true)
86 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
88 RCP<ParameterList> validParamList = rcp(
new ParameterList());
93 typedef Teuchos::StringToIntegralParameterEntryValidator<int> validatorType;
94#define SET_VALID_ENTRY(name) validParamList->setEntry(name, MasterList::getEntry(name))
99 validParamList->getEntry(
"aggregation: ordering").setValidator(
100 rcp(
new validatorType(Teuchos::tuple<std::string>(
"natural",
"graph",
"random"),
"aggregation: ordering")));
109 SET_VALID_ENTRY(
"aggregation: error on nodes with no on-rank neighbors");
113#undef SET_VALID_ENTRY
116 validParamList->set< RCP<const FactoryBase> >(
"Graph", null,
"Generating factory of the graph");
117 validParamList->set< RCP<const FactoryBase> >(
"DofsPerNode", null,
"Generating factory for variable \'DofsPerNode\', usually the same as for \'Graph\'");
120 validParamList->set< std::string > (
"OnePt aggregate map name",
"",
"Name of input map for single node aggregates. (default='')");
121 validParamList->set< std::string > (
"OnePt aggregate map factory",
"",
"Generating factory of (DOF) map for single node aggregates.");
124 return validParamList;
127 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
129 Input(currentLevel,
"Graph");
130 Input(currentLevel,
"DofsPerNode");
132 const ParameterList& pL = GetParameterList();
135 std::string mapOnePtName = pL.get<std::string>(
"OnePt aggregate map name");
136 if (mapOnePtName.length() > 0) {
137 std::string mapOnePtFactName = pL.get<std::string>(
"OnePt aggregate map factory");
138 if (mapOnePtFactName ==
"" || mapOnePtFactName ==
"NoFactory") {
141 RCP<const FactoryBase> mapOnePtFact = GetFactory(mapOnePtFactName);
142 currentLevel.
DeclareInput(mapOnePtName, mapOnePtFact.get());
147 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
150 using execution_space =
typename LWGraph_kokkos::execution_space;
151 using memory_space =
typename LWGraph_kokkos::memory_space;
152 using local_ordinal_type =
typename LWGraph_kokkos::local_ordinal_type;
155 ParameterList pL = GetParameterList();
156 bDefinitionPhase_ =
false;
158 if (pL.get<
int>(
"aggregation: max agg size") == -1)
159 pL.set(
"aggregation: max agg size", INT_MAX);
162 RCP<const FactoryBase> graphFact = GetFactory(
"Graph");
173 std::string mapOnePtName = pL.get<std::string>(
"OnePt aggregate map name");
174 RCP<Map> OnePtMap = Teuchos::null;
175 if (mapOnePtName.length()) {
176 std::string mapOnePtFactName = pL.get<std::string>(
"OnePt aggregate map factory");
177 if (mapOnePtFactName ==
"" || mapOnePtFactName ==
"NoFactory") {
180 RCP<const FactoryBase> mapOnePtFact = GetFactory(mapOnePtFactName);
181 OnePtMap = currentLevel.
Get<RCP<Map> >(mapOnePtName, mapOnePtFact.get());
185 RCP<const LWGraph_kokkos> graph = Get< RCP<LWGraph_kokkos> >(currentLevel,
"Graph");
189 aggregates->setObjectLabel(
"UC");
191 const LO numRows = graph->GetNodeNumVertices();
194 Kokkos::View<unsigned*, typename LWGraph_kokkos::device_type> aggStat(Kokkos::ViewAllocateWithoutInitializing(
"aggregation status"),
196 Kokkos::deep_copy(aggStat,
READY);
207 typename LWGraph_kokkos::boundary_nodes_type dirichletBoundaryMap = graph->getLocalLWGraph().GetBoundaryNodeMap();
208 Kokkos::parallel_for(
"MueLu - UncoupledAggregation: tagging boundary nodes in aggStat",
209 Kokkos::RangePolicy<local_ordinal_type, execution_space>(0, numRows),
210 KOKKOS_LAMBDA(
const local_ordinal_type nodeIdx) {
211 if (dirichletBoundaryMap(nodeIdx) ==
true) {
217 LO nDofsPerNode = Get<LO>(currentLevel,
"DofsPerNode");
218 GO indexBase = graph->GetDomainMap()->getIndexBase();
219 if (OnePtMap != Teuchos::null) {
220 typename Kokkos::View<unsigned*,typename LWGraph_kokkos::device_type>::HostMirror aggStatHost
221 = Kokkos::create_mirror_view(aggStat);
222 Kokkos::deep_copy(aggStatHost, aggStat);
224 for (LO i = 0; i < numRows; i++) {
226 GO grid = (graph->GetDomainMap()->getGlobalElement(i)-indexBase) * nDofsPerNode + indexBase;
228 for (LO kr = 0; kr < nDofsPerNode; kr++)
229 if (OnePtMap->isNodeGlobalElement(grid + kr))
230 aggStatHost(i) =
ONEPT;
233 Kokkos::deep_copy(aggStat, aggStatHost);
236 const RCP<const Teuchos::Comm<int> > comm = graph->GetComm();
237 GO numGlobalRows = 0;
241 LO numNonAggregatedNodes = numRows;
242 std::string aggAlgo = pL.get<std::string>(
"aggregation: coloring algorithm");
243 if(aggAlgo ==
"mis2 coarsening" || aggAlgo ==
"mis2 aggregation")
246 using graph_t =
typename LWGraph_kokkos::local_graph_type;
247 using device_t =
typename graph_t::device_type;
248 using exec_space =
typename device_t::execution_space;
249 using rowmap_t =
typename graph_t::row_map_type;
250 using colinds_t =
typename graph_t::entries_type;
251 using lno_t =
typename colinds_t::non_const_value_type;
252 rowmap_t aRowptrs = graph->getLocalLWGraph().getRowPtrs();
253 colinds_t aColinds = graph->getLocalLWGraph().getEntries();
255 typename colinds_t::non_const_type labels;
257 if(aggAlgo ==
"mis2 coarsening")
260 labels = KokkosGraph::graph_mis2_coarsen<device_t, rowmap_t, colinds_t>(aRowptrs, aColinds, numAggs);
262 else if(aggAlgo ==
"mis2 aggregation")
265 labels = KokkosGraph::graph_mis2_aggregate<device_t, rowmap_t, colinds_t>(aRowptrs, aColinds, numAggs);
267 auto vertex2AggId = aggregates->GetVertex2AggId()->getDeviceLocalView(Xpetra::Access::ReadWrite);
268 auto procWinner = aggregates->GetProcWinner() ->getDeviceLocalView(Xpetra::Access::OverwriteAll);
269 int rank = comm->getRank();
270 Kokkos::parallel_for(Kokkos::RangePolicy<exec_space>(0, numRows),
271 KOKKOS_LAMBDA(lno_t i)
273 procWinner(i, 0) = rank;
274 if(aggStat(i) ==
READY)
277 vertex2AggId(i, 0) = labels(i);
280 numNonAggregatedNodes = 0;
281 aggregates->SetNumAggregates(numAggs);
292 using graph_t =
typename LWGraph_kokkos::local_graph_type;
293 using KernelHandle = KokkosKernels::Experimental::
294 KokkosKernelsHandle<
typename graph_t::row_map_type::value_type,
295 typename graph_t::entries_type::value_type,
296 typename graph_t::entries_type::value_type,
297 typename graph_t::device_type::execution_space,
298 typename graph_t::device_type::memory_space,
299 typename graph_t::device_type::memory_space>;
302 kh.create_distance2_graph_coloring_handle();
305 auto coloringHandle = kh.get_distance2_graph_coloring_handle();
315 if(pL.get<
bool>(
"aggregation: deterministic") ==
true) {
316 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_SERIAL );
318 }
else if(aggAlgo ==
"serial") {
319 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_SERIAL );
321 }
else if(aggAlgo ==
"default") {
322 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_DEFAULT );
324 }
else if(aggAlgo ==
"vertex based") {
325 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_VB );
327 }
else if(aggAlgo ==
"vertex based bit set") {
328 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_VB_BIT );
330 }
else if(aggAlgo ==
"edge filtering") {
331 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_VB_BIT_EF );
333 }
else if(aggAlgo ==
"net based bit set") {
334 coloringHandle->set_algorithm( KokkosGraph::COLORING_D2_NB_BIT );
337 TEUCHOS_TEST_FOR_EXCEPTION(
true,std::invalid_argument,
"Unrecognized distance 2 coloring algorithm, valid options are: serial, default, matrix squared, vertex based, vertex based bit set, edge filtering")
341 typename graph_t::row_map_type aRowptrs = graph->getLocalLWGraph().getRowPtrs();
342 typename graph_t::entries_type aColinds = graph->getLocalLWGraph().getEntries();
346 KokkosGraph::Experimental::graph_color_distance2(&kh, numRows, aRowptrs, aColinds);
349 aggregates->SetGraphColors(coloringHandle->get_vertex_colors());
350 aggregates->SetGraphNumColors(
static_cast<LO
>(coloringHandle->get_num_colors()));
353 kh.destroy_distance2_graph_coloring_handle();
356 GetOStream(
Statistics1) <<
" num colors: " << aggregates->GetGraphNumColors() << std::endl;
358 GO numGlobalAggregatedPrev = 0, numGlobalAggsPrev = 0;
359 for (
size_t a = 0; a < algos_.size(); a++) {
360 std::string phase = algos_[a]->description();
363 int oldRank = algos_[a]->SetProcRankVerbose(this->GetProcRankVerbose());
364 algos_[a]->BuildAggregates(pL, *graph, *aggregates, aggStat, numNonAggregatedNodes);
365 algos_[a]->SetProcRankVerbose(oldRank);
368 GO numLocalAggregated = numRows - numNonAggregatedNodes, numGlobalAggregated = 0;
369 GO numLocalAggs = aggregates->GetNumAggregates(), numGlobalAggs = 0;
370 MueLu_sumAll(comm, numLocalAggregated, numGlobalAggregated);
373 double aggPercent = 100*as<double>(numGlobalAggregated)/as<double>(numGlobalRows);
374 if (aggPercent > 99.99 && aggPercent < 100.00) {
381 GetOStream(
Statistics1) <<
" aggregated : " << (numGlobalAggregated - numGlobalAggregatedPrev) <<
" (phase), " << std::fixed
382 << std::setprecision(2) << numGlobalAggregated <<
"/" << numGlobalRows <<
" [" << aggPercent <<
"%] (total)\n"
383 <<
" remaining : " << numGlobalRows - numGlobalAggregated <<
"\n"
384 <<
" aggregates : " << numGlobalAggs-numGlobalAggsPrev <<
" (phase), " << numGlobalAggs <<
" (total)" << std::endl;
385 numGlobalAggregatedPrev = numGlobalAggregated;
386 numGlobalAggsPrev = numGlobalAggs;
391 TEUCHOS_TEST_FOR_EXCEPTION(numNonAggregatedNodes,
Exceptions::RuntimeError,
"MueLu::UncoupledAggregationFactory::Build: Leftover nodes found! Error!");
393 aggregates->AggregatesCrossProcessors(
false);
394 aggregates->ComputeAggregateSizes(
true);
396 Set(currentLevel,
"Aggregates", aggregates);
#define SET_VALID_ENTRY(name)
#define MueLu_sumAll(rcpComm, in, out)
Exception throws to report errors in the internal logical of the program.
Timer to be used in factories. Similar to Monitor but with additional timers.
Class that holds all level-specific information.
void DeclareInput(const std::string &ename, const FactoryBase *factory, const FactoryBase *requestedBy=NoFactory::get())
Callback from FactoryBase::CallDeclareInput() and FactoryBase::DeclareInput()
T & Get(const std::string &ename, const FactoryBase *factory=NoFactory::get())
Get data without decrementing associated storage counter (i.e., read-only access)....
static const NoFactory * get()
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.
void DeclareInput(Level ¤tLevel) const
Input.
void Build(Level ¤tLevel) const
Build aggregates.
UncoupledAggregationFactory_kokkos()
Constructor.
Namespace for MueLu classes and methods.
@ Statistics1
Print more statistics.