63 #include "Tpetra_DefaultPlatform.hpp" 66 #include "Teuchos_RCP.hpp" 67 #include "Teuchos_GlobalMPISession.hpp" 68 #include "Teuchos_XMLParameterListHelpers.hpp" 69 #include "Teuchos_Hashtable.hpp" 72 #ifdef HAVE_ZOLTAN2_PARMA 85 using Teuchos::ParameterList;
87 using Teuchos::ArrayView;
93 typedef Tpetra::DefaultPlatform::DefaultPlatformType
Platform;
97 template <
typename Adapter>
99 typedef typename Adapter::gno_t gno_t;
100 typedef typename Adapter::lno_t lno_t;
101 typedef typename Adapter::scalar_t scalar_t;
104 ArrayView<const gno_t> Ids;
105 ArrayView<input_t> wgts;
107 ArrayView<bool> isOwner;
110 ArrayView<const gno_t> pins;
111 ArrayView<const lno_t> offsets;
114 std::set<gno_t> gids;
119 std::set<gno_t> ghosts;
123 if (gids.find(pin)==gids.end()) {
125 if (ghosts.find(pin)==ghosts.end())
129 std::cout<<
"[METRIC] " << PCU_Comm_Self() <<
" Total number of ghosts in the hypergraph: " << num_ghosts <<
"\n" 130 <<
"[METRIC] " << PCU_Comm_Self() <<
" Number of unique ghosts: " << ghosts.size() <<
"\n";
131 gno_t unique_ghosts =ghosts.size();
132 gno_t owned_and_ghosts =unique_ghosts+numOwned;
133 gno_t max_o_and_g,min_o_and_g;
134 gno_t max_ghosts,max_u_ghosts;
135 gno_t min_ghosts,min_u_ghosts;
136 max_ghosts = min_ghosts = num_ghosts;
137 max_u_ghosts = min_u_ghosts = unique_ghosts;
138 max_o_and_g = min_o_and_g = owned_and_ghosts;
139 double avg_ghosts,avg_u_ghosts,avg_o_and_g;
140 PCU_Add_Ints(&num_ghosts,1);
141 PCU_Add_Ints(&unique_ghosts,1);
142 PCU_Add_Ints(&owned_and_ghosts,1);
143 PCU_Max_Ints(&max_ghosts,1);
144 PCU_Max_Ints(&max_u_ghosts,1);
145 PCU_Max_Ints(&max_o_and_g,1);
146 PCU_Min_Ints(&min_ghosts,1);
147 PCU_Min_Ints(&min_u_ghosts,1);
148 PCU_Min_Ints(&min_o_and_g,1);
149 avg_ghosts = num_ghosts*1.0/PCU_Comm_Peers();
150 avg_u_ghosts = unique_ghosts*1.0/PCU_Comm_Peers();
151 avg_o_and_g = owned_and_ghosts*1.0/PCU_Comm_Peers();
152 if (!PCU_Comm_Self())
153 std::cout<<
"[METRIC] Global ghosts in the hypergraph (tot max min avg imb): " 154 << num_ghosts<<
" "<<max_ghosts<<
" "<<min_ghosts<<
" "<<avg_ghosts<<
" " 155 <<max_ghosts/avg_ghosts <<
"\n" 156 <<
"[METRIC] Global unique ghosts (tot max min avg imb): " 157 << unique_ghosts<<
" "<<max_u_ghosts<<
" "<<min_u_ghosts<<
" "<<avg_u_ghosts<<
" " 158 <<max_u_ghosts/avg_u_ghosts <<
"\n" 159 <<
"[METRIC] Global owned and ghosts (tot max min avg imb): " 160 << owned_and_ghosts<<
" "<<max_o_and_g<<
" "<<min_o_and_g<<
" "<<avg_o_and_g<<
" " 161 <<max_o_and_g/avg_o_and_g <<
"\n";
169 int main(
int narg,
char *arg[]) {
171 Teuchos::GlobalMPISession mpiSession(&narg, &arg,0);
172 Platform &platform = Tpetra::DefaultPlatform::getDefaultPlatform();
173 RCP<const Teuchos::Comm<int> > CommT = platform.getComm();
175 int me = CommT->getRank();
180 <<
"====================================================================\n" 182 <<
"| Example: Partition APF Mesh |\n" 184 <<
"| Questions? Contact Karen Devine (kddevin@sandia.gov), |\n" 185 <<
"| Erik Boman (egboman@sandia.gov), |\n" 186 <<
"| Siva Rajamanickam (srajama@sandia.gov). |\n" 188 <<
"| Zoltan2's website: http://trilinos.sandia.gov/packages/zoltan2 |\n" 189 <<
"| Trilinos website: http://trilinos.sandia.gov |\n" 191 <<
"====================================================================\n";
197 cout <<
"PARALLEL executable \n";
201 cout <<
"SERIAL executable \n";
210 std::string meshFileName(
"4/");
211 std::string modelFileName(
"torus.dmg");
212 std::string action(
"parma");
213 std::string parma_method(
"VtxElm");
214 std::string output_loc(
"");
215 int nParts = CommT->getSize();
216 double imbalance = 1.1;
218 int ghost_metric=
false;
220 Teuchos::CommandLineProcessor cmdp (
false,
false);
221 cmdp.setOption(
"meshfile", &meshFileName,
222 "Mesh file with APF specifications (.smb file(s))");
223 cmdp.setOption(
"modelfile", &modelFileName,
224 "Model file with APF specifications (.dmg file)");
225 cmdp.setOption(
"action", &action,
226 "Method to use: mj, scotch, zoltan_rcb, parma or color");
227 cmdp.setOption(
"parma_method", &parma_method,
228 "Method to use: Vertex, Element, VtxElm, VtxEdgeElm, Ghost, Shape, or Centroid ");
229 cmdp.setOption(
"nparts", &
nParts,
230 "Number of parts to create");
231 cmdp.setOption(
"imbalance", &imbalance,
232 "Target imbalance for the partitioning method");
233 cmdp.setOption(
"output", &output_loc,
234 "Location of new partitioned apf mesh. Ex: 4/torus.smb");
235 cmdp.setOption(
"layers", &layers,
236 "Number of layers for ghosting");
237 cmdp.setOption(
"ghost_metric", &ghost_metric,
238 "0 does not compute ghost metric otherwise compute both before and after");
239 cmdp.parse(narg, arg);
252 #ifdef HAVE_ZOLTAN2_PARMA 254 if (me == 0) cout <<
"Generating mesh ... \n\n";
261 apf::Mesh2* m = apf::loadMdsMesh(modelFileName.c_str(),meshFileName.c_str());
265 std::string primary=
"region";
266 std::string adjacency=
"face";
267 if (m->getDimension()==2) {
271 bool needSecondAdj=
false;
274 if (me == 0) cout <<
"Creating parameter list ... \n\n";
276 Teuchos::ParameterList params(
"test params");
277 params.set(
"timer_output_stream" ,
"std::cout");
279 bool do_partitioning =
false;
280 if (action ==
"mj") {
281 do_partitioning =
true;
282 params.set(
"debug_level",
"basic_status");
283 params.set(
"imbalance_tolerance", imbalance);
284 params.set(
"num_global_parts",
nParts);
285 params.set(
"algorithm",
"multijagged");
286 params.set(
"rectilinear",
"yes");
288 else if (action ==
"scotch") {
289 do_partitioning =
true;
290 params.set(
"debug_level",
"no_status");
291 params.set(
"imbalance_tolerance", imbalance);
292 params.set(
"num_global_parts",
nParts);
293 params.set(
"partitioning_approach",
"partition");
294 params.set(
"objects_to_partition",
"mesh_elements");
295 params.set(
"algorithm",
"scotch");
298 else if (action ==
"zoltan_rcb") {
299 do_partitioning =
true;
300 params.set(
"debug_level",
"verbose_detailed_status");
301 params.set(
"imbalance_tolerance", imbalance);
302 params.set(
"num_global_parts",
nParts);
303 params.set(
"partitioning_approach",
"partition");
304 params.set(
"algorithm",
"zoltan");
306 else if (action ==
"parma") {
307 do_partitioning =
true;
308 params.set(
"debug_level",
"no_status");
309 params.set(
"imbalance_tolerance", imbalance);
310 params.set(
"algorithm",
"parma");
311 Teuchos::ParameterList &pparams = params.sublist(
"parma_parameters",
false);
312 pparams.set(
"parma_method",parma_method);
313 pparams.set(
"step_size",1.1);
314 if (parma_method==
"Ghost") {
315 pparams.set(
"ghost_layers",layers);
316 pparams.set(
"ghost_bridge",m->getDimension()-1);
320 else if (action==
"zoltan_hg") {
321 do_partitioning =
true;
322 params.set(
"debug_level",
"no_status");
323 params.set(
"imbalance_tolerance", imbalance);
324 params.set(
"algorithm",
"zoltan");
325 params.set(
"num_global_parts",
nParts);
326 Teuchos::ParameterList &zparams = params.sublist(
"zoltan_parameters",
false);
327 zparams.set(
"LB_METHOD",
"HYPERGRAPH");
328 zparams.set(
"LB_APPROACH",
"PARTITION");
332 else if (action==
"hg_ghost") {
333 do_partitioning =
true;
334 params.set(
"debug_level",
"no_status");
335 params.set(
"imbalance_tolerance", imbalance);
336 params.set(
"algorithm",
"zoltan");
337 params.set(
"num_global_parts",
nParts);
338 params.set(
"hypergraph_model_type",
"ghosting");
339 params.set(
"ghost_layers",layers);
340 Teuchos::ParameterList &zparams = params.sublist(
"zoltan_parameters",
false);
341 zparams.set(
"LB_METHOD",
"HYPERGRAPH");
342 zparams.set(
"LB_APPROACH",
"PARTITION");
343 zparams.set(
"PHG_EDGE_SIZE_THRESHOLD",
"1.0");
348 else if (action ==
"color") {
349 params.set(
"debug_level",
"verbose_detailed_status");
350 params.set(
"debug_output_file",
"kdd");
351 params.set(
"debug_procs",
"all");
353 Parma_PrintPtnStats(m,
"before");
356 if (me == 0) cout <<
"Creating mesh adapter ... \n\n";
361 double time_1=PCU_Time();
363 new inputAdapter_t(*CommT, m,primary,adjacency,needSecondAdj);
364 double time_2=PCU_Time();
367 inputAdapter_t::scalar_t* arr =
368 new inputAdapter_t::scalar_t[ia->getLocalNumOf(ia->getPrimaryEntityType())];
369 for (
size_t i=0;i<ia->getLocalNumOf(ia->getPrimaryEntityType());i++) {
370 arr[i]=PCU_Comm_Self()+1;
373 const inputAdapter_t::scalar_t*
weights=arr;
374 ia->setWeights(ia->getPrimaryEntityType(),
weights,1);
378 const baseMeshAdapter_t *base_ia =
dynamic_cast<const baseMeshAdapter_t*
>(ia);
380 RCP<Zoltan2::Environment> env;
386 RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<
const Zoltan2::Environment>(env);
388 RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,
false);
395 double time_3 = PCU_Time();
396 if (do_partitioning) {
397 if (me == 0) cout <<
"Creating partitioning problem ... \n\n";
402 if (me == 0) cout <<
"Calling the partitioner ... \n\n";
408 if (me==0) cout <<
"Applying Solution to Mesh\n\n";
409 apf::Mesh2** new_mesh = &m;
410 ia->applyPartitioningSolution(m,new_mesh,problem.
getSolution());
413 RCP<quality_t> metricObject =
414 rcp(
new quality_t(ia, ¶ms, CommT, &problem.
getSolution()));
417 metricObject->printMetrics(cout);
421 if (me == 0) cout <<
"Creating coloring problem ... \n\n";
426 if (me == 0) cout <<
"Calling the coloring algorithm ... \n\n";
435 double time_4=PCU_Time();
443 inputAdapter_t ia2(*CommT, m,primary,adjacency,
true);
444 const baseMeshAdapter_t *base_ia =
dynamic_cast<const baseMeshAdapter_t*
>(&ia2);
447 RCP<Zoltan2::Environment> env;
452 RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<
const Zoltan2::Environment>(env);
453 RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,
false);
461 if (output_loc!=
"") {
462 m->writeNative(output_loc.c_str());
466 if (me == 0) cout <<
"Deleting the mesh ... \n\n";
469 PCU_Max_Doubles(&time_2,1);
470 PCU_Max_Doubles(&time_4,1);
472 std::cout<<
"\nConstruction time: "<<time_2<<
"\n" 473 <<
"Problem time: " << time_4<<
"\n\n";
483 std::cout <<
"PASS" << std::endl;
size_t getEdgeList(ArrayView< const gno_t > &Ids, ArrayView< input_t > &wgts) const
Sets pointers to this process' hyperedge Ids and their weights.
ColoringProblem sets up coloring problems for the user.
Defines the ColoringProblem class.
size_t getLocalNumVertices() const
Returns the number vertices on this process.
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
MeshAdapter defines the interface for mesh input.
std::bitset< NUM_MODEL_FLAGS > modelFlag_t
size_t getLocalNumOwnedVertices() const
Returns the number vertices on this process that are owned.
Defines the APFMeshAdapter class.
void PrintGhostMetrics(Zoltan2::HyperGraphModel< Adapter > &mdl)
size_t getLocalNumPins() const
Returns the local number of pins.
size_t getOwnedList(ArrayView< bool > &isOwner) const
Sets pointer to the ownership of this processes vertices.
The StridedData class manages lists of weights or coordinates.
Tpetra::DefaultPlatform::DefaultPlatformType Platform
int main(int narg, char *arg[])
The user parameters, debug, timing and memory profiling output objects, and error checking methods...
void printTimers() const
Return the communicator passed to the problem.
const PartitioningSolution< Adapter > & getSolution()
Get the solution to the problem.
PartitioningProblem sets up partitioning problems for the user.
size_t getPinList(ArrayView< const gno_t > &pinIds, ArrayView< const lno_t > &offsets, ArrayView< input_t > &wgts) const
Sets pointers to this process' pins global Ids based on the centric view given by getCentricView() ...
Defines the HyperGraphModel interface.
Defines the Environment class.
Defines the PartitioningProblem class.
A class that computes and returns quality metrics.
HyperGraphModel defines the interface required for hyper graph models.
void solve(bool updateInputData=true)
Direct the problem to create a solution.