Zoltan2
Zoltan2_ComparisonHelper.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // Zoltan2: A package of combinatorial algorithms for scientific computing
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Karen Devine (kddevin@sandia.gov)
39 // Erik Boman (egboman@sandia.gov)
40 // Siva Rajamanickam (srajama@sandia.gov)
41 //
42 // ***********************************************************************
43 //
44 // @HEADER
45 
50 #pragma once
51 
52 #include "Zoltan2_TestHelpers.hpp"
54 #include <Zoltan2_Typedefs.hpp>
55 #include <AdapterForTests.hpp>
59 
60 #include <Teuchos_DefaultComm.hpp>
61 #include <Teuchos_Time.hpp>
62 
63 #include <sstream>
64 #include <string>
65 #include <map>
66 #include <iostream>
67 
68 using Teuchos::Comm;
69 using Teuchos::RCP;
70 using Teuchos::ParameterList;
71 using Teuchos::Time;
72 using std::cout;
73 using std::endl;
74 using std::string;
75 using std::map;
76 using std::pair;
77 using std::ostringstream;
78 using Teuchos::reduceAll;
79 using namespace Zoltan2_TestingFramework;
80 
84 {
85 
86 public:
87  /* \brief Add a timer by name to the comparison sources timers map.
88  * \param name is the name of the timer to be defined
89  */
90  void addTimer(const std::string &name)
91  {
92  timers.insert(std::pair<const std::string &, RCP<Time> >(name,rcp(new Time(name))));
93  timers[name]->enable();
94  }
95 
96  RCP<Zoltan2::EvaluatePartition<basic_id_t> > metricObject;
97  RCP<base_problem_t> problem;
98  RCP<basic_id_t> adapter;
99  RCP<Zoltan2::VectorAdapter<tMVector_t> > coordinateAdapterRCP;
100  string problem_kind;
101  string adapter_kind;
102  std::map<const std::string, RCP<Time> > timers;
103 };
104 
108 {
109 
110 public:
111 
112  /* \brief Compare the solutions, metrics or timers of two Zoltan2 solutions.
113  * \param pList is a parameter list defining the comparison
114  * \param comm is the process communicator
115  */
116  bool Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm);
117 
118  /* \brief Add a new source by name to the comparison source map.
119  * \param name is the name of the new source
120  * \param source a problem source that to be used for comparison to another source
121  */
122  void AddSource(const string &name, ComparisonSource * source);
123 
124  /* \brief Return the total number of saved sources.
125  */
126  size_t getNumberOfSources() const
127  {
128  return this->sources.size();
129  }
130 
131 private:
132  map<const string,RCP<const ComparisonSource> > sources;
133 
134 
135  /* \brief Method called to compare two solutions
136  * \param p1 is the name of problem 1
137  * \param p2 is the name of problem 2
138  * \param comm is the process communicator
139  */
140  bool CompareSolutions(const string &p1,
141  const string &p2,
142  const RCP<const Comm<int> > &comm);
143 
144  /* \brief Method called to compare two paritioning solutions
145  * \param sourceA is a ptr to problem A's comparison source
146  * \param sourceB is a ptr to problem B's comparison source
147  * \param comm is the process communicator
148  */
149  bool ComparePartitionSolutions(const ComparisonSource * sourceA,
150  const ComparisonSource * sourceB,
151  const RCP<const Comm<int> > &comm);
152 
153  /* \brief Method called to compare two coloring solutions
154  * \param sourceA is a ptr to problem A's comparison source
155  * \param sourceB is a ptr to problem B's comparison source
156  * \param comm is the process communicator
157  */
158  bool CompareColoringSolutions(const ComparisonSource * sourceA,
159  const ComparisonSource * sourceB,
160  const RCP<const Comm<int> > &comm);
161 
162  /* \brief Method called to compare two ordering solutions
163  * \param sourceA is a ptr to problem A's comparison source
164  * \param sourceB is a ptr to problem B's comparison source
165  * \param comm is the process communicator
166  */
167  bool CompareOrderingSolutions(const ComparisonSource * sourceA,
168  const ComparisonSource * sourceB,
169  const RCP<const Comm<int> > &comm);
170 
171  /* \brief Method called to compare the metrics/timers of two problems.
172  * \param metricsPlist is a parameter list defining the comparison
173  * \param comm is the process communicator
174  */
175  bool CompareMetrics(const ParameterList &metricsPlist, const RCP<const Comm<int> > &comm);
176 
177  /* \brief Method that compares two metrics and returns a pass/fail message.
178  * \param[in] comm is the process communicator
179  * \param[in] metric is the metric to be compared to a reference metric
180  * \param[in] ref_metric is the reference metric for comparison
181  * \param[in] metricPlist is the parameter list defining the metric tolerances
182  * \param[out] msg is a returned pass/fail message
183  *
184  * \return boolean value indicated pass/fail status
185  */
186  static bool
187  metricComparisonTest(const RCP<const Comm<int> > &comm,
188  const MetricAnalyzerInfo & metric,
189  const MetricAnalyzerInfo &ref_metric,
190  const Teuchos::ParameterList & metricPlist,
191  ostringstream &msg);
192 
193  /* \brief Method that compares two timers and returns a pass/fail message.
194  * \param[in] comm is the process communicator
195  * \param[in] time is the timer data to be compared to a reference metric
196  * \param[in] ref_time is the reference timer for comparison
197  * \param[in] metricPlist is the parameter list defining the timer tolerances
198  * \param[out] msg is a returned pass/fail message
199  *
200  * \return boolean value indicated pass/fail status
201  */
202  static bool
203  timerComparisonTest(const RCP<const Comm<int> > &comm,
204  const double time,
205  const double ref_time,
206  const Teuchos::ParameterList & metricPlist,
207  ostringstream &msg);
208 
209  /* \brief Method for inserting data from all timers to a map of clocked times
210  * param[in] timers a map of timers
211  *
212  * \return a map with clocked times from timers
213  */
214  static std::map<const string, const double>
215  timerDataToMap(const map<const std::string, RCP<Time> > &timers);
216 
217 
218  /* \brief Method for extracting all methods to compare from a parameter list
219  * param[in] plist a parameter list defining 1 or more metric/timer comparisons
220  *
221  * \return a queue of metric comparison definitions
222  */
223  static std::queue<ParameterList>
224  getMetricsToCompare(const ParameterList & pList);
225 
226  static void
227  reduceWithMessage(const RCP<const Comm<int> > &comm,const std::string &msg_in,
228  int &local_status, std::ostringstream &msg);
229 
230 };
231 
232 
233 void ComparisonHelper::AddSource(const string &name, ComparisonSource * source)
234 {
235  typedef std::pair<const string &, RCP<const ComparisonSource> > pair_t;
236  this->sources.insert(pair_t(name, RCP<ComparisonSource>(source)));
237 }
238 
239 bool ComparisonHelper::Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm)
240 {
241  if(pList.isParameter("A") && pList.isParameter("B")) {
242  // comparing solutions
243  string pA = pList.get<string>("A");
244  if(this->sources.find(pA) == this->sources.end())
245  {
246  cout << "\nProblem: " + pA + ", was not saved for comparison.";
247  cout << "\nThis typically indicates that an error occurred while running the problem.";
248  cout << "\nSolution comparison FAILED." << endl;
249  return false;
250  }
251 
252  string pB = pList.get<string>("B");
253  if(this->sources.find(pB) == this->sources.end()) {
254  cout << "\nProblem: " + pB + ", was not saved for comparison.";
255  cout << "\nThis typically indicates that an error occurred while running the problem.";
256  cout << "\nSolution comparison FAILED." << endl;
257  return false;
258  }
259 
260  bool bResult = this->CompareSolutions(pA, pB, comm);
261  return bResult;
262  }
263  else if (pList.isParameter("Problem") && pList.isParameter("Reference")) {
264  // comparing metrics/timers
265  string prb = pList.get<string>("Problem");
266  if(this->sources.find(prb) == this->sources.end()) {
267  cout << "\nProblem: " + prb + ", was not saved for comparison.";
268  cout << "\nThis typically indicates that an error occurred while running the problem.";
269  cout << "\nMetric comparison FAILED." << endl;
270  return false;
271  }
272 
273  string ref = pList.get<string>("Reference");
274  if(this->sources.find(ref) == this->sources.end()) {
275  cout << "\nReference: " + ref + ", was not saved for comparison.";
276  cout << "\nThis typically indicates that an error occurred while running the problem.";
277  cout << "\nMetric comparison FAILED." << endl;
278  return false;
279  }
280 
281  bool bResult = this->CompareMetrics(pList, comm);
282  return bResult;
283  }
284  else if (pList.isParameter("A") || pList.isParameter("B"))
285  {
286  if(comm->getRank() == 0)
287  {
288  cout << "Problem A or Problem B is not specified -- check input.";
289  cout <<"\nSolution comparison FAILED." << endl;
290  }
291  }
292  else if (pList.isParameter("Problem") || pList.isParameter("Reference")) {
293  if(comm->getRank() == 0) {
294  cout << "Problem or reference is not specified -- check input.";
295  cout <<"\nMetric comparison FAILED." << endl;
296  }
297  }
298  else {
299  if (comm->getRank() == 0) {
300  cout << "ComparisonHelper did not understand how to read the xml. Test FAILED." << endl;
301  }
302  }
303  return false;
304 }
305 
306 bool ComparisonHelper::CompareSolutions(const string &p1,
307  const string &p2,
308  const RCP<const Comm<int> > &comm)
309 {
310  if(comm->getRank() == 0) printf("\nComparing: %s and %s\n",p1.c_str(),p2.c_str());
311  auto A = this->sources[p1];
312  auto B = this->sources[p2];
313  if(A->problem_kind != B->problem_kind) {
314  cout << "Problem A and B are of a different kind and cannot be compared.";
315  cout <<"\nSolution comparison FAILED." << endl;
316  }
317  else {
318  if(A->problem_kind == "partitioning") {
319  return this->ComparePartitionSolutions(A.getRawPtr(), B.getRawPtr(), comm);
320  }
321  else if(A->problem_kind == "coloring") {
322  return this->CompareColoringSolutions(A.getRawPtr(), B.getRawPtr(), comm);
323  }
324  else if(A->problem_kind == "ordering"){
325  return this->CompareOrderingSolutions(A.getRawPtr(), B.getRawPtr(), comm);
326  }
327  else {
328  cout << "Problem kind not recognized. Check spelling.";
329  cout <<"\nSolution comparison FAILED." << endl;
330  }
331  }
332  return false;
333 }
334 
335 void
336 ComparisonHelper::reduceWithMessage(const RCP<const Comm<int> > &comm, const std::string &msg_in,
337  int &local_status, std::ostringstream &msg)
338 {
339  comm->barrier();
340  int global_buff;
341  Teuchos::Ptr<int> global(&global_buff);
342  reduceAll<int,int>(*comm.get(), Teuchos::EReductionType::REDUCE_MAX, local_status , global);
343 
344  local_status = *global;
345  if (local_status == 1) {
346  msg << msg_in;
347  }
348 
349 }
350 
351 bool ComparisonHelper::ComparePartitionSolutions(const ComparisonSource * sourceA,
352  const ComparisonSource * sourceB,
353  const RCP<const Comm<int> > &comm)
354 {
355  int rank = comm->getRank();
356  ostringstream status;
357  int failed = 0;
358 
359  if(!sourceA->problem.getRawPtr()){ failed = 1;}
360  ComparisonHelper::reduceWithMessage(comm,
361  "Solution A is NULL. Solution comparison FAILED.",
362  failed,
363  status);
364 
365  if(!failed && !sourceB->problem.getRawPtr()){ failed = 1;}
366  ComparisonHelper::reduceWithMessage(comm,
367  "Solution B is NULL. Solution comparison FAILED.",
368  failed,
369  status);
370 
371  if(!failed) {
372  // typedef Zoltan2::PartitioningSolution<basic_id_t> partitioning_solution_t; // BDD unused
373  // have some solutions lets compare them
374  if (partitioning_problem_t * problem_a = reinterpret_cast<partitioning_problem_t *>(sourceA->problem.getRawPtr())) {
375  if(partitioning_problem_t * problem_b = reinterpret_cast<partitioning_problem_t *>(sourceB->problem.getRawPtr())) {
376  auto solution_a = problem_a->getSolution();
377  auto solution_b = problem_b->getSolution();
378 
379  if(sourceA->adapter->getLocalNumIDs() != sourceB->adapter->getLocalNumIDs()) {
380  failed = 1;
381  }
382  ComparisonHelper::reduceWithMessage(comm,
383  "Number of parts in Solution A != Solution B. \
384  Partitioning solution comparison FAILED.",
385  failed,
386  status);
387 
388  if (!failed) {
389  for(size_t i = 0; i < sourceA->adapter->getLocalNumIDs(); i++) {
390  if(!failed && solution_a.getPartListView()[i] != solution_b.getPartListView()[i]) {
391  failed = 1;
392  ComparisonHelper::reduceWithMessage(comm, "Solution sets A and B have different values for getPartListView(). Solution comparison FAILED.", failed, status);
393  }
394  }
395  }
396  }
397  else {
398  failed = 1;
399  ComparisonHelper::reduceWithMessage(comm,
400  "Solution sets A and B are from different problem types. \
401  Solution comparison FAILED.",
402  failed,
403  status);
404  }
405 
406  } else {
407  failed = 1;
408  ComparisonHelper::reduceWithMessage(comm,
409  "Could not cast solution A to valid problem type. \
410  Solution comparison FAILED.",
411  failed,
412  status);
413  }
414  }
415 
416  if(!failed) {
417  status << "Solution sets A and B are the same. ";
418  status << "Solution set comparison PASSED.";
419  }
420 
421  if(rank == 0) {
422  cout << status.str() << endl;
423  }
424  return (failed == 0);
425 }
426 
427 
428 bool ComparisonHelper::CompareColoringSolutions(const ComparisonSource * sourceA,
429  const ComparisonSource * sourceB,
430  const RCP<const Comm<int> > &comm)
431 {
432  int rank = comm->getRank();
433  ostringstream status;
434  int failed = 0;
435 
436  if(!sourceA->problem.getRawPtr()) {
437  failed = 1;
438  }
439  ComparisonHelper::reduceWithMessage(comm,
440  "Solution A is NULL. Solution comparison FAILED.",
441  failed,
442  status);
443 
444  if(!failed && !sourceB->problem.getRawPtr()) {
445  failed = 1;
446  }
447  ComparisonHelper::reduceWithMessage(comm,
448  "Solution B is NULL. Solution comparison FAILED.",
449  failed,
450  status);
451 
452  if (!failed) {
453  // have some solutions lets compare them
455  // have some solutions lets compare them
456  if(coloring_problem_t * problem_a = reinterpret_cast<coloring_problem_t *>(sourceA->problem.getRawPtr())) {
457  if(coloring_problem_t * problem_b = reinterpret_cast<coloring_problem_t *>(sourceB->problem.getRawPtr())) {
458  auto solution_a = problem_a->getSolution();
459  auto solution_b = problem_b->getSolution();
460 
461  if(solution_a->getNumColors() != solution_b->getNumColors()) {
462  failed = 1;
463  }
464  ComparisonHelper::reduceWithMessage(comm,
465  "Number of colors for Solution A != Solution B. \
466  Coloring solution comparison FAILED.",
467  failed,
468  status);
469 
470  if (!failed) {
471  if(solution_a->getColorsSize() != solution_b->getColorsSize()) {
472  failed = 1;
473  }
474  ComparisonHelper::reduceWithMessage(comm,
475  "Size of colors array for Solution A != Solution B. \
476  Coloring solution comparison FAILED.",
477  failed,
478  status);
479 
480  }
481 
482  if (!failed) {
483  for(size_t i = 0; i < solution_a->getColorsSize(); i++) {
484  if (solution_a->getColors()[i] != solution_b->getColors()[i]) {
485  if(!failed) {
486  failed = 1; // fail
487  }
488  }
489  }
490  ComparisonHelper::reduceWithMessage(comm,
491  "Coloring solution comparison FAILED.",
492  failed,
493  status);
494  }
495  }
496  else {
497  failed = 1;
498  ComparisonHelper::reduceWithMessage(comm,
499  "Solution sets A and B are from different problem types. \
500  Solution comparison FAILED.",
501  failed,
502  status);
503  }
504 
505  }
506  else {
507  failed = 1;
508  ComparisonHelper::reduceWithMessage(comm,
509  "Could not cast solution A to valid problem type. \
510  Solution comparison FAILED.",
511  failed,
512  status);
513  }
514  }
515 
516  if (!failed) {
517  status << "Solution sets A and B are the same. ";
518  status << "Solution set comparison PASSED.";
519  }
520 
521  if (rank == 0) {
522  cout << status.str() << endl;
523  }
524  return (failed == 0);
525 }
526 
527 bool ComparisonHelper::CompareOrderingSolutions(const ComparisonSource * sourceA,
528  const ComparisonSource * sourceB,
529  const RCP<const Comm<int> > &comm)
530 {
531  int rank = comm->getRank();
532  ostringstream status;
533  int failed = 0;
534 
535  if (!sourceA->problem.getRawPtr()) {
536  failed = 1;
537  }
538  ComparisonHelper::reduceWithMessage(comm,
539  "Solution A is NULL. Solution comparison FAILED.",
540  failed,
541  status);
542 
543  if(!failed && !sourceB->problem.getRawPtr()){ failed = 1;}
544  ComparisonHelper::reduceWithMessage(comm,
545  "Solution B is NULL. Solution comparison FAILED.",
546  failed,
547  status);
548 
549  // if(!failed) //BDD, finish implementation when ordering problem metrics defined
550  // {
551  // // have some solutions lets compare them
552  // typedef Zoltan2::OrderingProblem<basic_id_t> ordering_problem_t;
553  // // have some solutions lets compare them
554  // if(ordering_problem_t * problem_a = reinterpret_cast<ordering_problem_t *>(sourceA->problem.getRawPtr()))
555  // {
556  // if(ordering_problem_t * problem_b = reinterpret_cast<ordering_problem_t *>(sourceB->problem.getRawPtr()))
557  // {
558  //
559  // }else{
560  // status << "Solution sets A and B are from different problem types. ";
561  // status << "Solution comparison FAILED.";
562  // failed = true;
563  // }
564  //
565  //
566  // }else{
567  // if(rank == 0)
568  // {
569  // status << "Could not cast solution A to valid problem type. ";
570  // status << "Solution comparison FAILED.";
571  // }
572  // }
573  // }
574 
575 
576  if(!failed) {
577  status << "Solution sets A and B are the same. ";
578  status << "Solution set comparison PASSED.";
579  }
580 
581  if(rank == 0) {
582  cout << status.str() << endl;
583  }
584  return (failed == 0);
585 }
586 
587 // compare metrics
588 bool ComparisonHelper::CompareMetrics(const ParameterList &metricsPlist, const RCP<const Comm<int> > &comm)
589 {
590  int rank = comm->getRank();
591 
592  //get sources for problema nd reference
593  const string prb_name = metricsPlist.get<string>("Problem");
594  const string ref_name = metricsPlist.get<string>("Reference");
595  if(rank == 0) {
596  cout << "\nMetric/Timer comparison of: " << prb_name << " and ";
597  cout << ref_name <<" (reference source)\n";
598  }
599 
600  // get sources
601  RCP<const ComparisonSource> sourcePrb = this->sources[prb_name];
602  RCP<const ComparisonSource> sourceRef = this->sources[ref_name];
603 
604  // get timing data
605  std::map< const string, const double> prb_timers = this->timerDataToMap(sourcePrb->timers);
606  std::map< const string, const double> ref_timers = this->timerDataToMap(sourceRef->timers);
607 
608  // get all of the metrics to be tested
609  std::queue<ParameterList> metrics = ComparisonHelper::getMetricsToCompare(metricsPlist);
610 
611  // run comparison
612  int all_tests_pass = 1;
613  string metric_name;
614 
615  while(!metrics.empty()) {
616  // print their names...
617  ostringstream msg;
618  metric_name = metrics.front().name();
619 
620  if (metric_name == "Metrics") { // special key word means compare the metrics list
621  std::vector<MetricAnalyzerInfo> metricInfoSetPrb;
622  std::vector<MetricAnalyzerInfo> metricInfoSetRef;
623 
624  MetricAnalyzer::LoadMetricInfo(metricInfoSetPrb, sourcePrb.get()->metricObject, metricsPlist.sublist("Metrics"));
625  MetricAnalyzer::LoadMetricInfo(metricInfoSetRef, sourceRef.get()->metricObject, metricsPlist.sublist("Metrics"));
626 
627  // there is some redundancy here because the metric info holds both the questions and the results
628  // this happened because I wanted to reuse the MetricAnalyzer code for loading metric checks or comparisons
629  // we can iterate over either to get the questions
630  for (size_t n = 0; n < metricInfoSetPrb.size(); ++n) {
631  if(!ComparisonHelper::metricComparisonTest(comm, metricInfoSetPrb[n], metricInfoSetRef[n], metrics.front(), msg)) {
632  all_tests_pass = 0;
633  }
634  if(rank == 0) {
635  cout << msg.str() << endl;
636  }
637  }
638  }
639  else if(prb_timers.find(metric_name) != prb_timers.end() && ref_timers.find(metric_name) != ref_timers.end()) {
640  if(rank == 0) cout << "\ncomparing timer: " << metric_name << endl;
641  if(!ComparisonHelper::timerComparisonTest(comm,
642  prb_timers.at(metric_name),
643  ref_timers.at(metric_name),
644  metrics.front(), msg)) {
645  all_tests_pass = 0;
646  if (rank == 0) {
647  cout << "timer comparison test caused a FAILED event." << endl;
648  }
649  }
650 
651  if(rank == 0) {
652  cout << msg.str() << endl;
653  }
654  }
655 
656  metrics.pop();
657  }
658 
659  if(rank == 0) {
660  if(all_tests_pass == 1) {
661  cout << "\nAll metric/timer comparisons PASSED." << endl;
662  }
663  else {
664  cout << "\nMetric/timer metric comparisons FAILED." << endl;
665  }
666  }
667 
668  return (all_tests_pass == 1);
669 }
670 
671 std::map<const string, const double> ComparisonHelper::timerDataToMap(const map<const std::string, RCP<Time> > &timers)
672 {
673  typedef std::pair<const string,const double> pair_t;
674  std::map<const string, const double> time_data;
675  for (auto &i : timers) {
676  time_data.insert(pair_t(i.first, i.second->totalElapsedTime()));
677  }
678  return time_data;
679 }
680 
681 bool ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
682  const MetricAnalyzerInfo & metric,
683  const MetricAnalyzerInfo & ref_metric,
684  const Teuchos::ParameterList & metricPlist,
685  ostringstream &msg)
686 {
687  // run a comparison of min and max against a given metric
688  // return an error message on failure
689  bool pass = true;
690  string test_name = metricPlist.name() + " test";
691  double ref_value = ref_metric.theValue;
692  double value = metric.theValue;
693 
694  if (ref_value == 0) {
695  throw std::logic_error( "The parameter list had a 0 value for the reference value so a percentage cannot be calculated." );
696  }
697  double percentRatio = value / ref_value;
698 
699  // want to reduce value to max value for all procs
700 
701  if (ref_metric.bFoundLowerBound) {
702  double min = ref_metric.lowerValue;
703  if (percentRatio < min) {
704  msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which less than specified allowable minimum percent, " << min << ".\n";
705  pass = false;
706  }
707  else {
708  msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable minimum percent, " << min << ".\n";
709  }
710  }
711 
712  if (ref_metric.bFoundUpperBound) {
713  double max = ref_metric.upperValue;
714  if (percentRatio > max) {
715  msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable maximum percent, " << max << ".\n";
716  pass = false;
717  }
718  else {
719  msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is less than specified allowable maximum percent, " << max << ".\n";
720  }
721  }
722 
723  return pass;
724 }
725 // BDD, to do: print metrics even for pass
726 // reduce max metric to process 0
727 // print only on process 0 --- duh.
728 bool ComparisonHelper::timerComparisonTest(const RCP<const Comm<int> > &comm,
729  const double time,
730  const double ref_time,
731  const Teuchos::ParameterList & metricPlist,
732  ostringstream &msg)
733 {
734  // Reduce time from test
735  double global_time;
736  Teuchos::Ptr<double> global(&global_time);
737  comm->barrier();
738  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,time,global);
739 
740  // Reduce time from reference
741  double global_ref_time;
742  Teuchos::Ptr<double> globalRef(&global_ref_time);
743  comm->barrier();
744  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,ref_time,globalRef);
745 
746  // run a comparison of min and max against a given metric
747  // return an error message on failure
748  bool pass = true;
749  string test_name = metricPlist.name() + " test";
750 
751  if (metricPlist.isParameter("lower")) {
752  double min = metricPlist.get<double>("lower")*global_ref_time;
753 
754  if (global_time < min) {
755  msg << test_name << " FAILED: Minimum time, "
756  << time <<
757  "[s], less than specified allowable minimum time, " << min <<"[s]"<< ".\n";
758  pass = false;
759  }
760  else {
761  msg << test_name << " PASSED: Minimum time, "
762  << time <<
763  "[s], greater than specified allowable minimum time, " << min <<"[s]"<< ".\n";
764  }
765  }
766 
767  if (metricPlist.isParameter("upper" ) && pass != false) {
768  double max = metricPlist.get<double>("upper") * global_ref_time;
769  if (global_time > max) {
770  msg << test_name << " FAILED: Maximum time, "
771  << global_time <<
772  "[s], greater than specified allowable maximum time, " << max <<"[s]"<< ".\n";
773  pass = false;
774  }
775  else {
776  msg << test_name << " PASSED: Maximum time, "
777  << global_time <<
778  "[s], less than specified allowable maximum time, " << max <<"[s]"<< ".\n";
779  }
780  }
781 
782  return pass;
783 }
784 
785 std::queue<ParameterList> ComparisonHelper::getMetricsToCompare(const ParameterList &pList)
786 {
787  // extract all of the metrics to be tested
788  std::queue<ParameterList> metrics;
789  for(auto it = pList.begin(); it != pList.end(); ++it) {
790  if (pList.isSublist(it->first)) {
791  metrics.push(pList.sublist(it->first));
792  }
793  }
794  return metrics;
795 }
796 
keep typedefs that commonly appear in many places localized
ColoringProblem sets up coloring problems for the user.
Defines the ColoringProblem class.
common code used by tests
Zoltan2::ColoringProblem< basic_id_t > coloring_problem_t
void AddSource(const string &name, ComparisonSource *source)
bool Compare(const ParameterList &pList, const RCP< const Comm< int > > &comm)
RCP< Zoltan2::EvaluatePartition< basic_id_t > > metricObject
static void LoadMetricInfo(std::vector< MetricAnalyzerInfo > &metricInfoSet, const RCP< const Zoltan2::EvaluatePartition< basic_id_t > > &metricObject, const ParameterList &metricsParameters)
A class for comparing solutions, metrics, and timing data of Zoltan2 problems.
std::map< const std::string, RCP< Time > > timers
Defines the OrderingProblem class.
PartitioningProblem sets up partitioning problems for the user.
static const std::string pass
RCP< Zoltan2::VectorAdapter< tMVector_t > > coordinateAdapterRCP
A class used to save problem solutions and timers.
Defines the PartitioningProblem class.
RCP< base_problem_t > problem
void addTimer(const std::string &name)
Generate Adapter for testing purposes.