60 #include <Teuchos_DefaultComm.hpp> 61 #include <Teuchos_Time.hpp> 70 using Teuchos::ParameterList;
77 using std::ostringstream;
78 using Teuchos::reduceAll;
92 timers.insert(std::pair<
const std::string &, RCP<Time> >(name,rcp(
new Time(name))));
93 timers[name]->enable();
102 std::map<const std::string, RCP<Time> >
timers;
116 bool Compare(
const ParameterList &pList,
const RCP<
const Comm<int> > &comm);
128 return this->sources.size();
132 map<const string,RCP<const ComparisonSource> > sources;
140 bool CompareSolutions(
const string &p1,
142 const RCP<
const Comm<int> > &comm);
151 const RCP<
const Comm<int> > &comm);
160 const RCP<
const Comm<int> > &comm);
169 const RCP<
const Comm<int> > &comm);
175 bool CompareMetrics(
const ParameterList &metricsPlist,
const RCP<
const Comm<int> > &comm);
187 metricComparisonTest(
const RCP<
const Comm<int> > &comm,
190 const Teuchos::ParameterList & metricPlist,
203 timerComparisonTest(
const RCP<
const Comm<int> > &comm,
205 const double ref_time,
206 const Teuchos::ParameterList & metricPlist,
214 static std::map<const string, const double>
215 timerDataToMap(
const map<
const std::string, RCP<Time> > &timers);
223 static std::queue<ParameterList>
224 getMetricsToCompare(
const ParameterList & pList);
227 reduceWithMessage(
const RCP<
const Comm<int> > &comm,
const std::string &msg_in,
228 int &local_status, std::ostringstream &msg);
235 typedef std::pair<const string &, RCP<const ComparisonSource> > pair_t;
236 this->sources.insert(pair_t(name, RCP<ComparisonSource>(source)));
241 if(pList.isParameter(
"A") && pList.isParameter(
"B")) {
243 string pA = pList.get<
string>(
"A");
244 if(this->sources.find(pA) == this->sources.end())
246 cout <<
"\nProblem: " + pA +
", was not saved for comparison.";
247 cout <<
"\nThis typically indicates that an error occurred while running the problem.";
248 cout <<
"\nSolution comparison FAILED." << endl;
252 string pB = pList.get<
string>(
"B");
253 if(this->sources.find(pB) == this->sources.end()) {
254 cout <<
"\nProblem: " + pB +
", was not saved for comparison.";
255 cout <<
"\nThis typically indicates that an error occurred while running the problem.";
256 cout <<
"\nSolution comparison FAILED." << endl;
260 bool bResult = this->CompareSolutions(pA, pB, comm);
263 else if (pList.isParameter(
"Problem") && pList.isParameter(
"Reference")) {
265 string prb = pList.get<
string>(
"Problem");
266 if(this->sources.find(prb) == this->sources.end()) {
267 cout <<
"\nProblem: " + prb +
", was not saved for comparison.";
268 cout <<
"\nThis typically indicates that an error occurred while running the problem.";
269 cout <<
"\nMetric comparison FAILED." << endl;
273 string ref = pList.get<
string>(
"Reference");
274 if(this->sources.find(ref) == this->sources.end()) {
275 cout <<
"\nReference: " + ref +
", was not saved for comparison.";
276 cout <<
"\nThis typically indicates that an error occurred while running the problem.";
277 cout <<
"\nMetric comparison FAILED." << endl;
281 bool bResult = this->CompareMetrics(pList, comm);
284 else if (pList.isParameter(
"A") || pList.isParameter(
"B"))
286 if(comm->getRank() == 0)
288 cout <<
"Problem A or Problem B is not specified -- check input.";
289 cout <<
"\nSolution comparison FAILED." << endl;
292 else if (pList.isParameter(
"Problem") || pList.isParameter(
"Reference")) {
293 if(comm->getRank() == 0) {
294 cout <<
"Problem or reference is not specified -- check input.";
295 cout <<
"\nMetric comparison FAILED." << endl;
299 if (comm->getRank() == 0) {
300 cout <<
"ComparisonHelper did not understand how to read the xml. Test FAILED." << endl;
306 bool ComparisonHelper::CompareSolutions(
const string &p1,
308 const RCP<
const Comm<int> > &comm)
310 if(comm->getRank() == 0) printf(
"\nComparing: %s and %s\n",p1.c_str(),p2.c_str());
311 auto A = this->sources[p1];
312 auto B = this->sources[p2];
313 if(A->problem_kind != B->problem_kind) {
314 cout <<
"Problem A and B are of a different kind and cannot be compared.";
315 cout <<
"\nSolution comparison FAILED." << endl;
318 if(A->problem_kind ==
"partitioning") {
319 return this->ComparePartitionSolutions(A.getRawPtr(), B.getRawPtr(), comm);
321 else if(A->problem_kind ==
"coloring") {
322 return this->CompareColoringSolutions(A.getRawPtr(), B.getRawPtr(), comm);
324 else if(A->problem_kind ==
"ordering"){
325 return this->CompareOrderingSolutions(A.getRawPtr(), B.getRawPtr(), comm);
328 cout <<
"Problem kind not recognized. Check spelling.";
329 cout <<
"\nSolution comparison FAILED." << endl;
336 ComparisonHelper::reduceWithMessage(
const RCP<
const Comm<int> > &comm,
const std::string &msg_in,
337 int &local_status, std::ostringstream &msg)
341 Teuchos::Ptr<int> global(&global_buff);
342 reduceAll<int,int>(*comm.get(), Teuchos::EReductionType::REDUCE_MAX, local_status , global);
344 local_status = *global;
345 if (local_status == 1) {
351 bool ComparisonHelper::ComparePartitionSolutions(
const ComparisonSource * sourceA,
353 const RCP<
const Comm<int> > &comm)
355 int rank = comm->getRank();
356 ostringstream status;
359 if(!sourceA->
problem.getRawPtr()){ failed = 1;}
360 ComparisonHelper::reduceWithMessage(comm,
361 "Solution A is NULL. Solution comparison FAILED.",
365 if(!failed && !sourceB->
problem.getRawPtr()){ failed = 1;}
366 ComparisonHelper::reduceWithMessage(comm,
367 "Solution B is NULL. Solution comparison FAILED.",
376 auto solution_a = problem_a->getSolution();
377 auto solution_b = problem_b->getSolution();
379 if(sourceA->
adapter->getLocalNumIDs() != sourceB->
adapter->getLocalNumIDs()) {
382 ComparisonHelper::reduceWithMessage(comm,
383 "Number of parts in Solution A != Solution B. \ 384 Partitioning solution comparison FAILED.",
389 for(
size_t i = 0; i < sourceA->
adapter->getLocalNumIDs(); i++) {
390 if(!failed && solution_a.getPartListView()[i] != solution_b.getPartListView()[i]) {
392 ComparisonHelper::reduceWithMessage(comm,
"Solution sets A and B have different values for getPartListView(). Solution comparison FAILED.", failed, status);
399 ComparisonHelper::reduceWithMessage(comm,
400 "Solution sets A and B are from different problem types. \ 401 Solution comparison FAILED.",
408 ComparisonHelper::reduceWithMessage(comm,
409 "Could not cast solution A to valid problem type. \ 410 Solution comparison FAILED.",
417 status <<
"Solution sets A and B are the same. ";
418 status <<
"Solution set comparison PASSED.";
422 cout << status.str() << endl;
424 return (failed == 0);
428 bool ComparisonHelper::CompareColoringSolutions(
const ComparisonSource * sourceA,
430 const RCP<
const Comm<int> > &comm)
432 int rank = comm->getRank();
433 ostringstream status;
436 if(!sourceA->
problem.getRawPtr()) {
439 ComparisonHelper::reduceWithMessage(comm,
440 "Solution A is NULL. Solution comparison FAILED.",
444 if(!failed && !sourceB->
problem.getRawPtr()) {
447 ComparisonHelper::reduceWithMessage(comm,
448 "Solution B is NULL. Solution comparison FAILED.",
458 auto solution_a = problem_a->getSolution();
459 auto solution_b = problem_b->getSolution();
461 if(solution_a->getNumColors() != solution_b->getNumColors()) {
464 ComparisonHelper::reduceWithMessage(comm,
465 "Number of colors for Solution A != Solution B. \ 466 Coloring solution comparison FAILED.",
471 if(solution_a->getColorsSize() != solution_b->getColorsSize()) {
474 ComparisonHelper::reduceWithMessage(comm,
475 "Size of colors array for Solution A != Solution B. \ 476 Coloring solution comparison FAILED.",
483 for(
size_t i = 0; i < solution_a->getColorsSize(); i++) {
484 if (solution_a->getColors()[i] != solution_b->getColors()[i]) {
490 ComparisonHelper::reduceWithMessage(comm,
491 "Coloring solution comparison FAILED.",
498 ComparisonHelper::reduceWithMessage(comm,
499 "Solution sets A and B are from different problem types. \ 500 Solution comparison FAILED.",
508 ComparisonHelper::reduceWithMessage(comm,
509 "Could not cast solution A to valid problem type. \ 510 Solution comparison FAILED.",
517 status <<
"Solution sets A and B are the same. ";
518 status <<
"Solution set comparison PASSED.";
522 cout << status.str() << endl;
524 return (failed == 0);
527 bool ComparisonHelper::CompareOrderingSolutions(
const ComparisonSource * sourceA,
529 const RCP<
const Comm<int> > &comm)
531 int rank = comm->getRank();
532 ostringstream status;
535 if (!sourceA->
problem.getRawPtr()) {
538 ComparisonHelper::reduceWithMessage(comm,
539 "Solution A is NULL. Solution comparison FAILED.",
543 if(!failed && !sourceB->
problem.getRawPtr()){ failed = 1;}
544 ComparisonHelper::reduceWithMessage(comm,
545 "Solution B is NULL. Solution comparison FAILED.",
577 status <<
"Solution sets A and B are the same. ";
578 status <<
"Solution set comparison PASSED.";
582 cout << status.str() << endl;
584 return (failed == 0);
588 bool ComparisonHelper::CompareMetrics(
const ParameterList &metricsPlist,
const RCP<
const Comm<int> > &comm)
590 int rank = comm->getRank();
593 const string prb_name = metricsPlist.get<
string>(
"Problem");
594 const string ref_name = metricsPlist.get<
string>(
"Reference");
596 cout <<
"\nMetric/Timer comparison of: " << prb_name <<
" and ";
597 cout << ref_name <<
" (reference source)\n";
601 RCP<const ComparisonSource> sourcePrb = this->sources[prb_name];
602 RCP<const ComparisonSource> sourceRef = this->sources[ref_name];
605 std::map< const string, const double> prb_timers = this->timerDataToMap(sourcePrb->timers);
606 std::map< const string, const double> ref_timers = this->timerDataToMap(sourceRef->timers);
609 std::queue<ParameterList> metrics = ComparisonHelper::getMetricsToCompare(metricsPlist);
612 int all_tests_pass = 1;
615 while(!metrics.empty()) {
618 metric_name = metrics.front().name();
620 if (metric_name ==
"Metrics") {
621 std::vector<MetricAnalyzerInfo> metricInfoSetPrb;
622 std::vector<MetricAnalyzerInfo> metricInfoSetRef;
630 for (
size_t n = 0; n < metricInfoSetPrb.size(); ++n) {
631 if(!ComparisonHelper::metricComparisonTest(comm, metricInfoSetPrb[n], metricInfoSetRef[n], metrics.front(), msg)) {
635 cout << msg.str() << endl;
639 else if(prb_timers.find(metric_name) != prb_timers.end() && ref_timers.find(metric_name) != ref_timers.end()) {
640 if(rank == 0) cout <<
"\ncomparing timer: " << metric_name << endl;
641 if(!ComparisonHelper::timerComparisonTest(comm,
642 prb_timers.at(metric_name),
643 ref_timers.at(metric_name),
644 metrics.front(), msg)) {
647 cout <<
"timer comparison test caused a FAILED event." << endl;
652 cout << msg.str() << endl;
660 if(all_tests_pass == 1) {
661 cout <<
"\nAll metric/timer comparisons PASSED." << endl;
664 cout <<
"\nMetric/timer metric comparisons FAILED." << endl;
668 return (all_tests_pass == 1);
671 std::map<const string, const double> ComparisonHelper::timerDataToMap(
const map<
const std::string, RCP<Time> > &timers)
673 typedef std::pair<const string,const double> pair_t;
674 std::map<const string, const double> time_data;
675 for (
auto &i : timers) {
676 time_data.insert(pair_t(i.first, i.second->totalElapsedTime()));
681 bool ComparisonHelper::metricComparisonTest(
const RCP<
const Comm<int> > &comm,
684 const Teuchos::ParameterList & metricPlist,
690 string test_name = metricPlist.name() +
" test";
691 double ref_value = ref_metric.
theValue;
694 if (ref_value == 0) {
695 throw std::logic_error(
"The parameter list had a 0 value for the reference value so a percentage cannot be calculated." );
697 double percentRatio = value / ref_value;
703 if (percentRatio < min) {
704 msg << test_name <<
" FAILED: " << ref_metric.
parameterDescription <<
": " << value <<
" is " << percentRatio <<
" percent of the reference value " << ref_value <<
", which less than specified allowable minimum percent, " << min <<
".\n";
708 msg << test_name <<
" PASSED: " << ref_metric.
parameterDescription <<
": " << value <<
" is " << percentRatio <<
" percent of the reference value " << ref_value <<
", which is greater than specified allowable minimum percent, " << min <<
".\n";
714 if (percentRatio > max) {
715 msg << test_name <<
" FAILED: " << ref_metric.
parameterDescription <<
": " << value <<
" is " << percentRatio <<
" percent of the reference value " << ref_value <<
", which is greater than specified allowable maximum percent, " << max <<
".\n";
719 msg << test_name <<
" PASSED: " << ref_metric.
parameterDescription <<
": " << value <<
" is " << percentRatio <<
" percent of the reference value " << ref_value <<
", which is less than specified allowable maximum percent, " << max <<
".\n";
728 bool ComparisonHelper::timerComparisonTest(
const RCP<
const Comm<int> > &comm,
730 const double ref_time,
731 const Teuchos::ParameterList & metricPlist,
736 Teuchos::Ptr<double> global(&global_time);
738 reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,time,global);
741 double global_ref_time;
742 Teuchos::Ptr<double> globalRef(&global_ref_time);
744 reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,ref_time,globalRef);
749 string test_name = metricPlist.name() +
" test";
751 if (metricPlist.isParameter(
"lower")) {
752 double min = metricPlist.get<
double>(
"lower")*global_ref_time;
754 if (global_time < min) {
755 msg << test_name <<
" FAILED: Minimum time, " 757 "[s], less than specified allowable minimum time, " << min <<
"[s]"<<
".\n";
761 msg << test_name <<
" PASSED: Minimum time, " 763 "[s], greater than specified allowable minimum time, " << min <<
"[s]"<<
".\n";
767 if (metricPlist.isParameter(
"upper" ) &&
pass !=
false) {
768 double max = metricPlist.get<
double>(
"upper") * global_ref_time;
769 if (global_time > max) {
770 msg << test_name <<
" FAILED: Maximum time, " 772 "[s], greater than specified allowable maximum time, " << max <<
"[s]"<<
".\n";
776 msg << test_name <<
" PASSED: Maximum time, " 778 "[s], less than specified allowable maximum time, " << max <<
"[s]"<<
".\n";
785 std::queue<ParameterList> ComparisonHelper::getMetricsToCompare(
const ParameterList &pList)
788 std::queue<ParameterList> metrics;
789 for(
auto it = pList.begin(); it != pList.end(); ++it) {
790 if (pList.isSublist(it->first)) {
791 metrics.push(pList.sublist(it->first));
keep typedefs that commonly appear in many places localized
ColoringProblem sets up coloring problems for the user.
Defines the ColoringProblem class.
common code used by tests
Zoltan2::ColoringProblem< basic_id_t > coloring_problem_t
void AddSource(const string &name, ComparisonSource *source)
RCP< basic_id_t > adapter
bool Compare(const ParameterList &pList, const RCP< const Comm< int > > &comm)
RCP< Zoltan2::EvaluatePartition< basic_id_t > > metricObject
size_t getNumberOfSources() const
static void LoadMetricInfo(std::vector< MetricAnalyzerInfo > &metricInfoSet, const RCP< const Zoltan2::EvaluatePartition< basic_id_t > > &metricObject, const ParameterList &metricsParameters)
A class for comparing solutions, metrics, and timing data of Zoltan2 problems.
std::map< const std::string, RCP< Time > > timers
Defines the OrderingProblem class.
PartitioningProblem sets up partitioning problems for the user.
static const std::string pass
RCP< Zoltan2::VectorAdapter< tMVector_t > > coordinateAdapterRCP
A class used to save problem solutions and timers.
Defines the PartitioningProblem class.
RCP< base_problem_t > problem
std::string parameterDescription
void addTimer(const std::string &name)
Generate Adapter for testing purposes.