Zoltan2
Zoltan2_MetricAnalyzer.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // Zoltan2: A package of combinatorial algorithms for scientific computing
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Karen Devine (kddevin@sandia.gov)
39 // Erik Boman (egboman@sandia.gov)
40 // Siva Rajamanickam (srajama@sandia.gov)
41 //
42 // ***********************************************************************
43 //
44 // @HEADER
45 
46 /* \file Zoltan2_MetricAnalyzer.hpp
47  * \brief Used by the Zoltan2 test driver for running \
48  simple pass fail tests based on ranges of problem metrics.
49  */
50 #ifndef ZOLTAN2_METRIC_ANALYZER_HPP
51 #define ZOLTAN2_METRIC_ANALYZER_HPP
52 
53 #include <Zoltan2_TestHelpers.hpp>
54 #include <Zoltan2_Typedefs.hpp>
55 #include <Teuchos_DefaultComm.hpp>
56 #include <Teuchos_XMLObject.hpp>
57 #include <Teuchos_FileInputSource.hpp>
58 
59 #include <sstream>
60 #include <string>
61 #include <iostream>
62 
63 using Teuchos::ParameterList;
64 using Teuchos::Comm;
65 using Teuchos::RCP;
66 using Teuchos::ArrayRCP;
67 using namespace Zoltan2_TestingFramework;
68 
69 // these define key names which convert to an API call
70 #define API_STRING_getWeightImbalance "imbalance"
71 #define API_STRING_getTotalEdgeCuts "total edge cuts"
72 #define API_STRING_getMaxEdgeCuts "max edge cuts"
73 
74 // these define the options for a particular type
75 #define KEYWORD_PARAMETER_NAME "check" // would usually be the first entry and identify the API call
76 #define WEIGHT_PARAMETER_NAME "weight"
77 #define UPPER_PARAMETER_NAME "upper"
78 #define LOWER_PARAMETER_NAME "lower"
79 #define NORMED_PARAMETER_NAME "normed"
80 
81 #define UNDEFINED_PARAMETER_INT_INDEX -1 // didn't want to duplicate this value - a weight index should be 0 or larger but it's optional to specify it
82 
83 // general result for reading metrics - set this up to share for both metrics or comparisons (bounds are percents)
85 {
87  double upperValue;
88  double lowerValue;
91  std::string parameterDescription;
92 };
93 
95 public:
103  static bool analyzeMetrics(
104  const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject,
105  const ParameterList &metricsParameters,
106  std::ostringstream & msg_stream )
107  {
108  if (metricsParameters.numParams() == 0) {
109  return true; // specification is that we do nothing - we may just be testing our status
110  }
111 
112  bool bAllPassed = true;
113 
114  std::vector<MetricAnalyzerInfo> metricInfoSet;
115  LoadMetricInfo(metricInfoSet, metricObject, metricsParameters);
116 
117  int countFailedMetricChecks = 0;
118  for (auto metricInfo = metricInfoSet.begin();
119  metricInfo != metricInfoSet.end(); ++metricInfo) {
120  if (!MetricAnalyzer::executeMetricCheck(*metricInfo, msg_stream)) {
121  ++countFailedMetricChecks;
122  }
123  }
124 
125  // this code prints a summary of all metric checks and indicates how many failed, if any did fail
126  if(countFailedMetricChecks == 0) {
127  msg_stream << metricsParameters.numParams() << " out of " << metricsParameters.numParams() << " metric checks" << " PASSED." << std::endl;
128  }
129  else {
130  msg_stream << countFailedMetricChecks << " out of " << metricsParameters.numParams() << " metric checks " << " FAILED." << std::endl;
131  bAllPassed = false;
132  }
133  msg_stream << std::endl; // cosmetic spacer
134  return bAllPassed;
135  }
136 
137  static void LoadMetricInfo(
138  std::vector<MetricAnalyzerInfo> & metricInfoSet,
139  const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject,
140  const ParameterList &metricsParameters) {
141 
142  // at this point we should be looking at a metricsPlist with the following
143  // format - note that weight is optional
144 
145  // <ParameterList name="metriccheck1">
146  // <Parameter name="check" type="string" value="imbalance"/>
147  // <Parameter name="lower" type="double" value="0.99"/>
148  // <Parameter name="upper" type="double" value="1.4"/>
149  // </ParameterList>
150  // <ParameterList name="metriccheck2">
151  // <Parameter name="check" type="string" value="imbalance"/>
152  // <Parameter name="weight" type="int" value="0"/>
153  // <Parameter name="lower" type="double" value="0.99"/>
154  // <Parameter name="upper" type="double" value="1.4"/>
155  // </ParameterList>
156 
157  // first let's get a list of all the headings, so "metriccheck1", "metriccheck2" in this case
158  // I've currently got this enforcing those names strictly to make sure formatting is correct
159  // But really the headings could just be any unique names and are arbitrary
160  int headingIndex = 1;
161 
162  for (auto iterateArbitraryHeadingNames = metricsParameters.begin();
163  iterateArbitraryHeadingNames != metricsParameters.end();
164  ++iterateArbitraryHeadingNames) {
165  auto headingName = metricsParameters.name(iterateArbitraryHeadingNames);
166 
167  // we could be flexible on these headers but for now let's enforce it to get any convention inconsistencies cleaned up
168  std::string expectedHeadingName = "metriccheck" + std::to_string(headingIndex);
169  if( expectedHeadingName != headingName) {
170  throw std::logic_error( "The parameter list expected to find a heading with name '" + expectedHeadingName + "' but instead found '" + headingName );
171  }
172 
173  // get the parameters specific to the check we want to run
174  const ParameterList & metricCheckParameters = metricsParameters.sublist(headingName);
175 
176  MetricAnalyzerInfo metricInfo = getMetricInfo(metricCheckParameters, metricObject);
177  metricInfoSet.push_back(metricInfo);
178  ++headingIndex;
179  }
180  }
181 
182 private:
183 
184  static zscalar_t convertParameterChoicesToEvaluatePartitionAPICall(
185  const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject,
186  std::string keyWord,
187  int weightIndex,
188  int selectedNormedSetting )
189  {
190  // this is going to need some consideration - how is the adapter scalar_t type to be properly handled?
191  zscalar_t theValue = 0;
192 
193  if( weightIndex != UNDEFINED_PARAMETER_INT_INDEX && selectedNormedSetting != UNDEFINED_PARAMETER_INT_INDEX ) {
194  throw std::logic_error( "Both parameters 'normed' and 'weight' were specified. They should never appear together." );
195  }
196 
197  // this may not always be true - will have to find out how normed may be used late
198  if( keyWord != API_STRING_getWeightImbalance && selectedNormedSetting != UNDEFINED_PARAMETER_INT_INDEX ) {
199  throw std::logic_error( "'normed' was specified but this only has meaning for the 'imbalance' parameter." );
200  }
201 
202  // Here I am enforcing a parallel usage to the way API calls exist in EvaluatePartition
203  if (keyWord == API_STRING_getWeightImbalance) {
204  if( weightIndex == UNDEFINED_PARAMETER_INT_INDEX ) { // -1 is the code for optional (meaning it was not specified)
205  if( selectedNormedSetting == 1 ) {
206  theValue = metricObject->getNormedImbalance();
207  }
208  else {
209  theValue = metricObject->getObjectCountImbalance(); // this will be index
210  }
211  }
212  else {
213  theValue = metricObject->getWeightImbalance(weightIndex); // this will get the proper index specified
214  }
215  }
216  else if (keyWord == API_STRING_getTotalEdgeCuts) {
217  if( weightIndex == UNDEFINED_PARAMETER_INT_INDEX ) {
218  theValue = metricObject->getTotalEdgeCut();
219  }
220  else {
221  theValue = metricObject->getTotalWeightEdgeCut(weightIndex);
222  }
223  }
224  else if (keyWord == API_STRING_getMaxEdgeCuts) {
225  if( weightIndex == UNDEFINED_PARAMETER_INT_INDEX ) {
226  theValue = metricObject->getMaxEdgeCut();
227  }
228  else {
229  theValue = metricObject->getMaxWeightEdgeCut(weightIndex);
230  }
231  }
232  else {
233  // we have found an invalid key word - throw an error
234  throw std::logic_error( "The parameter '" + std::string(KEYWORD_PARAMETER_NAME) + "' was specified as '" + keyWord + "' which is not understood." );
235  }
236 
237  return theValue;
238  }
239 
240  static bool executeMetricCheck(
241  const MetricAnalyzerInfo & metricInfo,
242  std::ostringstream &msg_stream)
243  {
244  bool bDoesThisTestPass = true; // will set this false if a test fails
245  if (metricInfo.bFoundUpperBound && metricInfo.bFoundLowerBound) {
246  if (metricInfo.theValue < metricInfo.lowerValue ||
247  metricInfo.theValue > metricInfo.upperValue) {
248  msg_stream << "FAILED: " << metricInfo.parameterDescription
249  << " value: " << metricInfo.theValue << " is not in range: " << metricInfo.lowerValue << " to "
250  << metricInfo.upperValue << std::endl;
251  bDoesThisTestPass = false;
252  }
253  else {
254  msg_stream << "Success: " << metricInfo.parameterDescription
255  << " value: " << metricInfo.theValue << " is in range: "
256  << metricInfo.lowerValue << " to "
257  << metricInfo.upperValue << std::endl;
258  }
259  }
260  else if (metricInfo.bFoundUpperBound) {
261  if (metricInfo.theValue > metricInfo.upperValue) {
262  msg_stream << "FAILED: " << metricInfo.parameterDescription
263  << " value: " << metricInfo.theValue << " is not below "
264  << metricInfo.upperValue << std::endl;
265  bDoesThisTestPass = false;
266  }
267  else {
268  msg_stream << "Success: " << metricInfo.parameterDescription
269  << " value: " << metricInfo.theValue << " is below: "
270  << metricInfo.upperValue << std::endl;
271  }
272  }
273  else if (metricInfo.bFoundLowerBound) {
274  if (metricInfo.theValue < metricInfo.lowerValue) {
275  msg_stream << "FAILED: " << metricInfo.parameterDescription
276  << " value: " << metricInfo.theValue << " is not above "
277  << metricInfo.lowerValue << std::endl;
278  bDoesThisTestPass = false;
279  }
280  else {
281  msg_stream << "Success: " << metricInfo.parameterDescription
282  << " value: " << metricInfo.theValue << " is above: "
283  << metricInfo.lowerValue << std::endl;
284  }
285  }
286  return bDoesThisTestPass;
287  }
288 
289  static MetricAnalyzerInfo getMetricInfo(
290  const ParameterList & metricCheckParameters,
291  const RCP<const Zoltan2::EvaluatePartition <basic_id_t> > &metricObject)
292  {
293  MetricAnalyzerInfo result; // will fill these values
294  for (auto iterateAllKeys = metricCheckParameters.begin();
295  iterateAllKeys != metricCheckParameters.end(); ++iterateAllKeys) {
296  auto checkName = metricCheckParameters.name(iterateAllKeys);
297  if ( checkName != WEIGHT_PARAMETER_NAME &&
298  checkName != KEYWORD_PARAMETER_NAME &&
299  checkName != UPPER_PARAMETER_NAME &&
300  checkName != LOWER_PARAMETER_NAME &&
301  checkName != NORMED_PARAMETER_NAME ) {
302  throw std::logic_error( "Key name: '" + checkName + "' is not understood." );
303  }
304  }
305 
306  // pick up the weight index - this parameter is optional so we check it first - this way we can communicate with EvaluatePartition properly in the next step
307  int selectedWeightIndex = UNDEFINED_PARAMETER_INT_INDEX; // meaning not specified so default case
308  if( metricCheckParameters.isParameter(WEIGHT_PARAMETER_NAME)) {
309  selectedWeightIndex = metricCheckParameters.get<int>(WEIGHT_PARAMETER_NAME);
310  if( selectedWeightIndex < 0 ) {
311  throw std::logic_error( "Optional weight index was specified as: " + std::to_string(selectedWeightIndex) + " Weight index must be 0 or positive." ); // I think that's the best I can do for error checking weight index right now - we may want to specify the cap when we know it
312  }
313  }
314 
315  // pick up the norm index - this parameter is optional so we check it first - this way we can communicate with EvaluatePartition properly in the next step
316  int selectedNormedSetting = UNDEFINED_PARAMETER_INT_INDEX; // meaning not specified so default case
317  if( metricCheckParameters.isParameter(NORMED_PARAMETER_NAME)) {
318  bool bNormSetting = metricCheckParameters.get<bool>(NORMED_PARAMETER_NAME);
319  selectedNormedSetting = bNormSetting ? 1 : 0;
320  if( selectedNormedSetting != 0 && selectedNormedSetting != 1 ) {
321  throw std::logic_error( "Optional normed parameter was specified as: " + std::to_string(selectedNormedSetting) + " Normed parameter must be true or false." );
322  }
323  }
324 
325  // one of the parameters called "check" should define a string which is a keyword which correlates to an EvaluatePartition API all
326  // this area needs some consideration - how and where shall we map and define the naming conventions
327  if( metricCheckParameters.isParameter(KEYWORD_PARAMETER_NAME)) {
328  std::string theKeyWord = metricCheckParameters.get<std::string>(KEYWORD_PARAMETER_NAME);
329 
330  // this is going to need some consideration - how is the adapter scalar_t type to be properly handled?
331  result.theValue = convertParameterChoicesToEvaluatePartitionAPICall(metricObject, theKeyWord, selectedWeightIndex, selectedNormedSetting );
332 
333  // now we can obtain the upper and lower bounds for this test
334  result.bFoundUpperBound = metricCheckParameters.isParameter(UPPER_PARAMETER_NAME);
335  result.bFoundLowerBound = metricCheckParameters.isParameter(LOWER_PARAMETER_NAME);
336 
337 
338  if (result.bFoundUpperBound) {
339  result.upperValue = metricCheckParameters.get<double>(UPPER_PARAMETER_NAME);
340  }
341  if (result.bFoundLowerBound) {
342  result.lowerValue = metricCheckParameters.get<double>(LOWER_PARAMETER_NAME);
343  }
344 
345  result.parameterDescription = theKeyWord;
346  if( selectedWeightIndex != UNDEFINED_PARAMETER_INT_INDEX ) {
347  result.parameterDescription = result.parameterDescription + " (weight: " + std::to_string(selectedWeightIndex) + ")";
348  }
349  else if( selectedNormedSetting != UNDEFINED_PARAMETER_INT_INDEX ) { // throw above would catch the case where both of these were set
350  result.parameterDescription = result.parameterDescription + " (normed: " + ( ( selectedNormedSetting == 0 ) ? "false" : "true" ) + ")";
351  }
352  }
353  return result;
354  }
355 };
356 
357 
358 #endif //ZOLTAN2_METRIC_ANALYZER_HPP
keep typedefs that commonly appear in many places localized
#define API_STRING_getMaxEdgeCuts
#define UPPER_PARAMETER_NAME
double zscalar_t
common code used by tests
#define API_STRING_getTotalEdgeCuts
#define LOWER_PARAMETER_NAME
static void LoadMetricInfo(std::vector< MetricAnalyzerInfo > &metricInfoSet, const RCP< const Zoltan2::EvaluatePartition< basic_id_t > > &metricObject, const ParameterList &metricsParameters)
#define UNDEFINED_PARAMETER_INT_INDEX
#define NORMED_PARAMETER_NAME
#define API_STRING_getWeightImbalance
static bool analyzeMetrics(const RCP< const Zoltan2::EvaluatePartition< basic_id_t > > &metricObject, const ParameterList &metricsParameters, std::ostringstream &msg_stream)
Analyze metrics for a problem based on a range of tolerances.
#define KEYWORD_PARAMETER_NAME
A class that computes and returns quality metrics.
#define WEIGHT_PARAMETER_NAME