Zoltan2
Zoltan2_MachineTopoMgr.hpp
Go to the documentation of this file.
1 #ifndef _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_
2 #define _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_
3 
4 #include <Teuchos_Comm.hpp>
5 #include <Teuchos_CommHelpers.hpp>
6 #include <Zoltan2_Machine.hpp>
7 
8 #ifdef HAVE_ZOLTAN2_TOPOMANAGER
9 #include <TopoManager.h>
10 #endif
11 
12 namespace Zoltan2{
13 
18 template <typename pcoord_t, typename part_t>
19 class MachineTopoMgr : public Machine <pcoord_t, part_t> {
20 
21 public:
26  MachineTopoMgr(const Teuchos::Comm<int> &comm):
27  Machine<pcoord_t,part_t>(comm),
28 #if defined (CMK_BLUEGENEQ)
29  networkDim(6), tmgr(comm.getSize()),
30 #elif defined (CMK_BLUEGENEP)
31  networkDim(4), tmgr(comm.getSize()),
32 #else
33  networkDim(3),
34 #endif
35  procCoords(NULL), machine_extent(NULL),
36  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL), pl(NULL)
37  {
38  transformed_network_dim = networkDim - 1;
39  transformed_coordinates = procCoords;
40  machine_extent = new int[networkDim];
41  this->getMachineExtent(this->machine_extent);
42  //allocate memory for processor coordinates.
43  procCoords = new pcoord_t *[networkDim];
44  for (int i = 0; i < networkDim; ++i){
45  procCoords[i] = new pcoord_t[this->numRanks];
46  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
47  }
48 
49  //obtain the coordinate of the processor.
50  pcoord_t *xyz = new pcoord_t[networkDim];
52  for (int i = 0; i < networkDim; i++)
53  procCoords[i][this->myRank] = xyz[i];
54  delete [] xyz;
55 
56  //reduceAll the coordinates of each processor.
57  gatherMachineCoordinates(comm);
58 
59  }
60 
61  MachineTopoMgr(const Teuchos::Comm<int> &comm, const Teuchos::ParameterList &pl_ ):
62  Machine<pcoord_t,part_t>(comm),
63 #if defined (CMK_BLUEGENEQ)
64  networkDim(6), tmgr(comm.getSize()),
65 #elif defined (CMK_BLUEGENEP)
66  networkDim(4), tmgr(comm.getSize()),
67 #else
68  networkDim(3),
69 #endif
70  procCoords(NULL), machine_extent(NULL),
71  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL),
72  pl(&pl_)
73  {
74  transformed_network_dim = networkDim - 1;
75  transformed_coordinates = procCoords;
76  machine_extent = new int[networkDim];
77  this->getMachineExtent(this->machine_extent);
78  //allocate memory for processor coordinates.
79  procCoords = new pcoord_t *[networkDim];
80  for (int i = 0; i < networkDim; ++i){
81  procCoords[i] = new pcoord_t[this->numRanks];
82  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
83  }
84 
85  //obtain the coordinate of the processor.
86  pcoord_t *xyz = new pcoord_t[networkDim];
88  for (int i = 0; i < networkDim; i++)
89  procCoords[i][this->myRank] = xyz[i];
90  delete [] xyz;
91 
92  //reduceAll the coordinates of each processor.
93  gatherMachineCoordinates(comm);
94 
95  const Teuchos::ParameterEntry *pe = this->pl->getEntryPtr("machine_coord_transformation");
96 
97  if (pe){
98 
99  std::string approach;
100  approach = pe->getValue<std::string>(&approach);
101 
102  if (approach == "Node"){
103  transformed_network_dim = networkDim - 1;
104  transformed_coordinates = procCoords;
105  }
106 
107  else if (approach == "EIGNORE"){
108  if (this->myRank == 0) std::cout << "Ignoring E Dimension" << std::endl;
109  transformed_network_dim = networkDim - 2;
110  transformed_coordinates = procCoords;
111  }
112  }
113  }
114 
115  virtual ~MachineTopoMgr() {
116  for (int i = 0; i < networkDim; i++){
117  delete [] procCoords[i];
118  }
119  delete [] procCoords;
120  delete [] machine_extent;
121 
122  if (delete_transformed_coords){
123  for (int i = 0; i < transformed_network_dim; i++){
124  delete [] transformed_coordinates[i];
125  }
126  delete [] transformed_coordinates;
127  }
128 
129  }
130 
131  bool hasMachineCoordinates() const { return true; }
132 
133  int getMachineDim() const { return transformed_network_dim; }
134 
135  bool getMachineExtent(int *nxyz) const {
136 #if defined (CMK_BLUEGENEQ)
137  int dim = 0;
138  if (dim < transformed_network_dim)
139  nxyz[dim++] = tmgr.getDimNA();
140  if (dim < transformed_network_dim)
141  nxyz[dim++] = tmgr.getDimNB();
142  if (dim < transformed_network_dim)
143  nxyz[dim++] = tmgr.getDimNC();
144  if (dim < transformed_network_dim)
145  nxyz[dim++] = tmgr.getDimND();
146  if (dim < transformed_network_dim)
147  nxyz[dim++] = tmgr.getDimNE();
148  if (dim < transformed_network_dim)
149  nxyz[dim++] = tmgr.getDimNT();
150  return true;
151 #elif defined (CMK_BLUEGENEP)
152  int dim = 0;
153  if (dim < transformed_network_dim)
154  nxyz[dim++] = tmgr.getDimNX();
155  if (dim < transformed_network_dim)
156  nxyz[dim++] = tmgr.getDimNY();
157  if (dim < transformed_network_dim)
158  nxyz[dim++] = tmgr.getDimNZ();
159  if (dim < transformed_network_dim)
160  nxyz[dim++] = tmgr.getDimNT();
161  return true;
162 #else
163  return false;
164 #endif
165  }
166 
167  //MD TODO: Not always it has wrap-around links.
168  bool getMachineExtentWrapArounds(bool *wrap_around) const {
169 #if defined (CMK_BLUEGENEQ)
170  //leave it as this for now, figure out if there is a way to determine tourus from topomanager.
171  int dim = 0;
172  if (dim < transformed_network_dim)
173  wrap_around[dim++] = true;
174  if (dim < transformed_network_dim)
175  wrap_around[dim++] = true;
176  if (dim < transformed_network_dim)
177  wrap_around[dim++] = true;
178  if (dim < transformed_network_dim)
179  wrap_around[dim++] = true;
180  if (dim < transformed_network_dim)
181  wrap_around[dim++] = true;
182  if (dim < transformed_network_dim)
183  wrap_around[dim++] = true;
184 #elif defined (CMK_BLUEGENEP)
185  int dim = 0;
186  if (dim < transformed_network_dim)
187  wrap_around[dim++] = true;
188  if (dim < transformed_network_dim)
189  wrap_around[dim++] = true;
190  if (dim < transformed_network_dim)
191  wrap_around[dim++] = true;
192  if (dim < transformed_network_dim)
193  wrap_around[dim++] = true;
194 #else
195 #endif
196  return true;
197  }
198 
199  bool getMyMachineCoordinate(pcoord_t *xyz) {
200  for (int i = 0; i < this->transformed_network_dim; ++i){
201  xyz[i] = transformed_coordinates[i][this->myRank];
202  }
203  return true;
204  }
205 
206  bool getMyActualMachineCoordinate(pcoord_t *xyz) {
207 #if defined (CMK_BLUEGENEQ)
208  int a,b,c,d,e,t;
209  tmgr.rankToCoordinates(this->myRank, a,b,c,d,e,t);
210  xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = d; xyz[4] = e; xyz[5] = t;
211  //std::cout << "me:" << this->myRank << " " << a << " " << b << " " << c << " " << d << " " << e << " " << t << std::endl;
212  return true;
213 #elif defined (CMK_BLUEGENEP)
214  int a,b,c,t;
215  tmgr.rankToCoordinates(this->myRank, a,b,c,t);
216  xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = t;
217  return true;
218 #else
219  return false;
220 #endif
221  }
222 
223  bool getMachineExtentWrapArounds(part_t *wrap_around) const {
224 
225  int dim = 0;
226  if (dim < transformed_network_dim)
227  wrap_around[dim++] = true;
228 
229  if (dim < transformed_network_dim)
230  wrap_around[dim++] = true;
231 
232  if (dim < transformed_network_dim)
233  wrap_around[dim++] = true;
234 
235  if (dim < transformed_network_dim)
236  wrap_around[dim++] = true;
237 
238  if (dim < transformed_network_dim)
239  wrap_around[dim++] = true;
240 
241  if (dim < transformed_network_dim)
242  wrap_around[dim++] = true;
243  return true;
244  }
245  inline bool getMachineCoordinate(const int rank,
246  pcoord_t *xyz) const {
247  return false;
248  }
249 
250 
251  bool getMachineCoordinate(const char *nodename, pcoord_t *xyz) {
252  return false; // cannot yet return from nodename
253  }
254 
255  bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const {
256  allCoords = procCoords;
257  return true;
258  }
259 
260  virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops){
261  hops = 0;
262  for (int i = 0; i < networkDim - 1; ++i){
263  pcoord_t distance = procCoords[i][rank1] - procCoords[i][rank2];
264  if (distance < 0 ) distance = -distance;
265  if (machine_extent[i] - distance < distance) distance = machine_extent[i] - distance;
266  hops += distance;
267  }
268  return true;
269  }
270 
271 
272 private:
273 
274  int networkDim;
275 
276 #ifdef HAVE_ZOLTAN2_TOPOMANAGER
277  TopoManager tmgr;
278 #endif
279  pcoord_t **procCoords; // KDD Maybe should be RCP?
280  part_t *machine_extent;
281  const Teuchos::ParameterList *pl;
282 
283 
284  bool delete_transformed_coords;
285  int transformed_network_dim;
286  pcoord_t **transformed_coordinates;
287 
288  void gatherMachineCoordinates(const Teuchos::Comm<int> &comm) {
289  // reduces and stores all machine coordinates.
290  pcoord_t *tmpVect = new pcoord_t [this->numRanks];
291 
292  for (int i = 0; i < networkDim; i++) {
293  Teuchos::reduceAll<int, pcoord_t>(comm, Teuchos::REDUCE_SUM,
294  this->numRanks, procCoords[i], tmpVect);
295  pcoord_t *tmp = tmpVect;
296  tmpVect = procCoords[i];
297  procCoords[i] = tmp;
298  }
299  delete [] tmpVect;
300  }
301 };
302 }
303 #endif
MachineTopoMgr(const Teuchos::Comm< int > &comm)
Constructor: A BlueGeneQ network machine description;.
MachineClass Base class for representing machine coordinates, networks, etc.
bool getMyActualMachineCoordinate(pcoord_t *xyz)
bool getMyMachineCoordinate(pcoord_t *xyz)
bool getMachineExtentWrapArounds(part_t *wrap_around) const
MachineTopoMgr(const Teuchos::Comm< int > &comm, const Teuchos::ParameterList &pl_)
A Machine Class for testing only A more realistic machine should be used for task mapping...
virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops)
bool getMachineCoordinate(const char *nodename, pcoord_t *xyz)
bool getMachineExtentWrapArounds(bool *wrap_around) const
bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const
bool getMachineCoordinate(const int rank, pcoord_t *xyz) const
bool getMachineExtent(int *nxyz) const