1 #ifndef _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_ 2 #define _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_ 4 #include <Teuchos_Comm.hpp> 5 #include <Teuchos_CommHelpers.hpp> 8 #ifdef HAVE_ZOLTAN2_TOPOMANAGER 9 #include <TopoManager.h> 18 template <
typename pcoord_t,
typename part_t>
28 #if defined (CMK_BLUEGENEQ)
29 networkDim(6), tmgr(comm.getSize()),
30 #elif defined (CMK_BLUEGENEP)
31 networkDim(4), tmgr(comm.getSize()),
35 procCoords(NULL), machine_extent(NULL),
36 delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL), pl(NULL)
38 transformed_network_dim = networkDim - 1;
39 transformed_coordinates = procCoords;
40 machine_extent =
new int[networkDim];
43 procCoords =
new pcoord_t *[networkDim];
44 for (
int i = 0; i < networkDim; ++i){
45 procCoords[i] =
new pcoord_t[this->
numRanks];
46 memset(procCoords[i], 0,
sizeof(pcoord_t) * this->
numRanks);
50 pcoord_t *xyz =
new pcoord_t[networkDim];
52 for (
int i = 0; i < networkDim; i++)
53 procCoords[i][this->
myRank] = xyz[i];
57 gatherMachineCoordinates(comm);
61 MachineTopoMgr(
const Teuchos::Comm<int> &comm,
const Teuchos::ParameterList &pl_ ):
63 #if defined (CMK_BLUEGENEQ)
64 networkDim(6), tmgr(comm.getSize()),
65 #elif defined (CMK_BLUEGENEP)
66 networkDim(4), tmgr(comm.getSize()),
70 procCoords(NULL), machine_extent(NULL),
71 delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL),
74 transformed_network_dim = networkDim - 1;
75 transformed_coordinates = procCoords;
76 machine_extent =
new int[networkDim];
79 procCoords =
new pcoord_t *[networkDim];
80 for (
int i = 0; i < networkDim; ++i){
81 procCoords[i] =
new pcoord_t[this->
numRanks];
82 memset(procCoords[i], 0,
sizeof(pcoord_t) * this->
numRanks);
86 pcoord_t *xyz =
new pcoord_t[networkDim];
88 for (
int i = 0; i < networkDim; i++)
89 procCoords[i][this->
myRank] = xyz[i];
93 gatherMachineCoordinates(comm);
95 const Teuchos::ParameterEntry *pe = this->pl->getEntryPtr(
"machine_coord_transformation");
100 approach = pe->getValue<std::string>(&approach);
102 if (approach ==
"Node"){
103 transformed_network_dim = networkDim - 1;
104 transformed_coordinates = procCoords;
107 else if (approach ==
"EIGNORE"){
108 if (this->
myRank == 0) std::cout <<
"Ignoring E Dimension" << std::endl;
109 transformed_network_dim = networkDim - 2;
110 transformed_coordinates = procCoords;
116 for (
int i = 0; i < networkDim; i++){
117 delete [] procCoords[i];
119 delete [] procCoords;
120 delete [] machine_extent;
122 if (delete_transformed_coords){
123 for (
int i = 0; i < transformed_network_dim; i++){
124 delete [] transformed_coordinates[i];
126 delete [] transformed_coordinates;
136 #if defined (CMK_BLUEGENEQ) 138 if (dim < transformed_network_dim)
139 nxyz[dim++] = tmgr.getDimNA();
140 if (dim < transformed_network_dim)
141 nxyz[dim++] = tmgr.getDimNB();
142 if (dim < transformed_network_dim)
143 nxyz[dim++] = tmgr.getDimNC();
144 if (dim < transformed_network_dim)
145 nxyz[dim++] = tmgr.getDimND();
146 if (dim < transformed_network_dim)
147 nxyz[dim++] = tmgr.getDimNE();
148 if (dim < transformed_network_dim)
149 nxyz[dim++] = tmgr.getDimNT();
151 #elif defined (CMK_BLUEGENEP) 153 if (dim < transformed_network_dim)
154 nxyz[dim++] = tmgr.getDimNX();
155 if (dim < transformed_network_dim)
156 nxyz[dim++] = tmgr.getDimNY();
157 if (dim < transformed_network_dim)
158 nxyz[dim++] = tmgr.getDimNZ();
159 if (dim < transformed_network_dim)
160 nxyz[dim++] = tmgr.getDimNT();
169 #if defined (CMK_BLUEGENEQ) 172 if (dim < transformed_network_dim)
173 wrap_around[dim++] =
true;
174 if (dim < transformed_network_dim)
175 wrap_around[dim++] =
true;
176 if (dim < transformed_network_dim)
177 wrap_around[dim++] =
true;
178 if (dim < transformed_network_dim)
179 wrap_around[dim++] =
true;
180 if (dim < transformed_network_dim)
181 wrap_around[dim++] =
true;
182 if (dim < transformed_network_dim)
183 wrap_around[dim++] =
true;
184 #elif defined (CMK_BLUEGENEP) 186 if (dim < transformed_network_dim)
187 wrap_around[dim++] =
true;
188 if (dim < transformed_network_dim)
189 wrap_around[dim++] =
true;
190 if (dim < transformed_network_dim)
191 wrap_around[dim++] =
true;
192 if (dim < transformed_network_dim)
193 wrap_around[dim++] =
true;
200 for (
int i = 0; i < this->transformed_network_dim; ++i){
201 xyz[i] = transformed_coordinates[i][this->
myRank];
207 #if defined (CMK_BLUEGENEQ) 209 tmgr.rankToCoordinates(this->
myRank, a,b,c,d,e,t);
210 xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = d; xyz[4] = e; xyz[5] = t;
213 #elif defined (CMK_BLUEGENEP) 215 tmgr.rankToCoordinates(this->
myRank, a,b,c,t);
216 xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = t;
226 if (dim < transformed_network_dim)
227 wrap_around[dim++] =
true;
229 if (dim < transformed_network_dim)
230 wrap_around[dim++] =
true;
232 if (dim < transformed_network_dim)
233 wrap_around[dim++] =
true;
235 if (dim < transformed_network_dim)
236 wrap_around[dim++] =
true;
238 if (dim < transformed_network_dim)
239 wrap_around[dim++] =
true;
241 if (dim < transformed_network_dim)
242 wrap_around[dim++] =
true;
246 pcoord_t *xyz)
const {
256 allCoords = procCoords;
262 for (
int i = 0; i < networkDim - 1; ++i){
263 pcoord_t distance = procCoords[i][rank1] - procCoords[i][rank2];
264 if (distance < 0 ) distance = -distance;
265 if (machine_extent[i] - distance < distance) distance = machine_extent[i] - distance;
276 #ifdef HAVE_ZOLTAN2_TOPOMANAGER 279 pcoord_t **procCoords;
280 part_t *machine_extent;
281 const Teuchos::ParameterList *pl;
284 bool delete_transformed_coords;
285 int transformed_network_dim;
286 pcoord_t **transformed_coordinates;
288 void gatherMachineCoordinates(
const Teuchos::Comm<int> &comm) {
290 pcoord_t *tmpVect =
new pcoord_t [this->
numRanks];
292 for (
int i = 0; i < networkDim; i++) {
293 Teuchos::reduceAll<int, pcoord_t>(comm, Teuchos::REDUCE_SUM,
294 this->
numRanks, procCoords[i], tmpVect);
295 pcoord_t *tmp = tmpVect;
296 tmpVect = procCoords[i];
MachineTopoMgr(const Teuchos::Comm< int > &comm)
Constructor: A BlueGeneQ network machine description;.
MachineClass Base class for representing machine coordinates, networks, etc.
bool getMyActualMachineCoordinate(pcoord_t *xyz)
bool getMyMachineCoordinate(pcoord_t *xyz)
virtual ~MachineTopoMgr()
bool getMachineExtentWrapArounds(part_t *wrap_around) const
MachineTopoMgr(const Teuchos::Comm< int > &comm, const Teuchos::ParameterList &pl_)
A Machine Class for testing only A more realistic machine should be used for task mapping...
int getMachineDim() const
bool hasMachineCoordinates() const
virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops)
bool getMachineCoordinate(const char *nodename, pcoord_t *xyz)
bool getMachineExtentWrapArounds(bool *wrap_around) const
bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const
bool getMachineCoordinate(const int rank, pcoord_t *xyz) const
bool getMachineExtent(int *nxyz) const