MeshDistributor.h 23.9 KB
Newer Older
 Thomas Witkowski committed Apr 03, 2009 1 2 3 4 // ============================================================================ // == == // == AMDiS - Adaptive multidimensional simulations == // == == Thomas Witkowski committed Dec 16, 2010 5 // == http://www.amdis-fem.org == Thomas Witkowski committed Apr 03, 2009 6 7 // == == // ============================================================================ Thomas Witkowski committed Dec 16, 2010 8 9 10 11 12 13 14 15 16 17 18 19 // // Software License for AMDiS // // Copyright (c) 2010 Dresden University of Technology // All rights reserved. // Authors: Simon Vey, Thomas Witkowski et al. // // This file is part of AMDiS // // See also license.opensource.txt in the distribution. Thomas Witkowski committed Apr 03, 2009 20 21 /** \file MeshDistributor.h */ Thomas Witkowski committed Jul 09, 2009 22 23 24 #ifndef AMDIS_MESHDISTRIBUTOR_H #define AMDIS_MESHDISTRIBUTOR_H Thomas Witkowski committed Apr 03, 2009 25 26 Thomas Witkowski committed Feb 08, 2010 27 #include Thomas Witkowski committed Jan 17, 2012 28 #include "parallel/DofComm.h" Thomas Witkowski committed Feb 13, 2012 29 #include "parallel/ElementObjectDatabase.h" Thomas Witkowski committed Apr 21, 2011 30 #include "parallel/ParallelTypes.h" Thomas Witkowski committed Apr 11, 2012 31 #include "parallel/MeshLevelData.h" Thomas Witkowski committed Jan 24, 2011 32 #include "parallel/MeshPartitioner.h" Thomas Witkowski committed Nov 29, 2010 33 #include "parallel/InteriorBoundary.h" Thomas Witkowski committed Jan 26, 2012 34 #include "parallel/PeriodicMap.h" Thomas Witkowski committed Nov 08, 2011 35 #include "parallel/StdMpi.h" Thomas Witkowski committed Jan 27, 2011 36 #include "AMDiS_fwd.h" Thomas Witkowski committed Feb 09, 2012 37 #include "Containers.h" Thomas Witkowski committed Feb 08, 2010 38 #include "Global.h" Thomas Witkowski committed Apr 03, 2009 39 40 #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h" Thomas Witkowski committed Apr 15, 2009 41 #include "FiniteElemSpace.h" Thomas Witkowski committed Sep 29, 2009 42 #include "Serializer.h" Thomas Witkowski committed Feb 08, 2010 43 #include "BoundaryManager.h" Thomas Witkowski committed Jan 27, 2011 44 #include "SystemVector.h" Thomas Witkowski committed Jun 12, 2009 45 Thomas Witkowski committed Apr 03, 2009 46 namespace AMDiS { Thomas Witkowski committed Jan 07, 2011 47 48 using namespace std; Thomas Witkowski committed Apr 08, 2011 49 50 51 52 struct BoundaryDofInfo { Thomas Witkowski committed Apr 08, 2011 53 map geoDofs; Thomas Witkowski committed Apr 08, 2011 54 55 }; 56 57 58 59 60 61 62 63 64 65 66 67 struct DofData { /// Number of DOFs in the rank mesh. int nRankDofs; /// Is the index of the first global DOF index, which is owned by the rank. int rStartDofs; /// Number of DOFs in the whole domain. int nOverallDofs; 68 69 70 /// Maps all DOFs in ranks partition to a bool value. If it is true, the DOF /// is owned by the rank. Otherwise, its an interior boundary DOF that is /// owned by another rank. 71 72 73 DofIndexToBool isRankDof; /// Maps local to global dof indices. Thomas Witkowski committed Mar 15, 2012 74 DofMap mapDofToGlobal; 75 76 /// Maps local dof indices to real dof indices. Thomas Witkowski committed Mar 15, 2012 77 DofMap mapLocalToDof; 78 }; Thomas Witkowski committed Jan 17, 2012 79 80 Thomas Witkowski committed Jun 08, 2010 81 class MeshDistributor Thomas Witkowski committed Apr 03, 2009 82 { Thomas Witkowski committed May 19, 2011 83 private: Thomas Witkowski committed Mar 18, 2011 84 MeshDistributor(); Thomas Witkowski committed May 19, 2011 85 Thomas Witkowski committed Jun 08, 2010 86 virtual ~MeshDistributor() {} Thomas Witkowski committed Apr 03, 2009 87 Thomas Witkowski committed May 19, 2011 88 public: 89 void initParallelization(); Thomas Witkowski committed Apr 03, 2009 90 91 void exitParallelization(); Thomas Witkowski committed Jun 08, 2010 92 Thomas Witkowski committed Nov 14, 2011 93 94 95 /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector /// will be automatically interchanged between ranks when mesh is /// repartitioned. Thomas Witkowski committed Jan 19, 2011 96 97 98 99 100 void addInterchangeVector(DOFVector *vec) { interchangeVectors.push_back(vec); } Thomas Witkowski committed Jan 27, 2011 101 102 103 104 105 106 107 /// Adds all DOFVectors of a SystemVector to \ref interchangeVecs. void addInterchangeVector(SystemVector *vec) { for (int i = 0; i < vec->getSize(); i++) interchangeVectors.push_back(vec->getDOFVector(i)); } Thomas Witkowski committed Jan 29, 2010 108 /** \brief Thomas Witkowski committed Nov 14, 2011 109 110 111 112 113 * This function checks if the mesh has changed on at least on rank. In * this case, the interior boundaries are adapted on all ranks such that * they fit together on all ranks. Furthermore the function * \ref updateLocalGlobalNumbering() is called to update the DOF numberings * and mappings on all rank due to the new mesh structure. 114 * Thomas Witkowski committed Nov 14, 2011 115 116 117 118 119 * \param[in] tryRepartition If this parameter is true, repartitioning * may be done. This depends on several other * parameters. If the parameter is false, the * mesh is only checked and adapted but never * repartitioned. Thomas Witkowski committed Jan 29, 2010 120 */ 121 void checkMeshChange(bool tryRepartition = true); Thomas Witkowski committed Jan 29, 2010 122 Thomas Witkowski committed Jan 02, 2012 123 124 125 126 127 128 129 130 131 132 /** \brief * Checks if is required to repartition the mesh. If this is the case, a new * partition will be created and the mesh will be redistributed between the * ranks. */ void repartitionMesh(); /// Calculates the imbalancing factor and prints it to screen. void printImbalanceFactor(); Thomas Witkowski committed Jun 26, 2009 133 /** \brief Thomas Witkowski committed Nov 14, 2011 134 135 136 137 * Test, if the mesh consists of macro elements only. The mesh partitioning * of the parallelization works for macro meshes only and would fail, if the * mesh is already refined in some way. Therefore, this function will exit * the program if it finds a non macro element in the mesh. Thomas Witkowski committed Jun 26, 2009 138 139 */ void testForMacroMesh(); Thomas Witkowski committed Apr 03, 2009 140 Thomas Witkowski committed Nov 14, 2011 141 142 /// Set for each element on the partitioning level the number of /// leaf elements. 143 void setInitialElementWeights(); Thomas Witkowski committed Apr 03, 2009 144 Thomas Witkowski committed Jan 07, 2011 145 inline virtual string getName() Thomas Witkowski committed Jun 08, 2010 146 147 148 { return name; } 149 Thomas Witkowski committed Jan 14, 2011 150 151 152 153 154 inline Mesh* getMesh() { return mesh; } 155 156 /// Returns an FE space from \ref feSpaces. inline const FiniteElemSpace* getFeSpace(unsigned int i = 0) 157 { 158 159 160 161 162 FUNCNAME("MeshDistributor::getFeSpace()"); TEST_EXIT_DBG(i < feSpaces.size())("Should not happen!\n"); return feSpaces[i]; 163 } 164 165 166 167 168 169 170 /// Returns all FE spaces, thus \ref feSpaces. inline vector& getFeSpaces() { return feSpaces; } Thomas Witkowski committed Apr 18, 2012 171 172 /// Returns the number of DOFs in rank's domain for a given FE space. inline int getNumberRankDofs(const FiniteElemSpace *feSpace) Thomas Witkowski committed Jul 10, 2009 173 { Thomas Witkowski committed Jan 17, 2012 174 175 176 177 FUNCNAME("MeshDistributor::getNumberRankDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n"); 178 return dofFeData[feSpace].nRankDofs; Thomas Witkowski committed Jul 10, 2009 179 } Thomas Witkowski committed Apr 03, 2009 180 Thomas Witkowski committed Jan 18, 2012 181 182 183 184 185 186 187 188 189 190 191 192 193 /// Returns the number of DOFs in rank's domain for a set of FE spaces. inline int getNumberRankDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getNumberRankDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].nRankDofs; } return result; } Thomas Witkowski committed Feb 06, 2012 194 Thomas Witkowski committed Jan 18, 2012 195 /// Returns the first global DOF index of an FE space, owned by rank. 196 inline int getStartDofs(const FiniteElemSpace *feSpace) Thomas Witkowski committed Jan 03, 2012 197 { Thomas Witkowski committed Jan 17, 2012 198 199 200 201 FUNCNAME("MeshDistributor::getStartDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n"); 202 return dofFeData[feSpace].rStartDofs; Thomas Witkowski committed Jan 03, 2012 203 204 } Thomas Witkowski committed Jan 18, 2012 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 /// Returns the first global DOF index for a set of FE spaces, owned by rank. inline int getStartDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getStartDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].rStartDofs; } return result; } Thomas Witkowski committed Apr 18, 2012 220 221 /// Returns the global number of DOFs for a given FE space. inline int getNumberOverallDofs(const FiniteElemSpace *feSpace) Thomas Witkowski committed Jul 10, 2009 222 { Thomas Witkowski committed Jan 17, 2012 223 224 225 226 FUNCNAME("MeshDistributor::getNumberOverallDofs()"); TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n"); 227 return dofFeData[feSpace].nOverallDofs; Thomas Witkowski committed Jul 10, 2009 228 } Thomas Witkowski committed Apr 03, 2009 229 Thomas Witkowski committed Jan 18, 2012 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 /// Returns the global number of DOFs for a set of FE spaces. inline int getNumberOverallDofs(vector& feSpaces) { FUNCNAME("MeshDistributor::getNumberOverallDofs()"); int result = 0; for (unsigned int i = 0; i < feSpaces.size(); i++) { TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n"); result += dofFeData[feSpaces[i]].nOverallDofs; } return result; } Thomas Witkowski committed Mar 15, 2012 245 inline DofMap& getMapDofToGlobal(const FiniteElemSpace *feSpace) Thomas Witkowski committed Jan 14, 2011 246 { 247 FUNCNAME("MeshDistributor::getMapDofToGlobal()"); Thomas Witkowski committed Jan 17, 2012 248 249 250 TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n"); 251 return dofFeData[feSpace].mapDofToGlobal; Thomas Witkowski committed Jan 14, 2011 252 253 } Thomas Witkowski committed Mar 18, 2011 254 /// Maps a local DOF to its global index. 255 256 inline DegreeOfFreedom mapDofToGlobal(const FiniteElemSpace *feSpace, DegreeOfFreedom dof) Thomas Witkowski committed Jul 10, 2009 257 { 258 FUNCNAME("MeshDistributor::mapDofToGlobal()"); Thomas Witkowski committed Jan 17, 2012 259 260 261 262 TEST_EXIT_DBG(dofFeData.count(feSpace)) ("No DOF data for FE space at addr %p!\n", feSpace); 263 return dofFeData[feSpace].mapDofToGlobal[dof]; Thomas Witkowski committed Jul 10, 2009 264 } Thomas Witkowski committed Apr 03, 2009 265 266 267 268 269 /// Returns for a global index the DOF index in rank's subdomain. As there /// is no direct data structure that stores this information, we have to /// search for it in \ref dofFeData.mapDofToGlobal. This is not very /// efficient and this function should thus be used for debugging only. 270 271 DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace, DegreeOfFreedom dof); Thomas Witkowski committed Jan 17, 2011 272 Thomas Witkowski committed Mar 18, 2011 273 /// Maps a local DOF to its local index. 274 275 inline DegreeOfFreedom mapLocalToDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof) Thomas Witkowski committed Jun 09, 2010 276 { 277 FUNCNAME("MeshDistributor::mapLocalToDof()"); Thomas Witkowski committed Jan 17, 2012 278 279 280 281 TEST_EXIT_DBG(dofFeData.count(feSpace)) ("No DOF data for FE space at addr %p!\n", feSpace); 282 return dofFeData[feSpace].mapLocalToDof[dof]; Thomas Witkowski committed Jun 09, 2010 283 284 } Thomas Witkowski committed Jan 26, 2012 285 286 /// Returns the periodic mapping handler, \ref periodicMap. inline PeriodicMap& getPeriodicMap() Thomas Witkowski committed Jan 14, 2011 287 { Thomas Witkowski committed Jan 26, 2012 288 return periodicMap; Thomas Witkowski committed Apr 07, 2009 289 290 } Thomas Witkowski committed Jan 17, 2012 291 DofComm& getSendDofs() 292 293 294 295 { return sendDofs; } Thomas Witkowski committed Jan 17, 2012 296 DofComm& getRecvDofs() 297 298 299 300 { return recvDofs; } 301 302 303 304 305 DofComm& getPeriodicDofs() { return periodicDofs; } Thomas Witkowski committed Nov 14, 2011 306 307 /// Return true, if the given DOF is owned by the rank. If false, the DOF /// is in rank's partition, but is owned by some other rank. 308 inline bool getIsRankDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof) Thomas Witkowski committed Jun 08, 2010 309 { 310 311 if (dofFeData[feSpace].isRankDof.count(dof)) return dofFeData[feSpace].isRankDof[dof]; Thomas Witkowski committed Jan 18, 2011 312 313 return false; Thomas Witkowski committed Jun 08, 2010 314 } Thomas Witkowski committed Dec 08, 2009 315 316 inline DofIndexToBool& getIsRankDof(const FiniteElemSpace *feSpace) Thomas Witkowski committed Mar 17, 2011 317 { 318 return dofFeData[feSpace].isRankDof; Thomas Witkowski committed Mar 17, 2011 319 320 } Thomas Witkowski committed Jun 08, 2010 321 inline long getLastMeshChangeIndex() Thomas Witkowski committed Jun 10, 2009 322 { Thomas Witkowski committed Jun 08, 2010 323 return lastMeshChangeIndex; Thomas Witkowski committed Apr 07, 2009 324 } Thomas Witkowski committed Apr 03, 2009 325 Thomas Witkowski committed Jun 08, 2010 326 inline int getMpiRank() Thomas Witkowski committed Jun 11, 2009 327 { Thomas Witkowski committed Jun 08, 2010 328 return mpiRank; Thomas Witkowski committed Jun 11, 2009 329 } Thomas Witkowski committed Apr 03, 2009 330 Thomas Witkowski committed Jan 14, 2011 331 332 333 334 335 inline int getMpiSize() { return mpiSize; } Thomas Witkowski committed Jun 08, 2010 336 337 338 inline MPI::Intracomm& getMpiComm() { return mpiComm; Thomas Witkowski committed Apr 03, 2009 339 340 } 341 342 343 344 345 inline bool isInitialized() { return initialized; } Thomas Witkowski committed Sep 28, 2009 346 // Writes all data of this object to an output stream. Thomas Witkowski committed Jan 07, 2011 347 void serialize(ostream &out); Thomas Witkowski committed Jun 11, 2009 348 Thomas Witkowski committed Sep 28, 2009 349 // Reads the object data from an input stream. Thomas Witkowski committed Jan 07, 2011 350 void deserialize(istream &in); Thomas Witkowski committed Jun 08, 2010 351 352 /** \brief Thomas Witkowski committed Nov 14, 2011 353 354 355 356 * This function must be used if the values of a DOFVector must be * synchronised over all ranks. That means, that each rank sends the * values of the DOFs, which are owned by the rank and lie on an interior * bounday, to all other ranks also having these DOFs. Thomas Witkowski committed Jun 08, 2010 357 * Thomas Witkowski committed Nov 14, 2011 358 359 360 * This function must be used, for example, after the lineary system is * solved, or after the DOFVector is set by some user defined functions, * e.g., initial solution functions. Thomas Witkowski committed Jun 08, 2010 361 */ Thomas Witkowski committed Nov 08, 2011 362 363 364 365 366 template void synchVector(DOFVector &vec) { StdMpi > stdMpi(mpiComm); 367 368 const FiniteElemSpace *fe = vec.getFeSpace(); Thomas Witkowski committed Jan 17, 2012 369 for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) { Thomas Witkowski committed Nov 08, 2011 370 vector dofs; Thomas Witkowski committed Jan 17, 2012 371 dofs.reserve(it.getDofs().size()); Thomas Witkowski committed Nov 08, 2011 372 Thomas Witkowski committed Jan 17, 2012 373 374 for (; !it.endDofIter(); it.nextDof()) dofs.push_back(vec[it.getDofIndex()]); Thomas Witkowski committed Nov 08, 2011 375 Thomas Witkowski committed Jan 17, 2012 376 stdMpi.send(it.getRank(), dofs); Thomas Witkowski committed Nov 08, 2011 377 } Thomas Witkowski committed Jan 17, 2012 378 379 380 381 for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) stdMpi.recv(it.getRank()); Thomas Witkowski committed Nov 08, 2011 382 stdMpi.startCommunication(); Thomas Witkowski committed Jan 17, 2012 383 384 385 386 387 for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank()) for (; !it.endDofIter(); it.nextDof()) vec[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[it.getDofCounter()]; Thomas Witkowski committed Nov 08, 2011 388 389 } Thomas Witkowski committed Jun 08, 2010 390 /** \brief Thomas Witkowski committed Nov 14, 2011 391 392 393 * Works in the same way as the function above defined for DOFVectors. Due * to performance, this function does not call \ref synchVector for each * DOFVector, but instead sends all values of all DOFVectors all at once. Thomas Witkowski committed Jun 08, 2010 394 395 396 */ void synchVector(SystemVector &vec); Thomas Witkowski committed Jan 20, 2011 397 398 void check3dValidMesh(); Thomas Witkowski committed Apr 08, 2011 399 400 401 402 403 void setBoundaryDofRequirement(Flag flag) { createBoundaryDofFlag = flag; } 404 405 BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, int level = 0) Thomas Witkowski committed Apr 08, 2011 406 { Thomas Witkowski committed Apr 16, 2012 407 408 409 410 411 412 413 FUNCNAME("MeshDistributor::getBoundaryDofInfo()"); TEST_EXIT_DBG(level < static_cast(boundaryDofInfo.size())) ("Wrong level number: %d, whereas array size is %d!\n", level, boundaryDofInfo.size()); return boundaryDofInfo[level][feSpace]; Thomas Witkowski committed Apr 08, 2011 414 415 } 416 417 void getAllBoundaryDofs(const FiniteElemSpace *feSpace, int level, 418 DofContainer& dofs); Thomas Witkowski committed Apr 21, 2011 419 420 421 422 423 const ElementObjectDatabase& getElementObjectDb() { return elObjDb; } Thomas Witkowski committed May 19, 2011 424 425 426 /// Adds a stationary problem to the global mesh distributor objects. static void addProblemStatGlobal(ProblemStatSeq *probStat); Thomas Witkowski committed Apr 13, 2012 427 428 429 430 431 432 MeshLevelData& getMeshLevelData() { return levelData; } Thomas Witkowski committed Jun 11, 2009 433 protected: Thomas Witkowski committed May 19, 2011 434 435 void addProblemStat(ProblemStatSeq *probStat); Thomas Witkowski committed Nov 14, 2011 436 437 /// Determines the interior boundaries, i.e. boundaries between ranks, and /// stores all information about them in \ref interiorBoundary. Thomas Witkowski committed Oct 19, 2009 438 void createInteriorBoundaryInfo(); Thomas Witkowski committed Jun 11, 2009 439 Thomas Witkowski committed Nov 29, 2010 440 441 442 443 444 void updateInteriorBoundaryInfo(); void createMeshElementData(); void createBoundaryData(); Thomas Witkowski committed Jan 26, 2010 445 Thomas Witkowski committed Apr 08, 2011 446 447 void createBoundaryDofs(); Thomas Witkowski committed Apr 16, 2012 448 void createBoundaryDofs(const FiniteElemSpace *feSpace, int level); 449 Thomas Witkowski committed Nov 14, 2011 450 451 /// Removes all macro elements from the mesh that are not part of ranks /// partition. Thomas Witkowski committed Jun 11, 2009 452 453 void removeMacroElements(); 454 455 void updateLocalGlobalNumbering(); Thomas Witkowski committed Nov 14, 2011 456 457 /// Updates the local and global DOF numbering after the mesh has been /// changed. 458 void updateLocalGlobalNumbering(const FiniteElemSpace *feSpace); Thomas Witkowski committed Jun 11, 2009 459 460 461 462 463 /// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are /// handled by the mesh distributor. void createPeriodicMap(); 464 /** \brief 465 466 467 468 * Creates, for a specific FE space, to all DOFs in rank's partition that * are on a periodic boundary the mapping from dof index to the other * periodic dof indices. This information is stored in \ref periodicDofMap. */ 469 void createPeriodicMap(const FiniteElemSpace *feSpace); Thomas Witkowski committed Oct 14, 2009 470 Thomas Witkowski committed Nov 14, 2011 471 472 473 474 475 476 477 478 /** \brief * This function is called only once during the initialization when the * whole macro mesh is available on all cores. It copies the pointers of all * macro elements to \ref allMacroElements and stores all neighbour * information based on macro element indices (and not pointer based) in * \ref macroElementNeighbours. These information are then used to * reconstruct macro elements during mesh redistribution. */ Thomas Witkowski committed Nov 25, 2010 479 480 void createMacroElementInfo(); 481 482 void updateMacroElementInfo(); Thomas Witkowski committed Jan 19, 2010 483 /** \brief Thomas Witkowski committed Nov 14, 2011 484 485 486 487 488 489 * Checks for all given interior boundaries if the elements fit together on * both sides of the boundaries. If this is not the case, the mesh is * adapted. Because refinement of a certain element may forces the * refinement of other elements, it is not guaranteed that all rank's meshes * fit together after this function terminates. Hence, it must be called * until a stable mesh refinement is reached. Thomas Witkowski committed Jan 19, 2010 490 * Thomas Witkowski committed Nov 14, 2011 491 492 * \param[in] allBound Defines a map from rank to interior boundaries * which should be checked. 493 * Thomas Witkowski committed Nov 14, 2011 494 495 496 * \return If the mesh has been changed by this function, it returns * true. Otherwise, it returns false, i.e., the given interior * boundaries fit together on both sides. Thomas Witkowski committed Jan 19, 2010 497 498 */ bool checkAndAdaptBoundary(RankToBoundMap &allBound); Thomas Witkowski committed Nov 26, 2010 499 Thomas Witkowski committed Mar 28, 2011 500 501 /// Sets \ref isRankDof to all matrices and rhs vectors in a given /// stationary problem. Thomas Witkowski committed Apr 21, 2011 502 void setRankDofs(ProblemStatSeq *probStat); Thomas Witkowski committed Mar 28, 2011 503 Thomas Witkowski committed Nov 14, 2011 504 505 /// Sets \ref isRankDof to all matrices and rhs vectors in all /// stationary problems. Thomas Witkowski committed Jul 18, 2010 506 507 void setRankDofs(); Thomas Witkowski committed Jun 29, 2010 508 509 510 511 /// Removes all periodic boundary condition information from all matrices and /// vectors of all stationary problems and from the mesh itself. void removePeriodicBoundaryConditions(); Thomas Witkowski committed Feb 28, 2012 512 513 514 515 /// Removes all periodic boundary condition information from all matrices and /// vector of a given stationary problem. void removePeriodicBoundaryConditions(ProblemStatSeq *probStat); Thomas Witkowski committed Feb 08, 2010 516 // Removes all periodic boundaries from a given boundary map. Thomas Witkowski committed Jun 29, 2010 517 void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap); Thomas Witkowski committed Feb 08, 2010 518 Thomas Witkowski committed Apr 11, 2012 519 520 void createMeshLevelStructure(); Thomas Witkowski committed Sep 28, 2009 521 /// Writes a vector of dof pointers to an output stream. Thomas Witkowski committed Jan 07, 2011 522 void serialize(ostream &out, DofContainer &data); Thomas Witkowski committed Sep 28, 2009 523 524 525 526 527 /// Writes a \ref RankToDofContainer to an output stream. void serialize(ostream &out, map > &data); Thomas Witkowski committed Sep 28, 2009 528 /// Reads a vector of dof pointers from an input stream. Thomas Witkowski committed Jan 07, 2011 529 530 void deserialize(istream &in, DofContainer &data, map &dofMap); Thomas Witkowski committed Sep 28, 2009 531 532 /// Reads a \ref RankToDofContainer from an input stream. 533 534 535 void deserialize(istream &in, map > &data, map > &dofMap); Thomas Witkowski committed Sep 29, 2009 536 537 538 /// Writes a mapping from dof pointers to some values to an output stream. template Thomas Witkowski committed Jan 07, 2011 539 void serialize(ostream &out, map &data) Thomas Witkowski committed Sep 29, 2009 540 { 541 542 FUNCNAME("ParallelDomainBase::serialize()"); Thomas Witkowski committed Sep 29, 2009 543 int mapSize = data.size(); 544 SerUtil::serialize(out, mapSize); Thomas Witkowski committed Jan 07, 2011 545 for (typename map::iterator it = data.begin(); Thomas Witkowski committed Sep 29, 2009 546 547 548 it != data.end(); ++it) { int v1 = (*(it->first)); T v2 = it->second; 549 550 SerUtil::serialize(out, v1); SerUtil::serialize(out, v2); Thomas Witkowski committed Sep 29, 2009 551 552 553 554 555 } } /// Reads a mapping from dof pointer to some values from an input stream. template Thomas Witkowski committed Jan 07, 2011 556 557 void deserialize(istream &in, map &data, map &dofMap) Thomas Witkowski committed Sep 29, 2009 558 { Thomas Witkowski committed Oct 02, 2009 559 560 FUNCNAME("ParallelDomainBase::deserialize()"); Thomas Witkowski committed Sep 29, 2009 561 int mapSize = 0; 562 SerUtil::deserialize(in, mapSize); Thomas Witkowski committed Sep 29, 2009 563 564 565 for (int i = 0; i < mapSize; i++) { int v1 = 0; T v2; 566 567 SerUtil::deserialize(in, v1); SerUtil::deserialize(in, v2); Thomas Witkowski committed Oct 02, 2009 568 569 570 TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1); Thomas Witkowski committed Sep 29, 2009 571 572 573 data[dofMap[v1]] = v2; } } Thomas Witkowski committed Nov 30, 2010 574 Thomas Witkowski committed Apr 03, 2009 575 protected: Thomas Witkowski committed Nov 14, 2011 576 577 /// List of all stationary problems that are managed by this mesh /// distributor. Thomas Witkowski committed Apr 21, 2011 578 vector problemStat; Thomas Witkowski committed Feb 08, 2010 579 Thomas Witkowski committed Mar 28, 2011 580 581 582 /// If true, the mesh distributor is already initialized; bool initialized; Thomas Witkowski committed Apr 03, 2009 583 584 585 586 587 588 /// The rank of the current process. int mpiRank; /// Overall number of processes. int mpiSize; Thomas Witkowski committed Nov 14, 2011 589 590 /// MPI communicator collected all processes, which should be used for /// calculation. The Debug procces is not included in this communicator. Thomas Witkowski committed Apr 03, 2009 591 592 593 MPI::Intracomm mpiComm; /// Name of the problem (as used in the init files) Thomas Witkowski committed Jan 07, 2011 594 string name; Thomas Witkowski committed Apr 03, 2009 595 596 597 /// Finite element spaces of the problem. vector feSpaces; Thomas Witkowski committed Apr 15, 2009 598 Thomas Witkowski committed Apr 03, 2009 599 600 601 /// Mesh of the problem. Mesh *mesh; Thomas Witkowski committed Feb 13, 2012 602 603 604 /// A refinement manager that should be used on the mesh. It is used to /// refine elements at interior boundaries in order to fit together with /// elements on the other side of the interior boundary. 605 606 RefinementManager *refineManager; Thomas Witkowski committed Jun 26, 2009 607 608 609 /// Info level. int info; Thomas Witkowski committed Nov 14, 2011 610 611 /// Pointer to a mesh partitioner that is used to partition the mesh to /// the ranks. Thomas Witkowski committed Jan 24, 2011 612 MeshPartitioner *partitioner; Thomas Witkowski committed Apr 03, 2009 613 Thomas Witkowski committed Nov 14, 2011 614 615 /// Weights for the elements, i.e., the number of leaf elements within /// this element. Thomas Witkowski committed Jan 07, 2011 616 map elemWeights; Thomas Witkowski committed Apr 03, 2009 617 Thomas Witkowski committed Feb 13, 2012 618 619 /// Stores to every macro element index the number of the rank that owns this /// macro element. Thomas Witkowski committed Jan 24, 2011 620 map partitionMap; Thomas Witkowski committed Apr 03, 2009 621 622 map dofFeData; Thomas Witkowski committed Jun 08, 2010 623 Thomas Witkowski committed Feb 13, 2012 624 /// Database to store and query all sub-objects of all elements of the Thomas Witkowski committed Nov 14, 2011 625 /// macro mesh. Thomas Witkowski committed Feb 13, 2012 626 ElementObjectDatabase elObjDb; Thomas Witkowski committed Nov 29, 2010 627 Thomas Witkowski committed Nov 14, 2011 628 /// Maps to each macro element index a pointer to the corresponding element. Thomas Witkowski committed Jan 07, 2011 629 map macroElIndexMap; Thomas Witkowski committed Nov 29, 2010 630 Thomas Witkowski committed Nov 14, 2011 631 /// Maps to each macro element index the type of this element. Thomas Witkowski committed Jan 07, 2011 632 map macroElIndexTypeMap; Thomas Witkowski committed Nov 29, 2010 633 Thomas Witkowski committed Feb 13, 2012 634 635 636 637 /// Defines the interior boundaries of the domain that result from /// partitioning the whole mesh. Contains only the boundaries, which are /// owned by the rank, i.e., the object gives for every neighbour rank i /// the boundaries this rank owns and shares with rank i. Thomas Witkowski committed Feb 01, 2012 638 InteriorBoundary rankIntBoundary; Thomas Witkowski committed Jun 15, 2009 639 Thomas Witkowski committed Feb 13, 2012 640 641 642 643 /// Defines the interior boundaries of the domain that result from /// partitioning the whole mesh. Contains only the boundaries, which are /// not owned by the rank, i.e., the object gives for every neighbour rank /// i the boundaries that are owned by rank i and are shared with this rank. Thomas Witkowski committed Jun 15, 2009 644 InteriorBoundary otherIntBoundary; Thomas Witkowski committed Apr 27, 2009 645 Thomas Witkowski committed Feb 13, 2012 646 647 /// Defines the periodic boundaries with other ranks. Periodic boundaries /// have no owner, as it is the case of interior boundaries. Thomas Witkowski committed Oct 14, 2009 648 649 InteriorBoundary periodicBoundary; Thomas Witkowski committed Feb 13, 2012 650 651 /// This map contains for each rank the list of DOFs the current rank must /// end to exchange solution DOFs at the interior boundaries. Thomas Witkowski committed Jan 17, 2012 652 DofComm sendDofs; Thomas Witkowski committed Jun 10, 2009 653 Thomas Witkowski committed Feb 13, 2012 654 655 656 /// This map contains on each rank the list of DOFs from which the current /// rank will receive DOF values (i.e., this are all DOFs at an interior /// boundary). The DOF indices are given in rank's local numbering. Thomas Witkowski committed Jan 17, 2012 657 DofComm recvDofs; Thomas Witkowski committed Aug 13, 2009 658 Thomas Witkowski committed Feb 13, 2012 659 660 661 662 /// This map contains on each rank a list of DOFs along the interior bound- /// aries to communicate with other ranks. The DOF indices are given in rank's /// local numbering. Periodic boundaries within one subdomain are not /// considered here. 663 664 DofComm periodicDofs; Thomas Witkowski committed Jan 26, 2012 665 PeriodicMap periodicMap; Thomas Witkowski committed Jul 08, 2010 666 Thomas Witkowski committed Jan 19, 2011 667 668 669 670 /// This set of values must be interchanged between ranks when the mesh is /// repartitioned. vector*> interchangeVectors; Thomas Witkowski committed Feb 13, 2012 671 672 673 674 /// If the problem definition has been read from a serialization file, this /// variable is true, otherwise it is false. This variable is used to stop the /// initialization function, if the problem definition has already been read /// from a serialization file. 675 bool deserialized; Thomas Witkowski committed Dec 08, 2009 676 677 678 679 /// Denotes whether there exists a filewriter for this object. bool writeSerializationFile; Thomas Witkowski committed Nov 26, 2010 680 681 682 /// If true, it is possible to repartition the mesh during computations. bool repartitioningAllowed; Thomas Witkowski committed Nov 14, 2011 683 684 /// Stores the number of mesh changes that must lie in between to /// repartitionings. Thomas Witkowski committed Dec 20, 2010 685 686 int repartitionIthChange; Thomas Witkowski committed Nov 14, 2011 687 688 /// Counts the number of mesh changes after the last mesh repartitioning /// was done. 689 int nMeshChangesAfterLastRepartitioning; Thomas Witkowski committed Dec 03, 2010 690 691 692 693 /// Countes the number of mesh repartitions that were done. Till now, this /// variable is used only for debug outputs. int repartitioningCounter; Thomas Witkowski committed Dec 03, 2010 694 Thomas Witkowski committed Dec 16, 2010 695 /// Directory name where all debug output files should be written to. Thomas Witkowski committed Jan 07, 2011 696 string debugOutputDir; Thomas Witkowski committed Dec 16, 2010 697 Thomas Witkowski committed Feb 13, 2012 698 699 /// Stores the mesh change index. This is used to recognize changes in the /// mesh structure (e.g. through refinement or coarsening managers). Thomas Witkowski committed Dec 08, 2009 700 long lastMeshChangeIndex; 701 Thomas Witkowski committed Nov 14, 2011 702 703 704 705 /// Stores for all macro elements of the original macro mesh the /// neighbourhood information based on element indices. Thus, each macro /// element index is mapped to a vector containing all indices of /// neighbouring macro elements. Thomas Witkowski committed Jan 07, 2011 706 map > macroElementNeighbours; Thomas Witkowski committed Nov 25, 2010 707 Thomas Witkowski committed Nov 14, 2011 708 709 /// Store all macro elements of the overall mesh, i.e., before the /// mesh is redistributed for the first time. Thomas Witkowski committed Jan 07, 2011 710 vector allMacroElements; 711 Thomas Witkowski committed Apr 08, 2011 712 713 Flag createBoundaryDofFlag; Thomas Witkowski committed Apr 16, 2012 714 715 716 /// Stores on each mesh level for all FE spaces the information about /// all boundary DOFs. vector > boundaryDofInfo; Thomas Witkowski committed May 19, 2011 717 Thomas Witkowski committed Apr 11, 2012 718 719 MeshLevelData levelData; Thomas Witkowski committed Apr 08, 2011 720 public: Thomas Witkowski committed Jan 31, 2012 721 722 bool sebastianMode; Thomas Witkowski committed Apr 21, 2011 723 724 725 /// The boundary DOFs are sorted by subobject entities, i.e., first all /// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// communication structure vectors, \ref sendDofs and \ref recvDofs. Thomas Witkowski committed Apr 08, 2011 726 727 static const Flag BOUNDARY_SUBOBJ_SORTED; Thomas Witkowski committed Apr 21, 2011 728 729 730 731 732 733 734 735 736 /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will send to other ranks (thus, rank /// owned DOFs. static const Flag BOUNDARY_FILL_INFO_SEND_DOFS; /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will receive from other ranks (thus, DOFs /// that are owned by another rank). static const Flag BOUNDARY_FILL_INFO_RECV_DOFS; Thomas Witkowski committed Apr 08, 2011 737 Thomas Witkowski committed May 19, 2011 738 739 static MeshDistributor *globalMeshDistributor; 740 friend class ParallelDebug; Thomas Witkowski committed Apr 03, 2009 741 742 743 }; } 744 #endif // AMDIS_MESHDISTRIBUTOR_H