MeshDistributor.h 21.5 KB
 Thomas Witkowski committed Apr 03, 2009 1 2 3 4 // ============================================================================ // == == // == AMDiS - Adaptive multidimensional simulations == // == ==  Thomas Witkowski committed Dec 16, 2010 5 // == http://www.amdis-fem.org ==  Thomas Witkowski committed Apr 03, 2009 6 7 // == == // ============================================================================  Thomas Witkowski committed Dec 16, 2010 8 9 10 11 12 13 14 15 16 17 18 19 // // Software License for AMDiS // // Copyright (c) 2010 Dresden University of Technology // All rights reserved. // Authors: Simon Vey, Thomas Witkowski et al. // // This file is part of AMDiS // // See also license.opensource.txt in the distribution.  Thomas Witkowski committed Apr 03, 2009 20   21 /** \file MeshDistributor.h */  Thomas Witkowski committed Jul 09, 2009 22   23 24 #ifndef AMDIS_MESHDISTRIBUTOR_H #define AMDIS_MESHDISTRIBUTOR_H  Thomas Witkowski committed Apr 03, 2009 25 26   Thomas Witkowski committed Feb 08, 2010 27 #include  Thomas Witkowski committed Jan 17, 2012 28 #include "parallel/DofComm.h"  Thomas Witkowski committed Feb 13, 2012 29 #include "parallel/ElementObjectDatabase.h"  Thomas Witkowski committed Apr 21, 2011 30 #include "parallel/ParallelTypes.h"  Thomas Witkowski committed Apr 11, 2012 31 #include "parallel/MeshLevelData.h"  Thomas Witkowski committed Jan 24, 2011 32 #include "parallel/MeshPartitioner.h"  Thomas Witkowski committed Nov 29, 2010 33 #include "parallel/InteriorBoundary.h"  Thomas Witkowski committed Apr 18, 2012 34 #include "parallel/ParallelDofMapping.h"  Thomas Witkowski committed Jan 26, 2012 35 #include "parallel/PeriodicMap.h"  Thomas Witkowski committed Nov 08, 2011 36 #include "parallel/StdMpi.h"  Thomas Witkowski committed Jan 27, 2011 37 #include "AMDiS_fwd.h"  Thomas Witkowski committed Feb 09, 2012 38 #include "Containers.h"  Thomas Witkowski committed Feb 08, 2010 39 #include "Global.h"  Thomas Witkowski committed Apr 03, 2009 40 41 #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h"  Thomas Witkowski committed Apr 15, 2009 42 #include "FiniteElemSpace.h"  Thomas Witkowski committed Sep 29, 2009 43 #include "Serializer.h"  Thomas Witkowski committed Feb 08, 2010 44 #include "BoundaryManager.h"  Thomas Witkowski committed Jan 27, 2011 45 #include "SystemVector.h"  Thomas Witkowski committed Jun 12, 2009 46   Thomas Witkowski committed Apr 03, 2009 47 namespace AMDiS {  Thomas Witkowski committed Jan 07, 2011 48 49  using namespace std;  Thomas Witkowski committed Apr 08, 2011 50 51 52 53  struct BoundaryDofInfo {  Thomas Witkowski committed Apr 08, 2011 54  map geoDofs;  Thomas Witkowski committed Apr 08, 2011 55 56  };  57   Thomas Witkowski committed Jun 08, 2010 58  class MeshDistributor  Thomas Witkowski committed Apr 03, 2009 59  {  Thomas Witkowski committed May 19, 2011 60  private:  Thomas Witkowski committed Mar 18, 2011 61  MeshDistributor();  Thomas Witkowski committed May 19, 2011 62 63  public:  Thomas Witkowski committed Sep 05, 2012 64 65  ~MeshDistributor();  66  /// Initialization of mesh distributor.  67  void initParallelization();  Thomas Witkowski committed Apr 03, 2009 68   69  /// Clean up procedure for the mesh distributor and attached objects.  70  void exitParallelization();  Thomas Witkowski committed Jun 08, 2010 71   72 73 74 75 76 77 78 79  /** \brief * Register a parallel DOF mapping. This DOF mapping object will than * automatically updated by the mesh distributer after mesh changes. * * \param[in] dofMap Parallel DOF mapping object. */ void registerDofMap(ParallelDofMapping &dofMap);  Thomas Witkowski committed Nov 17, 2012 80 81 82 83 84 85 86  /** \brief * Removes a registered DOF mapping from the mesh distributor. * * \param[in] dofMap Parallel DOF mapping object to be removed. */ void removeDofMap(ParallelDofMapping &dofMap);  Thomas Witkowski committed Nov 14, 2011 87 88 89  /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector /// will be automatically interchanged between ranks when mesh is /// repartitioned.  Thomas Witkowski committed Jan 19, 2011 90 91 92 93 94  void addInterchangeVector(DOFVector *vec) { interchangeVectors.push_back(vec); }  Thomas Witkowski committed Jan 27, 2011 95 96 97 98 99 100 101  /// Adds all DOFVectors of a SystemVector to \ref interchangeVecs. void addInterchangeVector(SystemVector *vec) { for (int i = 0; i < vec->getSize(); i++) interchangeVectors.push_back(vec->getDOFVector(i)); }  Thomas Witkowski committed Jan 29, 2010 102  /** \brief  Thomas Witkowski committed Nov 14, 2011 103 104 105 106 107  * This function checks if the mesh has changed on at least on rank. In * this case, the interior boundaries are adapted on all ranks such that * they fit together on all ranks. Furthermore the function * \ref updateLocalGlobalNumbering() is called to update the DOF numberings * and mappings on all rank due to the new mesh structure.  108  *  Thomas Witkowski committed Nov 14, 2011 109 110 111 112 113  * \param[in] tryRepartition If this parameter is true, repartitioning * may be done. This depends on several other * parameters. If the parameter is false, the * mesh is only checked and adapted but never * repartitioned.  Thomas Witkowski committed Jan 29, 2010 114  */  115  void checkMeshChange(bool tryRepartition = true);  Thomas Witkowski committed Jan 29, 2010 116   Thomas Witkowski committed Jun 22, 2012 117 118 119  /// Checks if is required to repartition the mesh. If this is the case, a new /// partition will be created and the mesh will be redistributed between the /// ranks.  Thomas Witkowski committed Jan 02, 2012 120 121  void repartitionMesh();  Thomas Witkowski committed Jun 22, 2012 122 123 124 125 126 127 128  void getImbalanceFactor(double &imbalance, int &minDofs, int &maxDofs, int &sumDofs); double getImbalanceFactor();  Thomas Witkowski committed Jan 02, 2012 129 130 131  /// Calculates the imbalancing factor and prints it to screen. void printImbalanceFactor();  Thomas Witkowski committed Jun 22, 2012 132 133 134 135  /// Test, if the mesh consists of macro elements only. The mesh partitioning /// of the parallelization works for macro meshes only and would fail, if the /// mesh is already refined in some way. Therefore, this function will exit /// the program if it finds a non macro element in the mesh.  Thomas Witkowski committed Jun 26, 2009 136  void testForMacroMesh();  Thomas Witkowski committed Apr 03, 2009 137   Thomas Witkowski committed Sep 05, 2012 138  inline string getName()  Thomas Witkowski committed Jun 08, 2010 139 140 141  { return name; }  142   Thomas Witkowski committed Jan 14, 2011 143 144 145 146 147  inline Mesh* getMesh() { return mesh; }  Thomas Witkowski committed Apr 20, 2012 148 149  /// Returns the periodic mapping handler, \ref periodicMap. inline PeriodicMap& getPeriodicMap()  Thomas Witkowski committed Jul 10, 2009 150  {  Thomas Witkowski committed Apr 20, 2012 151  return periodicMap;  Thomas Witkowski committed Jul 10, 2009 152  }  Thomas Witkowski committed Apr 03, 2009 153   Thomas Witkowski committed Apr 30, 2012 154  DofComm& getDofComm()  155  {  Thomas Witkowski committed Apr 30, 2012 156  return dofComm;  157 158  }  Thomas Witkowski committed May 04, 2012 159 160 161 162 163  DofComm& getDofCommSd() { return dofCommSd; }  Thomas Witkowski committed May 02, 2012 164 165 166 167 168  InteriorBoundary& getIntBoundary() { return intBoundary; }  Thomas Witkowski committed May 04, 2012 169 170 171 172 173  InteriorBoundary& getIntBoundarySd() { return intBoundarySd; }  Thomas Witkowski committed Jun 08, 2010 174  inline long getLastMeshChangeIndex()  Thomas Witkowski committed Jun 10, 2009 175  {  Thomas Witkowski committed Jun 08, 2010 176  return lastMeshChangeIndex;  Thomas Witkowski committed Apr 07, 2009 177  }  Thomas Witkowski committed Apr 03, 2009 178   Thomas Witkowski committed Jun 08, 2010 179  inline int getMpiRank()  Thomas Witkowski committed Jun 11, 2009 180  {  Thomas Witkowski committed Jun 08, 2010 181  return mpiRank;  Thomas Witkowski committed Jun 11, 2009 182  }  Thomas Witkowski committed Apr 03, 2009 183   Thomas Witkowski committed Jan 14, 2011 184 185 186 187 188  inline int getMpiSize() { return mpiSize; }  Thomas Witkowski committed Jun 08, 2010 189 190 191  inline MPI::Intracomm& getMpiComm() { return mpiComm;  Thomas Witkowski committed Apr 03, 2009 192 193  }  194 195 196 197 198  inline bool isInitialized() { return initialized; }  Thomas Witkowski committed Sep 28, 2009 199  // Writes all data of this object to an output stream.  Thomas Witkowski committed Jan 07, 2011 200  void serialize(ostream &out);  Thomas Witkowski committed Jun 11, 2009 201   Thomas Witkowski committed Sep 28, 2009 202  // Reads the object data from an input stream.  Thomas Witkowski committed Jan 07, 2011 203  void deserialize(istream &in);  Thomas Witkowski committed Jun 08, 2010 204 205  /** \brief  Thomas Witkowski committed Nov 14, 2011 206 207 208 209  * This function must be used if the values of a DOFVector must be * synchronised over all ranks. That means, that each rank sends the * values of the DOFs, which are owned by the rank and lie on an interior * bounday, to all other ranks also having these DOFs.  Thomas Witkowski committed Jun 08, 2010 210  *  Thomas Witkowski committed Nov 14, 2011 211 212 213  * This function must be used, for example, after the lineary system is * solved, or after the DOFVector is set by some user defined functions, * e.g., initial solution functions.  Thomas Witkowski committed Jun 08, 2010 214  */  Thomas Witkowski committed Nov 08, 2011 215 216 217 218 219  template void synchVector(DOFVector &vec) { StdMpi > stdMpi(mpiComm);  220 221  const FiniteElemSpace *fe = vec.getFeSpace();  Thomas Witkowski committed Apr 30, 2012 222 223  for (DofComm::Iterator it(dofComm.getSendDofs(), fe); !it.end(); it.nextRank()) {  Thomas Witkowski committed Nov 08, 2011 224  vector dofs;  Thomas Witkowski committed Jan 17, 2012 225  dofs.reserve(it.getDofs().size());  Thomas Witkowski committed Nov 08, 2011 226   Thomas Witkowski committed Jan 17, 2012 227 228  for (; !it.endDofIter(); it.nextDof()) dofs.push_back(vec[it.getDofIndex()]);  Thomas Witkowski committed Nov 08, 2011 229   Thomas Witkowski committed Jan 17, 2012 230  stdMpi.send(it.getRank(), dofs);  Thomas Witkowski committed Nov 08, 2011 231  }  Thomas Witkowski committed Jan 17, 2012 232   Thomas Witkowski committed Apr 30, 2012 233 234  for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank())  Thomas Witkowski committed Jan 17, 2012 235 236  stdMpi.recv(it.getRank());  Thomas Witkowski committed Nov 08, 2011 237  stdMpi.startCommunication();  Thomas Witkowski committed Jan 17, 2012 238   Thomas Witkowski committed Apr 30, 2012 239 240  for (DofComm::Iterator it(dofComm.getRecvDofs(), fe); !it.end(); it.nextRank())  Thomas Witkowski committed Jan 17, 2012 241 242 243  for (; !it.endDofIter(); it.nextDof()) vec[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];  Thomas Witkowski committed Nov 08, 2011 244  }  Thomas Witkowski committed Sep 25, 2012 245   Thomas Witkowski committed May 11, 2012 246 247 248 249  /// Works in the same way as the function above defined for DOFVectors. Due /// to performance, this function does not call \ref synchVector for each /// DOFVector, but instead sends all values of all DOFVectors all at once. void synchVector(SystemVector &vec, int level = 0);  Thomas Witkowski committed Jun 08, 2010 250   Thomas Witkowski committed Sep 25, 2012 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285  /// Works quite similar to the function \ref synchVector, but instead the /// values of subdomain vectors are add along the boundaries. template void synchAddVector(DOFVector &vec) { StdMpi > stdMpi(mpiComm); const FiniteElemSpace *fe = vec.getFeSpace(); for (DofComm::Iterator it(dofComm.getRecvDofs(), fe); !it.end(); it.nextRank()) { vector dofs; dofs.reserve(it.getDofs().size()); for (; !it.endDofIter(); it.nextDof()) dofs.push_back(vec[it.getDofIndex()]); stdMpi.send(it.getRank(), dofs); } for (DofComm::Iterator it(dofComm.getSendDofs()); !it.end(); it.nextRank()) stdMpi.recv(it.getRank()); stdMpi.startCommunication(); for (DofComm::Iterator it(dofComm.getSendDofs(), fe); !it.end(); it.nextRank()) for (; !it.endDofIter(); it.nextDof()) vec[it.getDofIndex()] += stdMpi.getRecvData(it.getRank())[it.getDofCounter()]; synchVector(vec); }  Thomas Witkowski committed Aug 14, 2012 286 287 288 289 290 291  /// In 3D, a subdomain may not be a valid AMDiS mesh if it contains two /// parts which are only connected by an edge. In this case, the standard /// refinement algorithm does not work correctly, as two elements connected /// only on one edge are not neighours by definition. This functions checks /// for this situation and fix the problem. For this, the mesh is search for /// all edges connecting two elements that are otherwise not connected.  292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308  void fix3dMeshRefinement(); /** \brief Is used only within \ref fix3dMeshRefinement. * * \param[in] elems Set of macro element indices. * \param[in] edgeNoInEl Maps from each element index in elems to a * local edge number. * \param[out] disconnectedEls On output, this vector contains sets of * element indices. The union is equal to elems. * Each set contains all element indices, which * are reachable among each other by neighbour * relations. Elements within two different sets * cannot be reached via neigbourhood relation. */ void helpToFix(std::set &elems, map &edgeNoInEl, vector > &disconnectedEls);  Thomas Witkowski committed Jan 20, 2011 309   Thomas Witkowski committed Apr 08, 2011 310 311  void setBoundaryDofRequirement(Flag flag) {  Thomas Witkowski committed Oct 25, 2012 312  createBoundaryDofFlag |= flag;  Thomas Witkowski committed Apr 08, 2011 313 314  }  Thomas Witkowski committed Apr 27, 2012 315 316  BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, int level)  Thomas Witkowski committed Apr 08, 2011 317  {  Thomas Witkowski committed Apr 16, 2012 318 319 320 321 322 323 324  FUNCNAME("MeshDistributor::getBoundaryDofInfo()"); TEST_EXIT_DBG(level < static_cast(boundaryDofInfo.size())) ("Wrong level number: %d, whereas array size is %d!\n", level, boundaryDofInfo.size()); return boundaryDofInfo[level][feSpace];  Thomas Witkowski committed Apr 08, 2011 325 326  }  327 328  void getAllBoundaryDofs(const FiniteElemSpace *feSpace, int level,  329  DofContainer& dofs);  Thomas Witkowski committed Apr 21, 2011 330   Thomas Witkowski committed Nov 02, 2012 331  ElementObjectDatabase& getElementObjectDb()  332 333 334  { return elObjDb; }  Thomas Witkowski committed May 19, 2011 335 336 337  /// Adds a stationary problem to the global mesh distributor objects. static void addProblemStatGlobal(ProblemStatSeq *probStat);  Thomas Witkowski committed Apr 13, 2012 338 339 340 341 342 343  MeshLevelData& getMeshLevelData() { return levelData; }  Thomas Witkowski committed Oct 30, 2012 344 345 346  /// Updates all registered parallel DOF mappings, see \ref dofMaps. void updateParallelDofMappings();  Thomas Witkowski committed Jun 22, 2012 347 348 349 350 351 352 353  void updateLocalGlobalNumbering(); /// Updates the local and global DOF numbering after the mesh has been /// changed. void updateLocalGlobalNumbering(ParallelDofMapping &dmap, const FiniteElemSpace *feSpace);  Thomas Witkowski committed Jun 11, 2009 354  protected:  Thomas Witkowski committed Nov 07, 2012 355 356 357 358 359 360 361 362  /// Creates an initial paritioning of the mesh. void createInitialPartitioning(); /// Set for each element on the partitioning level the number of /// leaf elements. void setInitialElementWeights(); ///  Thomas Witkowski committed May 19, 2011 363 364  void addProblemStat(ProblemStatSeq *probStat);  Thomas Witkowski committed Nov 14, 2011 365 366  /// Determines the interior boundaries, i.e. boundaries between ranks, and /// stores all information about them in \ref interiorBoundary.  Thomas Witkowski committed May 02, 2012 367  void createInteriorBoundary(bool firstCall);  Thomas Witkowski committed Nov 29, 2010 368   Thomas Witkowski committed Nov 07, 2012 369  ///  Thomas Witkowski committed Apr 08, 2011 370 371  void createBoundaryDofs();  Thomas Witkowski committed Nov 14, 2011 372 373  /// Removes all macro elements from the mesh that are not part of ranks /// partition.  Thomas Witkowski committed Jun 11, 2009 374 375  void removeMacroElements();  376 377 378 379  /// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are /// handled by the mesh distributor. void createPeriodicMap();  Thomas Witkowski committed Apr 27, 2012 380 381 382  /// Creates, for a specific FE space, to all DOFs in rank's partition that /// are on a periodic boundary the mapping from dof index to the other /// periodic dof indices. This information is stored in \ref periodicDofMap.  383  void createPeriodicMap(const FiniteElemSpace *feSpace);  Thomas Witkowski committed Oct 14, 2009 384   Thomas Witkowski committed Apr 27, 2012 385 386 387 388 389 390  /// This function is called only once during the initialization when the /// whole macro mesh is available on all cores. It copies the pointers of all /// macro elements to \ref allMacroElements and stores all neighbour /// information based on macro element indices (and not pointer based) in /// \ref macroElementNeighbours. These information are then used to /// reconstruct macro elements during mesh redistribution.  Thomas Witkowski committed Nov 25, 2010 391 392  void createMacroElementInfo();  393 394  void updateMacroElementInfo();  Thomas Witkowski committed Jan 19, 2010 395  /** \brief  Thomas Witkowski committed Nov 14, 2011 396 397 398 399 400 401  * Checks for all given interior boundaries if the elements fit together on * both sides of the boundaries. If this is not the case, the mesh is * adapted. Because refinement of a certain element may forces the * refinement of other elements, it is not guaranteed that all rank's meshes * fit together after this function terminates. Hence, it must be called * until a stable mesh refinement is reached.  Thomas Witkowski committed Jan 19, 2010 402  *  Thomas Witkowski committed Nov 14, 2011 403 404  * \param[in] allBound Defines a map from rank to interior boundaries * which should be checked.  405  *  Thomas Witkowski committed Nov 14, 2011 406 407 408  * \return If the mesh has been changed by this function, it returns * true. Otherwise, it returns false, i.e., the given interior * boundaries fit together on both sides.  Thomas Witkowski committed Jan 19, 2010 409 410  */ bool checkAndAdaptBoundary(RankToBoundMap &allBound);  Thomas Witkowski committed Nov 26, 2010 411   Thomas Witkowski committed Jun 29, 2010 412 413 414 415  /// Removes all periodic boundary condition information from all matrices and /// vectors of all stationary problems and from the mesh itself. void removePeriodicBoundaryConditions();  Thomas Witkowski committed Feb 28, 2012 416 417 418 419  /// Removes all periodic boundary condition information from all matrices and /// vector of a given stationary problem. void removePeriodicBoundaryConditions(ProblemStatSeq *probStat);  Thomas Witkowski committed Feb 08, 2010 420  // Removes all periodic boundaries from a given boundary map.  Thomas Witkowski committed Jun 29, 2010 421  void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);  Thomas Witkowski committed Feb 08, 2010 422   Thomas Witkowski committed Apr 11, 2012 423 424  void createMeshLevelStructure();  Thomas Witkowski committed Sep 28, 2009 425  /// Writes a vector of dof pointers to an output stream.  Thomas Witkowski committed Jan 07, 2011 426  void serialize(ostream &out, DofContainer &data);  Thomas Witkowski committed Sep 28, 2009 427   428 429 430 431  /// Writes a \ref RankToDofContainer to an output stream. void serialize(ostream &out, map > &data);  Thomas Witkowski committed Sep 28, 2009 432  /// Reads a vector of dof pointers from an input stream.  Thomas Witkowski committed Jan 07, 2011 433  void deserialize(istream &in, DofContainer &data,  Thomas Witkowski committed Apr 18, 2012 434  map &dofIndexMap);  Thomas Witkowski committed Sep 28, 2009 435 436  /// Reads a \ref RankToDofContainer from an input stream.  437 438  void deserialize(istream &in, map > &data,  Thomas Witkowski committed Apr 18, 2012 439  map > &dofIndexMap);  Thomas Witkowski committed Sep 29, 2009 440 441 442  /// Writes a mapping from dof pointers to some values to an output stream. template  Thomas Witkowski committed Jan 07, 2011 443  void serialize(ostream &out, map &data)  Thomas Witkowski committed Sep 29, 2009 444  {  445 446  FUNCNAME("ParallelDomainBase::serialize()");  Thomas Witkowski committed Sep 29, 2009 447  int mapSize = data.size();  448  SerUtil::serialize(out, mapSize);  Thomas Witkowski committed Jan 07, 2011 449  for (typename map::iterator it = data.begin();  Thomas Witkowski committed Sep 29, 2009 450 451 452  it != data.end(); ++it) { int v1 = (*(it->first)); T v2 = it->second;  453 454  SerUtil::serialize(out, v1); SerUtil::serialize(out, v2);  Thomas Witkowski committed Sep 29, 2009 455 456 457 458 459  } } /// Reads a mapping from dof pointer to some values from an input stream. template  Thomas Witkowski committed Jan 07, 2011 460  void deserialize(istream &in, map &data,  Thomas Witkowski committed Apr 18, 2012 461  map &dofIndexMap)  Thomas Witkowski committed Sep 29, 2009 462  {  Thomas Witkowski committed Oct 02, 2009 463 464  FUNCNAME("ParallelDomainBase::deserialize()");  Thomas Witkowski committed Sep 29, 2009 465  int mapSize = 0;  466  SerUtil::deserialize(in, mapSize);  Thomas Witkowski committed Sep 29, 2009 467 468 469  for (int i = 0; i < mapSize; i++) { int v1 = 0; T v2;  470 471  SerUtil::deserialize(in, v1); SerUtil::deserialize(in, v2);  Thomas Witkowski committed Oct 02, 2009 472   Thomas Witkowski committed Apr 18, 2012 473 474  TEST_EXIT_DBG(dofIndexMap.count(v1) != 0) ("Cannot find DOF %d in map!\n", v1);  Thomas Witkowski committed Oct 02, 2009 475   Thomas Witkowski committed Apr 18, 2012 476  data[dofIndexMap[v1]] = v2;  Thomas Witkowski committed Sep 29, 2009 477 478  } }  Thomas Witkowski committed Nov 30, 2010 479   Thomas Witkowski committed Apr 03, 2009 480  protected:  Thomas Witkowski committed Nov 14, 2011 481 482  /// List of all stationary problems that are managed by this mesh /// distributor.  Thomas Witkowski committed Apr 21, 2011 483  vector problemStat;  Thomas Witkowski committed Feb 08, 2010 484   Thomas Witkowski committed Mar 28, 2011 485 486 487  /// If true, the mesh distributor is already initialized; bool initialized;  Thomas Witkowski committed Apr 03, 2009 488 489 490 491 492 493  /// The rank of the current process. int mpiRank; /// Overall number of processes. int mpiSize;  Thomas Witkowski committed Nov 14, 2011 494 495  /// MPI communicator collected all processes, which should be used for /// calculation. The Debug procces is not included in this communicator.  Thomas Witkowski committed Apr 03, 2009 496 497 498  MPI::Intracomm mpiComm; /// Name of the problem (as used in the init files)  Thomas Witkowski committed Jan 07, 2011 499  string name;  Thomas Witkowski committed Apr 03, 2009 500   Thomas Witkowski committed Oct 23, 2012 501 502  /// Set of all different FE spaces. vector feSpaces;  Thomas Witkowski committed Oct 19, 2012 503   504 505 506  /// Pointer to the only mesh. Note that we do not yet support multi mesh /// method, thus even if we consider coupled problems, all problems must /// be defined on the same mesh.  Thomas Witkowski committed Apr 03, 2009 507 508  Mesh *mesh;  Thomas Witkowski committed Feb 13, 2012 509 510 511  /// A refinement manager that should be used on the mesh. It is used to /// refine elements at interior boundaries in order to fit together with /// elements on the other side of the interior boundary.  512 513  RefinementManager *refineManager;  Thomas Witkowski committed Nov 14, 2011 514 515  /// Pointer to a mesh partitioner that is used to partition the mesh to /// the ranks.  Thomas Witkowski committed Jan 24, 2011 516  MeshPartitioner *partitioner;  Thomas Witkowski committed Apr 03, 2009 517   Thomas Witkowski committed Nov 14, 2011 518 519  /// Weights for the elements, i.e., the number of leaf elements within /// this element.  Thomas Witkowski committed Jan 07, 2011 520  map elemWeights;  Thomas Witkowski committed Apr 03, 2009 521   Thomas Witkowski committed Feb 13, 2012 522 523  /// Stores to every macro element index the number of the rank that owns this /// macro element.  Thomas Witkowski committed Jan 24, 2011 524  map partitionMap;  Thomas Witkowski committed Apr 03, 2009 525   Thomas Witkowski committed Feb 13, 2012 526  /// Database to store and query all sub-objects of all elements of the  Thomas Witkowski committed Nov 14, 2011 527  /// macro mesh.  Thomas Witkowski committed Feb 13, 2012 528  ElementObjectDatabase elObjDb;  Thomas Witkowski committed Nov 29, 2010 529   Thomas Witkowski committed Feb 13, 2012 530  /// Defines the interior boundaries of the domain that result from  Thomas Witkowski committed Apr 27, 2012 531 532  /// partitioning the whole mesh. InteriorBoundary intBoundary;  Thomas Witkowski committed Oct 14, 2009 533   Thomas Witkowski committed May 02, 2012 534 535  InteriorBoundary intBoundarySd;  Thomas Witkowski committed Apr 30, 2012 536  DofComm dofComm;  537   Thomas Witkowski committed May 02, 2012 538 539  DofComm dofCommSd;  Thomas Witkowski committed Jan 26, 2012 540  PeriodicMap periodicMap;  Thomas Witkowski committed Jul 08, 2010 541   Thomas Witkowski committed Jan 19, 2011 542 543 544 545  /// This set of values must be interchanged between ranks when the mesh is /// repartitioned. vector*> interchangeVectors;  Thomas Witkowski committed Feb 13, 2012 546 547 548 549  /// If the problem definition has been read from a serialization file, this /// variable is true, otherwise it is false. This variable is used to stop the /// initialization function, if the problem definition has already been read /// from a serialization file.  550  bool deserialized;  Thomas Witkowski committed Dec 08, 2009 551   552 553 554  /// Denotes whether there exists a filewriter for this object. bool writeSerializationFile;  Thomas Witkowski committed Nov 26, 2010 555 556 557  /// If true, it is possible to repartition the mesh during computations. bool repartitioningAllowed;  Thomas Witkowski committed Nov 14, 2011 558 559  /// Stores the number of mesh changes that must lie in between to /// repartitionings.  Thomas Witkowski committed Dec 20, 2010 560 561  int repartitionIthChange;  Thomas Witkowski committed Nov 09, 2012 562 563 564  /// int repartitioningWaitAfterFail;  Thomas Witkowski committed Nov 14, 2011 565 566  /// Counts the number of mesh changes after the last mesh repartitioning /// was done.  567  int nMeshChangesAfterLastRepartitioning;  Thomas Witkowski committed Dec 03, 2010 568   569 570 571  /// Countes the number of mesh repartitions that were done. Till now, this /// variable is used only for debug outputs. int repartitioningCounter;  Thomas Witkowski committed Dec 03, 2010 572   Thomas Witkowski committed Jun 26, 2012 573 574 575 576 577  /// If repartitioning of the mesh fail, this variable has a positive value /// that gives the number of mesh changes the mesh distributer will wait /// before trying new mesh repartitioning. int repartitioningFailed;  Thomas Witkowski committed Dec 16, 2010 578  /// Directory name where all debug output files should be written to.  Thomas Witkowski committed Jan 07, 2011 579  string debugOutputDir;  Thomas Witkowski committed Dec 16, 2010 580   Thomas Witkowski committed Feb 13, 2012 581 582  /// Stores the mesh change index. This is used to recognize changes in the /// mesh structure (e.g. through refinement or coarsening managers).  Thomas Witkowski committed Dec 08, 2009 583  long lastMeshChangeIndex;  584   Thomas Witkowski committed Nov 14, 2011 585 586 587 588  /// Stores for all macro elements of the original macro mesh the /// neighbourhood information based on element indices. Thus, each macro /// element index is mapped to a vector containing all indices of /// neighbouring macro elements.  Thomas Witkowski committed Jan 07, 2011 589  map > macroElementNeighbours;  Thomas Witkowski committed Nov 25, 2010 590   Thomas Witkowski committed Nov 14, 2011 591 592  /// Store all macro elements of the overall mesh, i.e., before the /// mesh is redistributed for the first time.  Thomas Witkowski committed Jan 07, 2011 593  vector allMacroElements;  594   Thomas Witkowski committed Apr 08, 2011 595 596  Flag createBoundaryDofFlag;  Thomas Witkowski committed Apr 16, 2012 597 598 599  /// Stores on each mesh level for all FE spaces the information about /// all boundary DOFs. vector > boundaryDofInfo;  Thomas Witkowski committed May 19, 2011 600   601 602  /// Stores information about hierarchical decomposition of the mesh levels. /// Is used to specify multi level solver methods.  Thomas Witkowski committed Apr 11, 2012 603 604  MeshLevelData levelData;  Thomas Witkowski committed Sep 06, 2012 605 606 607 608 609 610  /// If there is no mesh adaptivity, the mesh distributor can remove some /// data structures which are only used if mesh changes or it must be /// redistributed due to some local adaptivity. By default, this variable /// is set to true, and thus no special assumption are made. bool meshAdaptivity;  Thomas Witkowski committed Oct 24, 2012 611 612 613 614 615  /// Specifies whether the global domain has periodic boundaries. Thus, this /// variable is not related to rank's subdomain but to the global problem /// and therefore the value if the same on all ranks. bool hasPeriodicBoundary;  616 617 618 619  /// Set of all parallel DOF mapping object that are registered by parallel /// solver objects and must be updated automatically after mesh change. vector dofMaps;  Thomas Witkowski committed Oct 30, 2012 620 621 622  /// If true, detailed timings for benchmarking are printed. bool printTimings;  Thomas Witkowski committed Apr 08, 2011 623  public:  Thomas Witkowski committed Apr 21, 2011 624 625 626  /// The boundary DOFs are sorted by subobject entities, i.e., first all /// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// communication structure vectors, \ref sendDofs and \ref recvDofs.  Thomas Witkowski committed Apr 08, 2011 627 628  static const Flag BOUNDARY_SUBOBJ_SORTED;  Thomas Witkowski committed Apr 21, 2011 629 630 631 632 633 634 635 636 637  /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will send to other ranks (thus, rank /// owned DOFs. static const Flag BOUNDARY_FILL_INFO_SEND_DOFS; /// When boundary DOFs are created, \ref boundaryDofInfo is filled for /// all DOFs that this rank will receive from other ranks (thus, DOFs /// that are owned by another rank). static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;  Thomas Witkowski committed Apr 08, 2011 638   Thomas Witkowski committed May 19, 2011 639 640  static MeshDistributor *globalMeshDistributor;  641  friend class ParallelDebug;  Thomas Witkowski committed Apr 03, 2009 642 643 644  }; }  645 #endif // AMDIS_MESHDISTRIBUTOR_H