ParallelDomainBase.h 15.4 KB
 Thomas Witkowski committed Apr 03, 2009 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 // ============================================================================ // == == // == AMDiS - Adaptive multidimensional simulations == // == == // ============================================================================ // == == // == crystal growth group == // == == // == Stiftung caesar == // == Ludwig-Erhard-Allee 2 == // == 53175 Bonn == // == germany == // == == // ============================================================================ // == == // == http://www.caesar.de/cg/AMDiS == // == == // ============================================================================  Thomas Witkowski committed Jul 09, 2009 20 21 22 23 /** \file ParallelDomainBase.h */ #ifndef AMDIS_PARALLELDOMAINBASE_H #define AMDIS_PARALLELDOMAINBASE_H  Thomas Witkowski committed Apr 03, 2009 24 25 26  #include  Thomas Witkowski committed Jun 11, 2009 27 #include  Thomas Witkowski committed Apr 03, 2009 28 29 30 31 #include #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h"  Thomas Witkowski committed Apr 15, 2009 32 #include "FiniteElemSpace.h"  Thomas Witkowski committed Apr 03, 2009 33 #include "AdaptInfo.h"  Thomas Witkowski committed Apr 27, 2009 34 #include "InteriorBoundary.h"  Thomas Witkowski committed Jun 12, 2009 35 36 #include "AMDiS_fwd.h"  Thomas Witkowski committed Jun 09, 2009 37 38 #include "petsc.h" #include "petscsys.h"  Thomas Witkowski committed Apr 07, 2009 39 #include "petscao.h"  Thomas Witkowski committed Apr 03, 2009 40 41 #include "mpi.h"  Thomas Witkowski committed Jul 09, 2009 42 43 #include "Global.h"  Thomas Witkowski committed Apr 03, 2009 44 45 46 47 namespace AMDiS { class ParMetisPartitioner;  Thomas Witkowski committed Jun 26, 2009 48  class ParallelDomainBase : public ProblemIterationInterface,  Thomas Witkowski committed Jul 09, 2009 49  public ProblemTimeInterface  Thomas Witkowski committed Apr 03, 2009 50  {  Thomas Witkowski committed Jun 29, 2009 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66  private: /// Defines type for a vector of DOFs. typedef std::vector DofContainer; /// Defines a mapping type from DOFs to rank numbers. typedef std::map DofToRank; /// Defines a mapping type from DOFs to a set of rank numbers. typedef std::map > DofToPartitions; /// Defines a mapping type from rank numbers to sets of DOFs. typedef std::map RankToDofContainer; /// Defines a mapping type from DOF indices to DOF indices. typedef std::map DofMapping;  Thomas Witkowski committed Aug 13, 2009 67 68 69  /// Defines a mapping type from DOFs to boolean values. typedef std::map DofToBool;  Thomas Witkowski committed Jun 29, 2009 70  /// Defines a mapping type from DOF indices to boolean values.  Thomas Witkowski committed Aug 13, 2009 71  typedef std::map DofIndexToBool;  Thomas Witkowski committed Jun 29, 2009 72   Thomas Witkowski committed Jul 01, 2009 73 74 75 76 77 78  /// Defines a mapping type from rank numbers to sets of coordinates. typedef std::map > > RankToCoords; /// Forward type (it maps rank numbers to the interior boundary objects). typedef InteriorBoundary::RankToBoundMap RankToBoundMap;  Thomas Witkowski committed Jul 01, 2009 79 80  typedef std::map ElementIdxToDofs;  Thomas Witkowski committed Jul 07, 2009 81 82  typedef std::map DofIndexMap;  Thomas Witkowski committed Apr 03, 2009 83  public:  Thomas Witkowski committed Jul 20, 2009 84  ParallelDomainBase(std::string name,  Thomas Witkowski committed Jun 29, 2009 85 86 87 88  ProblemIterationInterface *iterationIF, ProblemTimeInterface *timeIF, FiniteElemSpace *feSpace, RefinementManager *refineManager);  Thomas Witkowski committed Apr 03, 2009 89   Thomas Witkowski committed Jun 26, 2009 90  virtual ~ParallelDomainBase() {}  Thomas Witkowski committed Apr 03, 2009 91   Thomas Witkowski committed Jun 11, 2009 92  virtual void initParallelization(AdaptInfo *adaptInfo);  Thomas Witkowski committed Apr 03, 2009 93   Thomas Witkowski committed Jun 26, 2009 94 95  void exitParallelization(AdaptInfo *adaptInfo);  Thomas Witkowski committed Jul 08, 2009 96 97  void updateDofAdmins();  Thomas Witkowski committed Jun 26, 2009 98 99 100 101 102 103 104  /** \brief * Test, if the mesh consists of macro elements only. The mesh partitioning of * the parallelization works for macro meshes only and would fail, if the mesh * is already refined in some way. Therefore, this function will exit the program * if it finds a non macro element in the mesh. */ void testForMacroMesh();  Thomas Witkowski committed Apr 03, 2009 105 106 107 108 109 110  /// Set for each element on the partitioning level the number of leaf elements. double setElemWeights(AdaptInfo *adaptInfo); void partitionMesh(AdaptInfo *adaptInfo);  Thomas Witkowski committed Jul 10, 2009 111 112 113 114 115  virtual void setTime(AdaptInfo *adaptInfo) { if (timeIF) timeIF->setTime(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 116   Thomas Witkowski committed Jul 10, 2009 117 118 119 120 121  virtual void initTimestep(AdaptInfo *adaptInfo) { if (timeIF) timeIF->initTimestep(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 122   Thomas Witkowski committed Jul 10, 2009 123 124 125 126 127  virtual void closeTimestep(AdaptInfo *adaptInfo) { if (timeIF) timeIF->closeTimestep(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 128   Thomas Witkowski committed Jul 10, 2009 129  virtual void solveInitialProblem(AdaptInfo *adaptInfo)  Thomas Witkowski committed Aug 31, 2009 130  {  Thomas Witkowski committed Jul 10, 2009 131 132 133  if (timeIF) timeIF->solveInitialProblem(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 134   Thomas Witkowski committed Jul 10, 2009 135 136 137 138 139  virtual void transferInitialSolution(AdaptInfo *adaptInfo) { if (timeIF) timeIF->transferInitialSolution(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 140   Thomas Witkowski committed Jun 10, 2009 141 142  virtual void beginIteration(AdaptInfo *adaptInfo) {  Thomas Witkowski committed Apr 07, 2009 143 144 145  iterationIF->beginIteration(adaptInfo); }  Thomas Witkowski committed Jul 09, 2009 146  virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION);  Thomas Witkowski committed Apr 07, 2009 147   Thomas Witkowski committed Jun 10, 2009 148 149  virtual void endIteration(AdaptInfo *adaptInfo) {  Thomas Witkowski committed Apr 07, 2009 150 151  iterationIF->endIteration(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 152   Thomas Witkowski committed Jul 09, 2009 153 154  virtual void solve() {}  Thomas Witkowski committed Jun 11, 2009 155 156 157 158  virtual int getNumProblems() { return 0; }  Thomas Witkowski committed Apr 03, 2009 159   Thomas Witkowski committed Jul 20, 2009 160  inline virtual std::string getName()  Thomas Witkowski committed Jun 10, 2009 161  {  Thomas Witkowski committed Apr 03, 2009 162 163 164  return name; }  Thomas Witkowski committed Apr 07, 2009 165  /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.  Thomas Witkowski committed Sep 25, 2009 166  int getNumberRankDofs()  Thomas Witkowski committed Jun 10, 2009 167  {  Thomas Witkowski committed Sep 25, 2009 168  return nRankDofs;  Thomas Witkowski committed Apr 07, 2009 169 170  }  Thomas Witkowski committed Jun 09, 2009 171 172  void fillPetscMatrix(DOFMatrix *mat, DOFVector *vec);  Thomas Witkowski committed Jul 09, 2009 173 174  void fillPetscMatrix(Matrix *mat, SystemVector *vec);  Thomas Witkowski committed Jun 09, 2009 175 176  void solvePetscMatrix(DOFVector *vec);  Thomas Witkowski committed Aug 31, 2009 177  void solvePetscMatrix(SystemVector &vec);  Thomas Witkowski committed Jul 09, 2009 178   Thomas Witkowski committed Jun 11, 2009 179 180 181 182  virtual ProblemStatBase *getProblem(int number = 0) { return NULL; }  Thomas Witkowski committed Apr 03, 2009 183   Thomas Witkowski committed Sep 28, 2009 184  // Writes all data of this object to an output stream.  Thomas Witkowski committed Sep 25, 2009 185  virtual void serialize(std::ostream &out);  Thomas Witkowski committed Jun 11, 2009 186   Thomas Witkowski committed Sep 28, 2009 187  // Reads the object data from an input stream.  Thomas Witkowski committed Sep 25, 2009 188  virtual void deserialize(std::istream &in);  Thomas Witkowski committed Jun 11, 2009 189 190  protected:  Thomas Witkowski committed Jun 11, 2009 191 192 193 194  /** \brief * Determine the interior boundaries, i.e. boundaries between ranks, and store * all information about them in \ref interiorBoundary. */  Thomas Witkowski committed Jul 09, 2009 195  void createInteriorBoundaryInfo(DofContainer& rankDOFs);  Thomas Witkowski committed Jun 11, 2009 196 197 198 199  /// Removes all macro elements from the mesh that are not part of ranks partition. void removeMacroElements();  Thomas Witkowski committed Jun 16, 2009 200 201 202 203  /** \brief * Creates from a macro mesh a correct local and global DOF index numbering. *  Thomas Witkowski committed Jun 29, 2009 204  * \param[out] rankDOFs Returns all DOFs from the macro mesh, which are owned  Thomas Witkowski committed Jun 16, 2009 205  * by the rank after partitioning the macro mesh.  Thomas Witkowski committed Jun 29, 2009 206 207  * \param[out] nRankDOFs Number of DOFs owned by rank. * \param[out] nOverallDOFs Number of all DOFs in macro mesh.  Thomas Witkowski committed Jun 16, 2009 208  */  Thomas Witkowski committed Jun 29, 2009 209  void createLocalGlobalNumbering(DofContainer& rankDOFs,  Thomas Witkowski committed Jun 12, 2009 210 211 212 213  int& nRankDOFs, int& nOverallDOFs); void updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs);  Thomas Witkowski committed Jun 11, 2009 214   Thomas Witkowski committed Jul 09, 2009 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231  /** \brief * This function create new mappings from local to global indices, * \ref mapLocalGlobalDOFs, and from local to dof indices, \ref mapLocalToDofIndex. * Furthermore, using the first argument the dof indices in ranks partition are * changed. * * \param[in] rankDofsNewLocalIndex Map from dof pointers of all dofs in rank * to new dof indices. * \param[in] rankOwnedDofsNewLocalIndex Map from dof pointers of dofs owned by * the rank to the new local index. * \param[in] rankDofsNewGlobalIndex Map from dof pointers of all dofs in rank * to the new global index. */ void createLocalMappings(DofIndexMap &rankDofsNewLocalIndex, DofIndexMap &rankOwnedDofsNewLocalIndex, DofIndexMap &rankDofsNewGlobalIndex);  Thomas Witkowski committed Jun 29, 2009 232 233 234  void addAllVertexDOFs(Element *el, int ithEdge, DofContainer& dofs); void addAllEdgeDOFs(Element *el, int ithEdge, DofContainer& dofs);  Thomas Witkowski committed Jun 15, 2009 235   Thomas Witkowski committed Jun 11, 2009 236 237  /** \brief * This function traverses the whole mesh, i.e. before it is really partitioned,  Thomas Witkowski committed Jun 15, 2009 238 239 240  * and collects information about which DOF corresponds to which rank. Can only * be used, if \ref partitionVec is set correctly. This is only the case, when * the macro mesh is partitioned.  Thomas Witkowski committed Jun 11, 2009 241  *  Thomas Witkowski committed Jun 29, 2009 242  * \param[out] partionDOFs Stores to each DOF pointer the set of ranks the DOF is  Thomas Witkowski committed Jun 11, 2009 243  * part of.  Thomas Witkowski committed Jun 29, 2009 244 245  * \param[out] rankDOFs Stores all rank DOFs. * \param[out] boundaryDOFs Stores all DOFs in ranks partition that are on an  Thomas Witkowski committed Jun 11, 2009 246 247  * interior boundary but correspond to another rank. */  Thomas Witkowski committed Jul 03, 2009 248 249 250  void createDOFMemberInfo(DofToPartitions& partitionDofs, DofContainer& rankOwnedDofs, DofContainer& rankAllDofs,  Thomas Witkowski committed Aug 13, 2009 251 252  DofToRank& boundaryDofs, DofToBool& vertexDof);  Thomas Witkowski committed Jun 22, 2009 253   Thomas Witkowski committed Jul 09, 2009 254 255 256 257 258  void setDofMatrix(DOFMatrix* mat, int dispMult = 1, int dispAddRow = 0, int dispAddCol = 0); void setDofVector(DOFVector* vec, int disMult = 1, int dispAdd = 0);  Thomas Witkowski committed Jul 01, 2009 259 260 261  void DbgCreateElementMap(ElementIdxToDofs &elMap); void DbgTestElementMap(ElementIdxToDofs &elMap);  Thomas Witkowski committed Jul 01, 2009 262 263  void DbgTestInteriorBoundary();  Thomas Witkowski committed Jun 11, 2009 264   Thomas Witkowski committed Jun 22, 2009 265 266 267 268 269  /** \brief * This function is used for debugging only. It traverses all interior boundaries * and compares the dof indices on them with the dof indices of the boundarys * neighbours. The function fails, when dof indices on an interior boundary does * not fit together.  Thomas Witkowski committed Jul 08, 2009 270 271 272  * * \param printCoords If true, the coords of all common dofs are printed to * the screen.  Thomas Witkowski committed Jun 22, 2009 273  */  Thomas Witkowski committed Jul 08, 2009 274  void DbgTestCommonDofs(bool printCoords = false);  Thomas Witkowski committed Jun 11, 2009 275   Thomas Witkowski committed Aug 17, 2009 276 277 278 279 280 281 282  /** \brief * This functions create a Paraview file with the macro mesh where the elements * are colored by the partition they are part of. This function can be used for * debugging. */ void writePartitioningMesh(std::string filename);  Thomas Witkowski committed Aug 31, 2009 283 284 285 286 287 288 289 290 291 292 293 294  /** \brief * This function must be used if the values of a SystemVector must be synchronised * over all ranks. That means, that each rank sends the values of the DOFs, which * are owned by the rank and lie on an interior bounday, to all other ranks also * having this DOF. * * This function must be used, for example, after the lineary system is solved, or * after the SystemVector is set by some user defined functions, e.g., initial * solution functions. */ void synchVectors(SystemVector &vec);  Thomas Witkowski committed Sep 28, 2009 295 296 297 298 299 300 301 302 303 304 305 306 307 308  /// Writes a vector of dof pointers to an output stream. void serialize(std::ostream &out, DofContainer &data); /// Reads a vector of dof pointers from an input stream. void deserialize(std::istream &in, DofContainer &data, std::map &dofMap); /// Writes a \ref RankToDofContainer to an output stream. void serialize(std::ostream &out, RankToDofContainer &data); /// Reads a \ref RankToDofContainer from an input stream. void deserialize(std::istream &in, RankToDofContainer &data, std::map &dofMap);  Thomas Witkowski committed Jul 01, 2009 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337  inline void orderDOFs(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2, const DegreeOfFreedom* dof3, DofContainer &vec) { vec.resize(3); if (*dof1 < *dof2 && *dof1 < *dof3) vec[0] = dof1; else if (*dof2 < *dof1 && *dof2 < *dof3) vec[0] = dof2; else vec[0] = dof3; if (*dof1 > *dof2 && *dof1 > *dof3) vec[2] = dof1; else if (*dof2 > *dof1 && *dof2 > *dof3) vec[2] = dof2; else vec[2] = dof3; if (dof1 != vec[0] && dof1 != vec[2]) vec[1] = dof1; else if (dof2 != vec[0] && dof2 != vec[2]) vec[1] = dof2; else vec[1] = dof3; }  Thomas Witkowski committed Apr 03, 2009 338  protected:  Thomas Witkowski committed Apr 07, 2009 339 340 341 342 343 344  /// ProblemIterationInterface *iterationIF; /// ProblemTimeInterface *timeIF;  Thomas Witkowski committed Apr 03, 2009 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360  /// The rank of the current process. int mpiRank; /// Overall number of processes. int mpiSize; /** \brief * MPI communicator collected all processes, which should * be used for calculation. The Debug procces is not included * in this communicator. */ MPI::Intracomm mpiComm; /// Name of the problem (as used in the init files) std::string name;  Thomas Witkowski committed Apr 15, 2009 361 362 363  /// Finite element space of the problem. FiniteElemSpace *feSpace;  Thomas Witkowski committed Apr 03, 2009 364 365 366  /// Mesh of the problem. Mesh *mesh;  Thomas Witkowski committed Jun 26, 2009 367 368 369  /// Info level. int info;  Thomas Witkowski committed Jun 12, 2009 370 371 372  /// Refinement manager for the mesh. RefinementManager *refinementManager;  Thomas Witkowski committed Apr 03, 2009 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392  /// Pointer to the paritioner which is used to devide a mesh into partitions. ParMetisPartitioner *partitioner; /// Weights for the elements, i.e., the number of leaf elements within this element. std::map elemWeights; /// Is true, if the mesh was not partitioned before, otherwise it's false. bool initialPartitionMesh; /** \brief * Stores to every coarse element index the number of the partition it * corresponds to. */ std::map partitionVec; /** \brief * Stores an old partitioning of elements. To every element index the number * of the parition it corresponds to is stored. */ std::map oldPartitionVec;  Thomas Witkowski committed Apr 07, 2009 393   Thomas Witkowski committed Sep 25, 2009 394  /// Petsc's matrix structure.  Thomas Witkowski committed Jun 09, 2009 395 396  Mat petscMatrix;  Thomas Witkowski committed Sep 25, 2009 397 398 399 400 401  /** \brief * Petsc's vector structures for the rhs vector, the solution vector and a * temporary vector for calculating the final residuum. */ Vec petscRhsVec, petscSolVec, petscTmpVec;  Thomas Witkowski committed Jun 09, 2009 402   Thomas Witkowski committed Apr 07, 2009 403  /// Number of DOFs in the rank mesh.  Thomas Witkowski committed Sep 25, 2009 404  int nRankDofs;  Thomas Witkowski committed Apr 27, 2009 405 406  /** \brief  Thomas Witkowski committed Jun 15, 2009 407 408 409 410 411 412 413 414 415 416 417 418  * Defines the interior boundaries of the domain that result from partitioning * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e., * the object gives for every neighbour rank i the boundaries this rank owns and * shares with rank i. */ InteriorBoundary myIntBoundary; /** \brief * Defines the interior boundaries of the domain that result from partitioning * the whole mesh. Contains only the boundaries, which are not owned by the rank, * i.e., the object gives for every neighbour rank i the boundaries that are * owned by rank i and are shared with this rank.  Thomas Witkowski committed Apr 27, 2009 419  */  Thomas Witkowski committed Jun 15, 2009 420  InteriorBoundary otherIntBoundary;  Thomas Witkowski committed Apr 27, 2009 421   Thomas Witkowski committed Jun 10, 2009 422 423 424 425  /** \brief * This map contains for each rank the list of dofs the current rank must send * to exchange solution dofs at the interior boundaries. */  Thomas Witkowski committed Jun 29, 2009 426  RankToDofContainer sendDofs;  Thomas Witkowski committed Jun 10, 2009 427 428 429 430 431  /** \brief * This map contains for each rank the list of dofs from which the current rank * must receive solution values of dofs at the interior boundaries. */  Thomas Witkowski committed Jun 29, 2009 432  RankToDofContainer recvDofs;  Thomas Witkowski committed Jun 11, 2009 433 434  /// Maps local to global dof indices.  Thomas Witkowski committed Jun 29, 2009 435  DofMapping mapLocalGlobalDOFs;  Thomas Witkowski committed Jun 11, 2009 436   Thomas Witkowski committed Jul 20, 2009 437  /// Maps local dof indices to real dof indices.  Thomas Witkowski committed Jul 08, 2009 438  DofMapping mapLocalToDofIndex;  Thomas Witkowski committed Jun 11, 2009 439 440 441 442 443 444  /** \brief * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is * owned by the rank. Otherwise, its an interior boundary DOF that is owned by * another rank. */  Thomas Witkowski committed Aug 13, 2009 445 446 447  DofIndexToBool isRankDof; DofToBool vertexDof;  Thomas Witkowski committed Jul 03, 2009 448   Thomas Witkowski committed Sep 25, 2009 449  /// Is the index of the first row of the linear system, which is owned by the rank.  Thomas Witkowski committed Jul 03, 2009 450  int rstart;  Thomas Witkowski committed Jul 09, 2009 451   Thomas Witkowski committed Sep 25, 2009 452 453 454 455  /** \brief * Number of components of the equation. Is used to calculate the exact number * of rows in the the overall linear system. */  Thomas Witkowski committed Jul 09, 2009 456  int nComponents;  Thomas Witkowski committed Jul 23, 2009 457   Thomas Witkowski committed Sep 25, 2009 458  /// Number of rows of the whole linear system that are stored on this rank.  Thomas Witkowski committed Jul 23, 2009 459 460  int nRankRows;  Thomas Witkowski committed Sep 25, 2009 461  /// Overall number of the rows in the lineary system.  Thomas Witkowski committed Jul 23, 2009 462  int nOverallRows;  Thomas Witkowski committed Apr 03, 2009 463 464 465 466  }; }  Thomas Witkowski committed Jul 09, 2009 467 #endif // AMDIS_PARALLELDOMAINBASE_H