ParallelDomainProblem.h 10.2 KB
 Thomas Witkowski committed Apr 03, 2009 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 // ============================================================================ // == == // == AMDiS - Adaptive multidimensional simulations == // == == // ============================================================================ // == == // == crystal growth group == // == == // == Stiftung caesar == // == Ludwig-Erhard-Allee 2 == // == 53175 Bonn == // == germany == // == == // ============================================================================ // == == // == http://www.caesar.de/cg/AMDiS == // == == // ============================================================================ /** \file ParallelDomainProblem.h */ #ifndef AMDIS_PARALLELDOMAINPROBLEM_H #define AMDIS_PARALLELDOMAINPROBLEM_H #include  Thomas Witkowski committed Jun 11, 2009 26 #include  Thomas Witkowski committed Apr 03, 2009 27 28 29 30 #include #include "ProblemTimeInterface.h" #include "ProblemIterationInterface.h"  Thomas Witkowski committed Apr 15, 2009 31 #include "FiniteElemSpace.h"  Thomas Witkowski committed Apr 03, 2009 32 #include "AdaptInfo.h"  Thomas Witkowski committed Apr 27, 2009 33 #include "InteriorBoundary.h"  Thomas Witkowski committed Jun 12, 2009 34 35 #include "AMDiS_fwd.h"  Thomas Witkowski committed Jun 09, 2009 36 37 #include "petsc.h" #include "petscsys.h"  Thomas Witkowski committed Apr 07, 2009 38 #include "petscao.h"  Thomas Witkowski committed Apr 03, 2009 39 40 41 42 43 44 45 46 47 48 49 50 51 #include "mpi.h" namespace AMDiS { class ParMetisPartitioner; class ParallelDomainProblemBase : public ProblemIterationInterface, public ProblemTimeInterface { public: ParallelDomainProblemBase(const std::string& name, ProblemIterationInterface *iterationIF, ProblemTimeInterface *timeIF,  Thomas Witkowski committed Jun 12, 2009 52 53  FiniteElemSpace *feSpace, RefinementManager *refineManager);  Thomas Witkowski committed Apr 03, 2009 54 55 56  virtual ~ParallelDomainProblemBase() {}  Thomas Witkowski committed Jun 11, 2009 57  virtual void initParallelization(AdaptInfo *adaptInfo);  Thomas Witkowski committed Apr 03, 2009 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75  void exitParallelization(AdaptInfo *adaptInfo); /// Set for each element on the partitioning level the number of leaf elements. double setElemWeights(AdaptInfo *adaptInfo); void partitionMesh(AdaptInfo *adaptInfo); virtual void setTime(AdaptInfo *adaptInfo) {} virtual void initTimestep(AdaptInfo *adaptInfo) {} virtual void closeTimestep(AdaptInfo *adaptInfo) {} virtual void solveInitialProblem(AdaptInfo *adaptInfo) {} virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}  Thomas Witkowski committed Jun 10, 2009 76 77  virtual void beginIteration(AdaptInfo *adaptInfo) {  Thomas Witkowski committed Apr 07, 2009 78 79 80  iterationIF->beginIteration(adaptInfo); }  Thomas Witkowski committed Jun 10, 2009 81 82  virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) {  Thomas Witkowski committed Jun 09, 2009 83 84 85  ERROR_EXIT("Not implemented!\n"); return 0; }  Thomas Witkowski committed Apr 07, 2009 86   Thomas Witkowski committed Jun 10, 2009 87 88  virtual void endIteration(AdaptInfo *adaptInfo) {  Thomas Witkowski committed Apr 07, 2009 89 90  iterationIF->endIteration(adaptInfo); }  Thomas Witkowski committed Apr 03, 2009 91   Thomas Witkowski committed Jun 11, 2009 92 93 94 95  virtual int getNumProblems() { return 0; }  Thomas Witkowski committed Apr 03, 2009 96   Thomas Witkowski committed Jun 10, 2009 97 98  inline virtual const std::string& getName() {  Thomas Witkowski committed Apr 03, 2009 99 100 101  return name; }  Thomas Witkowski committed Apr 07, 2009 102  /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.  Thomas Witkowski committed Jun 10, 2009 103 104  int getNumberRankDOFs() {  Thomas Witkowski committed Apr 07, 2009 105 106 107  return nRankDOFs; }  Thomas Witkowski committed Jun 09, 2009 108 109  void fillPetscMatrix(DOFMatrix *mat, DOFVector *vec);  Thomas Witkowski committed Jun 09, 2009 110 111  void solvePetscMatrix(DOFVector *vec);  Thomas Witkowski committed Jun 11, 2009 112 113 114 115  virtual ProblemStatBase *getProblem(int number = 0) { return NULL; }  Thomas Witkowski committed Apr 03, 2009 116 117 118 119  virtual void serialize(std::ostream&) {} virtual void deserialize(std::istream&) {}  Thomas Witkowski committed Jun 11, 2009 120 121 122  protected:  Thomas Witkowski committed Jun 11, 2009 123 124 125 126  /** \brief * Determine the interior boundaries, i.e. boundaries between ranks, and store * all information about them in \ref interiorBoundary. */  Thomas Witkowski committed Jun 12, 2009 127 128  void createInteriorBoundaryInfo(std::vector& rankDOFs, std::map& boundaryDOFs);  Thomas Witkowski committed Jun 11, 2009 129 130 131 132  /// Removes all macro elements from the mesh that are not part of ranks partition. void removeMacroElements();  Thomas Witkowski committed Jun 16, 2009 133 134 135 136 137 138 139 140 141 142 143 144  /** \brief * Creates from a macro mesh a correct local and global DOF index numbering. * * @param[out] rankDOFs Returns all DOFs from the macro mesh, which are owned * by the rank after partitioning the macro mesh. * @param[out] boundaryDOFs Returns all DOFs from the macro mesh, which lies at * an interior boundary of the rank. This object maps * each such DOF to the rank that owns this DOF. * @param[out] nRankDOFs Number of DOFs owned by rank. * @param[out] nOverallDOFs Number of all DOFs in macro mesh. */  Thomas Witkowski committed Jun 12, 2009 145 146 147 148 149 150  void createLocalGlobalNumbering(std::vector& rankDOFs, std::map& boundaryDOFs, int& nRankDOFs, int& nOverallDOFs); void updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs);  Thomas Witkowski committed Jun 11, 2009 151   Thomas Witkowski committed Jun 15, 2009 152  void addAllDOFs(Element *el, int ithEdge,  Thomas Witkowski committed Jun 16, 2009 153  std::vector& dofs);  Thomas Witkowski committed Jun 15, 2009 154   Thomas Witkowski committed Jun 11, 2009 155 156  /** \brief * This function traverses the whole mesh, i.e. before it is really partitioned,  Thomas Witkowski committed Jun 15, 2009 157 158 159  * and collects information about which DOF corresponds to which rank. Can only * be used, if \ref partitionVec is set correctly. This is only the case, when * the macro mesh is partitioned.  Thomas Witkowski committed Jun 11, 2009 160 161 162 163 164 165 166  * * @param[out] partionDOFs Stores to each DOF pointer the set of ranks the DOF is * part of. * @param[out] rankDOFs Stores all rank DOFs. * @param[out] boundaryDOFs Stores all DOFs in ranks partition that are on an * interior boundary but correspond to another rank. */  Thomas Witkowski committed Jun 11, 2009 167  void createDOFMemberInfo(std::map >& partitionDOFs,  Thomas Witkowski committed Jun 11, 2009 168 169  std::vector& rankDOFs, std::map& boundaryDOFs);  Thomas Witkowski committed Jun 22, 2009 170   Thomas Witkowski committed Jun 11, 2009 171   Thomas Witkowski committed Jun 22, 2009 172 173 174 175 176 177 178  /** \brief * This function is used for debugging only. It traverses all interior boundaries * and compares the dof indices on them with the dof indices of the boundarys * neighbours. The function fails, when dof indices on an interior boundary does * not fit together. */ void testInteriorBoundary();  Thomas Witkowski committed Jun 11, 2009 179   Thomas Witkowski committed Apr 03, 2009 180  protected:  Thomas Witkowski committed Apr 07, 2009 181 182 183 184 185 186  /// ProblemIterationInterface *iterationIF; /// ProblemTimeInterface *timeIF;  Thomas Witkowski committed Apr 03, 2009 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202  /// The rank of the current process. int mpiRank; /// Overall number of processes. int mpiSize; /** \brief * MPI communicator collected all processes, which should * be used for calculation. The Debug procces is not included * in this communicator. */ MPI::Intracomm mpiComm; /// Name of the problem (as used in the init files) std::string name;  Thomas Witkowski committed Apr 15, 2009 203 204 205  /// Finite element space of the problem. FiniteElemSpace *feSpace;  Thomas Witkowski committed Apr 03, 2009 206 207 208  /// Mesh of the problem. Mesh *mesh;  Thomas Witkowski committed Jun 12, 2009 209 210 211  /// Refinement manager for the mesh. RefinementManager *refinementManager;  Thomas Witkowski committed Apr 03, 2009 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231  /// Pointer to the paritioner which is used to devide a mesh into partitions. ParMetisPartitioner *partitioner; /// Weights for the elements, i.e., the number of leaf elements within this element. std::map elemWeights; /// Is true, if the mesh was not partitioned before, otherwise it's false. bool initialPartitionMesh; /** \brief * Stores to every coarse element index the number of the partition it * corresponds to. */ std::map partitionVec; /** \brief * Stores an old partitioning of elements. To every element index the number * of the parition it corresponds to is stored. */ std::map oldPartitionVec;  Thomas Witkowski committed Apr 07, 2009 232   Thomas Witkowski committed Jun 09, 2009 233 234 235 236  Mat petscMatrix; Vec petscRhsVec;  Thomas Witkowski committed Jun 09, 2009 237  Vec petscSolVec;  Thomas Witkowski committed Jun 09, 2009 238   Thomas Witkowski committed Apr 07, 2009 239 240  /// Number of DOFs in the rank mesh. int nRankDOFs;  Thomas Witkowski committed Apr 27, 2009 241   Thomas Witkowski committed Jun 16, 2009 242 243 244 245 246 247  /** \brief * Set of all interior boundary DOFs in ranks partition. The object maps to * each such DOF to the number of the rank that owns this DOF. */ std::map boundaryDOFs;  Thomas Witkowski committed Apr 27, 2009 248  /** \brief  Thomas Witkowski committed Jun 15, 2009 249 250 251 252 253 254 255 256 257 258 259 260  * Defines the interior boundaries of the domain that result from partitioning * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e., * the object gives for every neighbour rank i the boundaries this rank owns and * shares with rank i. */ InteriorBoundary myIntBoundary; /** \brief * Defines the interior boundaries of the domain that result from partitioning * the whole mesh. Contains only the boundaries, which are not owned by the rank, * i.e., the object gives for every neighbour rank i the boundaries that are * owned by rank i and are shared with this rank.  Thomas Witkowski committed Apr 27, 2009 261  */  Thomas Witkowski committed Jun 15, 2009 262  InteriorBoundary otherIntBoundary;  Thomas Witkowski committed Apr 27, 2009 263   Thomas Witkowski committed Jun 10, 2009 264 265 266 267  /** \brief * This map contains for each rank the list of dofs the current rank must send * to exchange solution dofs at the interior boundaries. */  Thomas Witkowski committed Jun 16, 2009 268  std::map > sendDofs;  Thomas Witkowski committed Jun 10, 2009 269 270 271 272 273  /** \brief * This map contains for each rank the list of dofs from which the current rank * must receive solution values of dofs at the interior boundaries. */  Thomas Witkowski committed Jun 16, 2009 274  std::map > recvDofs;  Thomas Witkowski committed Jun 11, 2009 275 276 277 278 279 280 281 282 283 284 285 286 287  /// Maps local to global dof indices. std::map mapLocalGlobalDOFs; /// Maps global to local dof indices. std::map mapGlobalLocalDOFs; /** \brief * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is * owned by the rank. Otherwise, its an interior boundary DOF that is owned by * another rank. */ std::map isRankDOF;  Thomas Witkowski committed Apr 03, 2009 288 289 290 291 292 293 294 295 296  }; class ParallelDomainProblemScal : public ParallelDomainProblemBase { public: ParallelDomainProblemScal(const std::string& name, ProblemScal *problem, ProblemInstatScal *problemInstat);  Thomas Witkowski committed Jun 11, 2009 297 298  void initParallelization(AdaptInfo *adaptInfo);  Thomas Witkowski committed Jun 09, 2009 299 300 301 302 303  virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION); protected: ProblemScal *probScal;  Thomas Witkowski committed Apr 03, 2009 304 305 306 307  }; } #endif // AMDIS_PARALLELDOMAINPROBLEM_H