MeshDistributor.h 20.2 KB
Newer Older
1 2 3 4 5 6
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
7
// ==  TU Dresden                                                            ==
8
// ==                                                                        ==
9 10 11
// ==  Institut fr Wissenschaftliches Rechnen                               ==
// ==  Zellescher Weg 12-14                                                  ==
// ==  01069 Dresden                                                         ==
12 13 14 15
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
16
// ==  https://gforge.zih.tu-dresden.de/projects/amdis/                      ==
17 18 19
// ==                                                                        ==
// ============================================================================

20
/** \file MeshDistributor.h */
21

22 23
#ifndef AMDIS_MESHDISTRIBUTOR_H
#define AMDIS_MESHDISTRIBUTOR_H
24 25 26


#include <map>
27
#include <set>
28
#include <vector>
Thomas Witkowski's avatar
Thomas Witkowski committed
29
#include <mpi.h>
30

Thomas Witkowski's avatar
Thomas Witkowski committed
31
#include "Global.h"
32 33
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
34
#include "FiniteElemSpace.h"
35
#include "AdaptInfo.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
36
#include "parallel/InteriorBoundary.h"
37
#include "Serializer.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
38
#include "BoundaryManager.h"
39 40
#include "AMDiS_fwd.h"

41
namespace AMDiS {
42
  
43 44
  class ParMetisPartitioner;

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
  struct ElementNeighbour {
    ElementNeighbour()
      : elIndex(-1),
	ith(0),
	boundaryIndex(INTERIOR)
    {}

    ElementNeighbour(int a, int b, BoundaryType c = INTERIOR)
      : elIndex(a),
	ith(b),
	boundaryIndex(c)
    {}
    
    int elIndex;
    
    int ith;
    
    BoundaryType boundaryIndex;
  };


66
  class MeshDistributor
67
  {
68
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
69 70 71 72 73 74 75 76 77 78 79 80
    /// Defines a mapping type from DOFs to rank numbers.
    typedef std::map<const DegreeOfFreedom*, int> DofToRank;

    /// Defines a mapping type from DOFs to a set of rank numbers.
    typedef std::map<const DegreeOfFreedom*, std::set<int> > DofToPartitions;

    /// Defines a mapping type from rank numbers to sets of DOFs.
    typedef std::map<int, DofContainer> RankToDofContainer;

    /// Defines a mapping type from DOF indices to DOF indices.
    typedef std::map<DegreeOfFreedom, DegreeOfFreedom> DofMapping;

81 82 83
    /// Defines a mapping type from DOFs to boolean values.
    typedef std::map<const DegreeOfFreedom*, bool> DofToBool;

Thomas Witkowski's avatar
Thomas Witkowski committed
84
    /// Defines a mapping type from DOF indices to boolean values.
85
    typedef std::map<DegreeOfFreedom, bool> DofIndexToBool;
Thomas Witkowski's avatar
Thomas Witkowski committed
86

Thomas Witkowski's avatar
Thomas Witkowski committed
87 88 89
    /// Forward type (it maps rank numbers to the interior boundary objects).
    typedef InteriorBoundary::RankToBoundMap RankToBoundMap;

Thomas Witkowski's avatar
Thomas Witkowski committed
90 91
    typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;

92 93 94
    /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic 
    /// dof mapping.
    typedef std::map<BoundaryType, DofMapping> PeriodicDofMap;
95 96
    
    typedef std::vector<MeshStructure> MeshCodeVec;
97

98
  public:
99 100 101
    MeshDistributor(std::string str);
		          
    virtual ~MeshDistributor() {}
102

103
    void initParallelization(AdaptInfo *adaptInfo);
104

105 106 107
    void exitParallelization(AdaptInfo *adaptInfo);

    void addProblemStat(ProblemVec *probVec);
108

109 110 111 112 113 114 115 116 117
    /** \brief
     * This function checks if the mesh has changed on at least on rank. In this case,
     * the interior boundaries are adapted on all ranks such that they fit together on
     * all ranks. Furthermore the function \ref updateLocalGlobalNumbering() is called
     * to update the dof numberings and mappings on all rank due to the new mesh
     * structure.
     */
    void checkMeshChange();

118
    void updateDofAdmins();    
119

120 121 122 123 124 125 126
    /** \brief
     * Test, if the mesh consists of macro elements only. The mesh partitioning of
     * the parallelization works for macro meshes only and would fail, if the mesh
     * is already refined in some way. Therefore, this function will exit the program
     * if it finds a non macro element in the mesh.
     */
    void testForMacroMesh();
127 128 129 130 131 132

    /// Set for each element on the partitioning level the number of leaf elements.
    double setElemWeights(AdaptInfo *adaptInfo);

    void partitionMesh(AdaptInfo *adaptInfo);

133 134 135 136
    inline virtual std::string getName() 
    { 
      return name; 
    }
137 138 139 140 141 142

    /// Returns \ref feSpace.
    inline const FiniteElemSpace* getFeSpace()
    {
      return feSpace;
    }
143 144 145
    
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
    inline int getNumberRankDofs() 
146
    {
147
      return nRankDofs;
148
    }
149

150
    /// Returns \ref nOverallDofs, the global number of DOFs.
151
    inline int getNumberOverallDofs()
152
    {
153
      return nOverallDofs;
154
    }
155

156
    /// Maps a local dof to its global index.
157
    inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
158
    {
159
      return mapLocalGlobalDofs[dof];
160
    }
161

162
    /// Maps a local dof to its local index.
163 164 165 166 167
    inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
    {
      return mapLocalDofIndex[dof];
    }

168 169
    /// Returns for a global dof index its periodic mapping for a given boundary type.
    inline int getPeriodicMapping(BoundaryType type, int globalDofIndex)
170
    {
171 172 173 174 175 176
      TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
	("Should not happen!\n");

      return periodicDof[type][globalDofIndex];
    }

177 178 179
    /// For a given global DOF index, this function returns the set of periodic
    /// associations, i.e., the boundary types the DOF is associated to, for this DOF.
    inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
180
    {      
181
      return periodicDofAssociations[globalDofIndex];
182
    }
183

184
    /// Returns true, if the DOF (global index) is a periodic DOF.
185
    inline bool isPeriodicDof(int globalDofIndex)
186
    {
187
      return (periodicDofAssociations.count(globalDofIndex) > 0);
188 189
    }

190 191 192
    /// Returns true, if the DOF (global index) is a periodic DOF for the given
    /// boundary type.
    inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
193 194
    {
      return (periodicDof[type].count(globalDofIndex) > 0);
195 196
    }

197 198
    /// Return true, if the given DOF is owned by the rank. If false, the DOF is in
    /// rank's partition, but is owned by some other rank.
199 200 201 202
    inline bool getIsRankDof(DegreeOfFreedom dof)
    {
      return isRankDof[dof];
    }
203

204
    inline long getLastMeshChangeIndex()
205
    {
206
      return lastMeshChangeIndex;
207
    }
208

209 210 211 212
    inline int getRstart()
    {
      return rstart;
    }
213

214
    inline int getMpiRank()
215
    {
216
      return mpiRank;
217
    }
218

219 220 221
    inline MPI::Intracomm& getMpiComm()
    {
      return mpiComm;
222 223
    }

224
    inline RankToDofContainer& getSendDofs()
225
    {
226
      return sendDofs;
227 228
    }

229
    inline RankToDofContainer& getRecvDofs()
230
    {
231
      return recvDofs;
232
    }
233

234
    // Writes all data of this object to an output stream.
235
    void serialize(std::ostream &out);
236

237
    // Reads the object data from an input stream.
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
    void deserialize(std::istream &in);

    /** \brief
     * This function must be used if the values of a DOFVector must be synchronised
     * over all ranks. That means, that each rank sends the values of the DOFs, which
     * are owned by the rank and lie on an interior bounday, to all other ranks also
     * having these DOFs.
     *
     * This function must be used, for example, after the lineary system is solved, or
     * after the DOFVector is set by some user defined functions, e.g., initial
     * solution functions.
     */    
    void synchVector(DOFVector<double> &vec);

    /** \brief
     * Works in the same way as the function above defined for DOFVectors. Due to
     * performance, this function does not call \ref synchVector for each DOFVector,
     * but instead sends all values of all DOFVectors all at once.
     */
    void synchVector(SystemVector &vec);

259
  protected:
260
    /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
261
     * Determines the interior boundaries, i.e. boundaries between ranks, and stores
262 263
     * all information about them in \ref interiorBoundary.
     */
264
    void createInteriorBoundaryInfo();
265

266
    void createBoundaryDataStructure();
Thomas Witkowski's avatar
Thomas Witkowski committed
267

268 269 270
    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

271 272
    /// Creates from a macro mesh a correct local and global DOF index numbering.
    void createLocalGlobalNumbering();
273

274 275
    /// Updates the local and global DOF numbering after the mesh has been changed.
    void updateLocalGlobalNumbering();
276

277 278 279 280 281
    /** \brief
     * Creates to all dofs in rank's partition that are on a periodic boundary the
     * mapping from dof index to the other periodic dof indices. This information
     * is stored in \ref periodicDof.
     */
282 283
    void createPeriodicMap();

284 285
    /** \brief
     * This function create new mappings from local to global indices, 
286
     * \ref mapLocalGlobalDofs, and from local to dof indices, \ref mapLocalDofIndex.
287 288 289 290 291 292 293 294 295 296 297 298 299 300
     * Furthermore, using the first argument the dof indices in ranks partition are
     * changed.
     * 
     * \param[in] rankDofsNewLocalIndex       Map from dof pointers of all dofs in rank
     *                                        to new dof indices.
     * \param[in] rankOwnedDofsNewLocalIndex  Map from dof pointers of dofs owned by
     *                                        the rank to the new local index.
     * \param[in] rankDofsNewGlobalIndex      Map from dof pointers of all dofs in rank
     *                                        to the new global index.
     */
    void createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
			     DofIndexMap &rankOwnedDofsNewLocalIndex,
			     DofIndexMap &rankDofsNewGlobalIndex);

301 302
    /** \brief
     * This function traverses the whole mesh, i.e. before it is really partitioned,
Thomas Witkowski's avatar
Thomas Witkowski committed
303 304 305
     * and collects information about which DOF corresponds to which rank. Can only
     * be used, if \ref partitionVec is set correctly. This is only the case, when
     * the macro mesh is partitioned.
306
     *
307 308 309 310 311
     * \param[out] partitionDOFs   Stores to each DOF pointer the set of ranks the DOF
     *                             is part of.
     * \param[out] rankDOFs        Stores all rank DOFs.
     * \param[out] boundaryDOFs    Stores all DOFs in ranks partition that are on an 
     *                             interior boundary but correspond to another rank.
312
     */
313
    void createDofMemberInfo(DofToPartitions& partitionDofs,
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
314 315
			     DofContainer& rankOwnedDofs,
			     DofContainer& rankAllDofs,
316
			     DofToRank& boundaryDofs,
317
			     DofToBool& isVertexDof);
Thomas Witkowski's avatar
Thomas Witkowski committed
318

319 320 321 322 323 324 325 326 327 328 329 330 331
    /** \brief
     * Checks for all given interior boundaries if the elements fit together on both
     * sides of the boundaries. If this is not the case, the mesh is adapted. Because
     * refinement of a certain element may forces the refinement of other elements,
     * it is not guaranteed that all rank's meshes fit together after this function
     * terminates. Hence, it must be called until a stable mesh refinement is reached.
     * If the mesh has  been changed by this function, it returns true. Otherwise, it
     * returns false, i.e., the given interior boundaries fit together on both sides.
     *
     * \param[in] allBound   Defines a map from rank to interior boundaries which 
     *                       should be checked.
     */
    bool checkAndAdaptBoundary(RankToBoundMap &allBound);
Thomas Witkowski's avatar
Thomas Witkowski committed
332
    
333 334 335 336 337 338 339
    /** \brief
     * This functions create a Paraview file with the macro mesh where the elements
     * are colored by the partition they are part of. This function can be used for
     * debugging.
     */
    void writePartitioningMesh(std::string filename);

340 341 342
    /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
    void setRankDofs();

343 344 345 346
    /// Removes all periodic boundary condition information from all matrices and
    /// vectors of all stationary problems and from the mesh itself.
    void removePeriodicBoundaryConditions();

Thomas Witkowski's avatar
Thomas Witkowski committed
347
    // Removes all periodic boundaries from a given boundary map.
348
    void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
349

350 351
    bool fitElementToMeshCode(MeshStructure &code, 
			      Element *el, 
352 353
			      GeoIndex subObj,
			      int ithObj, 
354 355
			      int elType,
			      bool reverseMode);
356 357
    
    bool fitElementToMeshCode2(MeshStructure &code, 
358
			       TraverseStack &stack,
359 360
			       GeoIndex subObj,
			       int ithObj, 
361 362
			       int elType,
			       bool reverseMode);
363

364 365 366 367 368 369 370 371 372 373 374 375 376
    /// Writes a vector of dof pointers to an output stream.
    void serialize(std::ostream &out, DofContainer &data);

    /// Reads a vector of dof pointers from an input stream.
    void deserialize(std::istream &in, DofContainer &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap);

    /// Writes a \ref RankToDofContainer to an output stream.
    void serialize(std::ostream &out, RankToDofContainer &data);

    /// Reads a \ref RankToDofContainer from an input stream.
    void deserialize(std::istream &in, RankToDofContainer &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap);
377

378 379 380
    /// Writes a periodic dof mapping to an output stream.
    void serialize(std::ostream &out, PeriodicDofMap &data);

381 382
    void serialize(std::ostream &out, std::map<int, std::set<int> >& data);

383 384 385
    /// Reads a periodic dof mapping from an input stream.
    void deserialize(std::istream &in, PeriodicDofMap &data);

386 387
    void deserialize(std::istream &in, std::map<int, std::set<int> >& data);

388 389 390 391
    /// Writes a mapping from dof pointers to some values to an output stream.
    template<typename T>
    void serialize(std::ostream &out, std::map<const DegreeOfFreedom*, T> &data)
    {
392 393
      FUNCNAME("ParallelDomainBase::serialize()");

394
      int mapSize = data.size();
395
      SerUtil::serialize(out, mapSize);
396 397 398 399
      for (typename std::map<const DegreeOfFreedom*, T>::iterator it = data.begin();
	   it != data.end(); ++it) {
	int v1 = (*(it->first));
	T v2 = it->second;
400 401
	SerUtil::serialize(out, v1);
	SerUtil::serialize(out, v2);
402 403 404 405 406 407 408 409
      }
    }

    /// Reads a mapping from dof pointer to some values from an input stream.
    template<typename T>
    void deserialize(std::istream &in, std::map<const DegreeOfFreedom*, T> &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap)
    {
410 411
      FUNCNAME("ParallelDomainBase::deserialize()");

412
      int mapSize = 0;
413
      SerUtil::deserialize(in, mapSize);
414 415 416
      for (int i = 0; i < mapSize; i++) {
	int v1 = 0;
	T v2;
417 418
	SerUtil::deserialize(in, v1);
	SerUtil::deserialize(in, v2);
419 420 421

	TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);

422 423 424 425
	data[dofMap[v1]] = v2;
      }
    }
		        
426
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
427
    ///
428
    std::vector<ProblemVec*> probStat;
Thomas Witkowski's avatar
Thomas Witkowski committed
429

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
    std::string name;

446 447 448
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

449 450 451
    /// Mesh of the problem.
    Mesh *mesh;

452 453 454 455 456 457 458
    /** \brief
     * A refinement manager that should be used on the mesh. It is used to refine
     * elements at interior boundaries in order to fit together with elements on the
     * other side of the interior boundary.
     */    
    RefinementManager *refineManager;

459 460 461
    /// Info level.
    int info;

462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
    /// Pointer to the paritioner which is used to devide a mesh into partitions.
    ParMetisPartitioner *partitioner;

    /// Weights for the elements, i.e., the number of leaf elements within this element.
    std::map<int, double> elemWeights;

    /// Is true, if the mesh was not partitioned before, otherwise it's false.
    bool initialPartitionMesh;

    /** \brief
     * Stores to every coarse element index the number of the partition it 
     * corresponds to.
     */
    std::map<int, int> partitionVec;

    /** \brief
     * Stores an old partitioning of elements. To every element index the number
     * of the parition it corresponds to is stored.
     */
    std::map<int, int> oldPartitionVec;    
482
   
483
    /// Number of DOFs in the rank mesh.
484
    int nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
485

486
    /// Number of DOFs in the whole domain.
487 488
    int nOverallDofs;

Thomas Witkowski's avatar
Thomas Witkowski committed
489
    /** \brief 
Thomas Witkowski's avatar
Thomas Witkowski committed
490 491 492 493 494 495 496 497 498 499 500 501
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
     * the object gives for every neighbour rank i the boundaries this rank owns and 
     * shares with rank i.
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are not owned by the rank,
     * i.e., the object gives for every neighbour rank i the boundaries that are
     * owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
502
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
503
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
504

505
    /** \brief
506 507
     * Defines the periodic boundaries with other ranks. Periodic boundaries have
     * no owner, as it is the case of interior boundaries.
508 509 510
     */
    InteriorBoundary periodicBoundary;

511 512 513 514
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
515
    RankToDofContainer sendDofs;
516 517

    /** \brief
518 519 520
     * This map contains for each rank the list of DOFs from which the current rank 
     * will receive DOF values (i.e., this are all DOFs at an interior boundary). The
     * DOF indices are given in rank's local numbering.
521
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
522
    RankToDofContainer recvDofs;
523 524

    /// Maps local to global dof indices.
525
    DofMapping mapLocalGlobalDofs;
526

Thomas Witkowski's avatar
Thomas Witkowski committed
527
    /// Maps local dof indices to real dof indices.
528
    DofMapping mapLocalDofIndex;  
529 530 531 532 533 534

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
535 536
    DofIndexToBool isRankDof;

537 538
    /** \brief
     * Maps every dof pointer in ranks macro mesh to a boolean variable indicating 
539
     * whether this dof is a vertex dof (true) or not (false).
540
     */
541
    DofToBool vertexDof;
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
542

543
    /** \brief
544 545 546 547
     * If periodic boundaries are used, this map stores, for each periodic boundary
     * type, for all DOFs in rank's partition (that are on periodic boundaries), the 
     * corresponding mapped periodic DOFs. The mapping is defined by using global 
     * dof indices.
548
     */
549
    PeriodicDofMap periodicDof;
550 551 552 553 554 555 556 557
    
    /** \brief
     * If periodic boundaries are used, this map stores to each periodic DOF in rank's
     * partition the set of periodic boundaries the DOF is associated to. In 2D, most
     * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
     * being periodic, the for corners are associated by two different boundaries.     
     */
    std::map<int, std::set<BoundaryType> > periodicDofAssociations;
558

559
    /// Is the index of the first row of the linear system, which is owned by the rank.
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
560
    int rstart;
561

562 563 564 565 566 567 568
    /** \brief
     * If the problem definition has been read from a serialization file, this 
     * variable is true, otherwise it is false. This variable is used to stop the
     * initialization function, if the problem definition has already been read from
     * a serialization file.
     */
    bool deserialized;
569

570 571 572
    /// Denotes whether there exists a filewriter for this object.
    bool writeSerializationFile;

573 574 575 576 577
    /** \brief
     * Stores the mesh change index. This is used to recognize changes in the mesh 
     * structure (e.g. through refinement or coarsening managers).
     */
    long lastMeshChangeIndex;
578

579 580 581 582 583
    /// This variable is true, if the macro elements are consistent with all other
    /// data structures. Within the initialization and during mesh redistribution this
    /// may not be the case.
    bool macroElementStructureConsisten;

584
    friend class ParallelDebug;
585 586 587
  };
}

588
#endif // AMDIS_MESHDISTRIBUTOR_H