MeshDistributor.h 20.2 KB
Newer Older
1
2
3
4
5
6
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
7
// ==  TU Dresden                                                            ==
8
// ==                                                                        ==
9
10
11
// ==  Institut fr Wissenschaftliches Rechnen                               ==
// ==  Zellescher Weg 12-14                                                  ==
// ==  01069 Dresden                                                         ==
12
13
14
15
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
16
// ==  https://gforge.zih.tu-dresden.de/projects/amdis/                      ==
17
18
19
// ==                                                                        ==
// ============================================================================

20
/** \file MeshDistributor.h */
21

22
23
#ifndef AMDIS_MESHDISTRIBUTOR_H
#define AMDIS_MESHDISTRIBUTOR_H
24
25
26


#include <map>
27
#include <set>
28
#include <vector>
Thomas Witkowski's avatar
Thomas Witkowski committed
29
#include <mpi.h>
30

Thomas Witkowski's avatar
Thomas Witkowski committed
31
#include "Global.h"
32
33
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
34
#include "FiniteElemSpace.h"
35
#include "AdaptInfo.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
36
#include "parallel/InteriorBoundary.h"
37
#include "Serializer.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
38
#include "BoundaryManager.h"
39
40
#include "AMDiS_fwd.h"

41
namespace AMDiS {
42
  
43
44
  class ParMetisPartitioner;

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
  struct ElementNeighbour {
    ElementNeighbour()
      : elIndex(-1),
	ith(0),
	boundaryIndex(INTERIOR)
    {}

    ElementNeighbour(int a, int b, BoundaryType c = INTERIOR)
      : elIndex(a),
	ith(b),
	boundaryIndex(c)
    {}
    
    int elIndex;
    
    int ith;
    
    BoundaryType boundaryIndex;
  };


66
  class MeshDistributor
67
  {
68
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
69
70
71
72
73
74
75
76
77
78
79
80
    /// Defines a mapping type from DOFs to rank numbers.
    typedef std::map<const DegreeOfFreedom*, int> DofToRank;

    /// Defines a mapping type from DOFs to a set of rank numbers.
    typedef std::map<const DegreeOfFreedom*, std::set<int> > DofToPartitions;

    /// Defines a mapping type from rank numbers to sets of DOFs.
    typedef std::map<int, DofContainer> RankToDofContainer;

    /// Defines a mapping type from DOF indices to DOF indices.
    typedef std::map<DegreeOfFreedom, DegreeOfFreedom> DofMapping;

81
82
83
    /// Defines a mapping type from DOFs to boolean values.
    typedef std::map<const DegreeOfFreedom*, bool> DofToBool;

Thomas Witkowski's avatar
Thomas Witkowski committed
84
    /// Defines a mapping type from DOF indices to boolean values.
85
    typedef std::map<DegreeOfFreedom, bool> DofIndexToBool;
Thomas Witkowski's avatar
Thomas Witkowski committed
86

Thomas Witkowski's avatar
Thomas Witkowski committed
87
88
89
    /// Forward type (it maps rank numbers to the interior boundary objects).
    typedef InteriorBoundary::RankToBoundMap RankToBoundMap;

Thomas Witkowski's avatar
Thomas Witkowski committed
90
91
    typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;

92
93
94
    /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic 
    /// dof mapping.
    typedef std::map<BoundaryType, DofMapping> PeriodicDofMap;
95
96
    
    typedef std::vector<MeshStructure> MeshCodeVec;
97

98
  public:
99
100
101
    MeshDistributor(std::string str);
		          
    virtual ~MeshDistributor() {}
102

103
    void initParallelization(AdaptInfo *adaptInfo);
104

105
106
107
    void exitParallelization(AdaptInfo *adaptInfo);

    void addProblemStat(ProblemVec *probVec);
108

109
110
111
112
113
114
115
116
117
    /** \brief
     * This function checks if the mesh has changed on at least on rank. In this case,
     * the interior boundaries are adapted on all ranks such that they fit together on
     * all ranks. Furthermore the function \ref updateLocalGlobalNumbering() is called
     * to update the dof numberings and mappings on all rank due to the new mesh
     * structure.
     */
    void checkMeshChange();

118
    void updateDofAdmins();    
119

120
121
122
123
124
125
126
    /** \brief
     * Test, if the mesh consists of macro elements only. The mesh partitioning of
     * the parallelization works for macro meshes only and would fail, if the mesh
     * is already refined in some way. Therefore, this function will exit the program
     * if it finds a non macro element in the mesh.
     */
    void testForMacroMesh();
127
128
129
130
131
132

    /// Set for each element on the partitioning level the number of leaf elements.
    double setElemWeights(AdaptInfo *adaptInfo);

    void partitionMesh(AdaptInfo *adaptInfo);

133
134
135
136
    inline virtual std::string getName() 
    { 
      return name; 
    }
137
138
139
140
141
142

    /// Returns \ref feSpace.
    inline const FiniteElemSpace* getFeSpace()
    {
      return feSpace;
    }
143
144
145
    
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
    inline int getNumberRankDofs() 
146
    {
147
      return nRankDofs;
148
    }
149

150
    /// Returns \ref nOverallDofs, the global number of DOFs.
151
    inline int getNumberOverallDofs()
152
    {
153
      return nOverallDofs;
154
    }
155

156
    /// Maps a local dof to its global index.
157
    inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
158
    {
159
      return mapLocalGlobalDofs[dof];
160
    }
161

162
    /// Maps a local dof to its local index.
163
164
165
166
167
    inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
    {
      return mapLocalDofIndex[dof];
    }

168
169
    /// Returns for a global dof index its periodic mapping for a given boundary type.
    inline int getPeriodicMapping(BoundaryType type, int globalDofIndex)
170
    {
171
172
173
174
175
176
      TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
	("Should not happen!\n");

      return periodicDof[type][globalDofIndex];
    }

177
178
179
    /// For a given global DOF index, this function returns the set of periodic
    /// associations, i.e., the boundary types the DOF is associated to, for this DOF.
    inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
180
    {      
181
      return periodicDofAssociations[globalDofIndex];
182
    }
183

184
    /// Returns true, if the DOF (global index) is a periodic DOF.
185
    inline bool isPeriodicDof(int globalDofIndex)
186
    {
187
      return (periodicDofAssociations.count(globalDofIndex) > 0);
188
189
    }

190
191
192
    /// Returns true, if the DOF (global index) is a periodic DOF for the given
    /// boundary type.
    inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
193
194
    {
      return (periodicDof[type].count(globalDofIndex) > 0);
195
196
    }

197
198
    /// Return true, if the given DOF is owned by the rank. If false, the DOF is in
    /// rank's partition, but is owned by some other rank.
199
200
201
202
    inline bool getIsRankDof(DegreeOfFreedom dof)
    {
      return isRankDof[dof];
    }
203

204
    inline long getLastMeshChangeIndex()
205
    {
206
      return lastMeshChangeIndex;
207
    }
208

209
210
211
212
    inline int getRstart()
    {
      return rstart;
    }
213

214
    inline int getMpiRank()
215
    {
216
      return mpiRank;
217
    }
218

219
220
221
    inline MPI::Intracomm& getMpiComm()
    {
      return mpiComm;
222
223
    }

224
    inline RankToDofContainer& getSendDofs()
225
    {
226
      return sendDofs;
227
228
    }

229
    inline RankToDofContainer& getRecvDofs()
230
    {
231
      return recvDofs;
232
    }
233

234
    // Writes all data of this object to an output stream.
235
    void serialize(std::ostream &out);
236

237
    // Reads the object data from an input stream.
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
    void deserialize(std::istream &in);

    /** \brief
     * This function must be used if the values of a DOFVector must be synchronised
     * over all ranks. That means, that each rank sends the values of the DOFs, which
     * are owned by the rank and lie on an interior bounday, to all other ranks also
     * having these DOFs.
     *
     * This function must be used, for example, after the lineary system is solved, or
     * after the DOFVector is set by some user defined functions, e.g., initial
     * solution functions.
     */    
    void synchVector(DOFVector<double> &vec);

    /** \brief
     * Works in the same way as the function above defined for DOFVectors. Due to
     * performance, this function does not call \ref synchVector for each DOFVector,
     * but instead sends all values of all DOFVectors all at once.
     */
    void synchVector(SystemVector &vec);

259
  protected:
260
    /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
261
     * Determines the interior boundaries, i.e. boundaries between ranks, and stores
262
263
     * all information about them in \ref interiorBoundary.
     */
264
    void createInteriorBoundaryInfo();
265

266
    void createBoundaryDataStructure();
Thomas Witkowski's avatar
Thomas Witkowski committed
267

268
269
270
    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

271
272
    /// Creates from a macro mesh a correct local and global DOF index numbering.
    void createLocalGlobalNumbering();
273

274
275
    /// Updates the local and global DOF numbering after the mesh has been changed.
    void updateLocalGlobalNumbering();
276

277
278
279
280
281
    /** \brief
     * Creates to all dofs in rank's partition that are on a periodic boundary the
     * mapping from dof index to the other periodic dof indices. This information
     * is stored in \ref periodicDof.
     */
282
283
    void createPeriodicMap();

284
285
    /** \brief
     * This function create new mappings from local to global indices, 
286
     * \ref mapLocalGlobalDofs, and from local to dof indices, \ref mapLocalDofIndex.
287
288
289
290
291
292
293
294
295
296
297
298
299
300
     * Furthermore, using the first argument the dof indices in ranks partition are
     * changed.
     * 
     * \param[in] rankDofsNewLocalIndex       Map from dof pointers of all dofs in rank
     *                                        to new dof indices.
     * \param[in] rankOwnedDofsNewLocalIndex  Map from dof pointers of dofs owned by
     *                                        the rank to the new local index.
     * \param[in] rankDofsNewGlobalIndex      Map from dof pointers of all dofs in rank
     *                                        to the new global index.
     */
    void createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
			     DofIndexMap &rankOwnedDofsNewLocalIndex,
			     DofIndexMap &rankDofsNewGlobalIndex);

301
302
    /** \brief
     * This function traverses the whole mesh, i.e. before it is really partitioned,
Thomas Witkowski's avatar
Thomas Witkowski committed
303
304
305
     * and collects information about which DOF corresponds to which rank. Can only
     * be used, if \ref partitionVec is set correctly. This is only the case, when
     * the macro mesh is partitioned.
306
     *
307
308
309
310
311
     * \param[out] partitionDOFs   Stores to each DOF pointer the set of ranks the DOF
     *                             is part of.
     * \param[out] rankDOFs        Stores all rank DOFs.
     * \param[out] boundaryDOFs    Stores all DOFs in ranks partition that are on an 
     *                             interior boundary but correspond to another rank.
312
     */
313
    void createDofMemberInfo(DofToPartitions& partitionDofs,
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
314
315
			     DofContainer& rankOwnedDofs,
			     DofContainer& rankAllDofs,
316
			     DofToRank& boundaryDofs,
317
			     DofToBool& isVertexDof);
Thomas Witkowski's avatar
Thomas Witkowski committed
318

319
320
321
322
323
324
325
326
327
328
329
330
331
    /** \brief
     * Checks for all given interior boundaries if the elements fit together on both
     * sides of the boundaries. If this is not the case, the mesh is adapted. Because
     * refinement of a certain element may forces the refinement of other elements,
     * it is not guaranteed that all rank's meshes fit together after this function
     * terminates. Hence, it must be called until a stable mesh refinement is reached.
     * If the mesh has  been changed by this function, it returns true. Otherwise, it
     * returns false, i.e., the given interior boundaries fit together on both sides.
     *
     * \param[in] allBound   Defines a map from rank to interior boundaries which 
     *                       should be checked.
     */
    bool checkAndAdaptBoundary(RankToBoundMap &allBound);
Thomas Witkowski's avatar
Thomas Witkowski committed
332
    
333
334
335
336
337
338
339
    /** \brief
     * This functions create a Paraview file with the macro mesh where the elements
     * are colored by the partition they are part of. This function can be used for
     * debugging.
     */
    void writePartitioningMesh(std::string filename);

340
341
342
    /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
    void setRankDofs();

343
344
345
346
    /// Removes all periodic boundary condition information from all matrices and
    /// vectors of all stationary problems and from the mesh itself.
    void removePeriodicBoundaryConditions();

Thomas Witkowski's avatar
Thomas Witkowski committed
347
    // Removes all periodic boundaries from a given boundary map.
348
    void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
349

350
351
    bool fitElementToMeshCode(MeshStructure &code, 
			      Element *el, 
352
353
			      GeoIndex subObj,
			      int ithObj, 
354
355
			      int elType,
			      bool reverseMode);
356
357
    
    bool fitElementToMeshCode2(MeshStructure &code, 
358
			       TraverseStack &stack,
359
360
			       GeoIndex subObj,
			       int ithObj, 
361
362
			       int elType,
			       bool reverseMode);
363

364
365
366
367
368
369
370
371
372
373
374
375
376
    /// Writes a vector of dof pointers to an output stream.
    void serialize(std::ostream &out, DofContainer &data);

    /// Reads a vector of dof pointers from an input stream.
    void deserialize(std::istream &in, DofContainer &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap);

    /// Writes a \ref RankToDofContainer to an output stream.
    void serialize(std::ostream &out, RankToDofContainer &data);

    /// Reads a \ref RankToDofContainer from an input stream.
    void deserialize(std::istream &in, RankToDofContainer &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap);
377

378
379
380
    /// Writes a periodic dof mapping to an output stream.
    void serialize(std::ostream &out, PeriodicDofMap &data);

381
382
    void serialize(std::ostream &out, std::map<int, std::set<int> >& data);

383
384
385
    /// Reads a periodic dof mapping from an input stream.
    void deserialize(std::istream &in, PeriodicDofMap &data);

386
387
    void deserialize(std::istream &in, std::map<int, std::set<int> >& data);

388
389
390
391
    /// Writes a mapping from dof pointers to some values to an output stream.
    template<typename T>
    void serialize(std::ostream &out, std::map<const DegreeOfFreedom*, T> &data)
    {
392
393
      FUNCNAME("ParallelDomainBase::serialize()");

394
      int mapSize = data.size();
395
      SerUtil::serialize(out, mapSize);
396
397
398
399
      for (typename std::map<const DegreeOfFreedom*, T>::iterator it = data.begin();
	   it != data.end(); ++it) {
	int v1 = (*(it->first));
	T v2 = it->second;
400
401
	SerUtil::serialize(out, v1);
	SerUtil::serialize(out, v2);
402
403
404
405
406
407
408
409
      }
    }

    /// Reads a mapping from dof pointer to some values from an input stream.
    template<typename T>
    void deserialize(std::istream &in, std::map<const DegreeOfFreedom*, T> &data,
		     std::map<int, const DegreeOfFreedom*> &dofMap)
    {
410
411
      FUNCNAME("ParallelDomainBase::deserialize()");

412
      int mapSize = 0;
413
      SerUtil::deserialize(in, mapSize);
414
415
416
      for (int i = 0; i < mapSize; i++) {
	int v1 = 0;
	T v2;
417
418
	SerUtil::deserialize(in, v1);
	SerUtil::deserialize(in, v2);
419
420
421

	TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);

422
423
424
425
	data[dofMap[v1]] = v2;
      }
    }
		        
426
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
427
    ///
428
    std::vector<ProblemVec*> probStat;
Thomas Witkowski's avatar
Thomas Witkowski committed
429

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
    std::string name;

446
447
448
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

449
450
451
    /// Mesh of the problem.
    Mesh *mesh;

452
453
454
455
456
457
458
    /** \brief
     * A refinement manager that should be used on the mesh. It is used to refine
     * elements at interior boundaries in order to fit together with elements on the
     * other side of the interior boundary.
     */    
    RefinementManager *refineManager;

459
460
461
    /// Info level.
    int info;

462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
    /// Pointer to the paritioner which is used to devide a mesh into partitions.
    ParMetisPartitioner *partitioner;

    /// Weights for the elements, i.e., the number of leaf elements within this element.
    std::map<int, double> elemWeights;

    /// Is true, if the mesh was not partitioned before, otherwise it's false.
    bool initialPartitionMesh;

    /** \brief
     * Stores to every coarse element index the number of the partition it 
     * corresponds to.
     */
    std::map<int, int> partitionVec;

    /** \brief
     * Stores an old partitioning of elements. To every element index the number
     * of the parition it corresponds to is stored.
     */
    std::map<int, int> oldPartitionVec;    
482
   
483
    /// Number of DOFs in the rank mesh.
484
    int nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
485

486
    /// Number of DOFs in the whole domain.
487
488
    int nOverallDofs;

Thomas Witkowski's avatar
Thomas Witkowski committed
489
    /** \brief 
Thomas Witkowski's avatar
Thomas Witkowski committed
490
491
492
493
494
495
496
497
498
499
500
501
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
     * the object gives for every neighbour rank i the boundaries this rank owns and 
     * shares with rank i.
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are not owned by the rank,
     * i.e., the object gives for every neighbour rank i the boundaries that are
     * owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
502
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
503
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
504

505
    /** \brief
506
507
     * Defines the periodic boundaries with other ranks. Periodic boundaries have
     * no owner, as it is the case of interior boundaries.
508
509
510
     */
    InteriorBoundary periodicBoundary;

511
512
513
514
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
515
    RankToDofContainer sendDofs;
516
517

    /** \brief
518
519
520
     * This map contains for each rank the list of DOFs from which the current rank 
     * will receive DOF values (i.e., this are all DOFs at an interior boundary). The
     * DOF indices are given in rank's local numbering.
521
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
522
    RankToDofContainer recvDofs;
523
524

    /// Maps local to global dof indices.
525
    DofMapping mapLocalGlobalDofs;
526

Thomas Witkowski's avatar
Thomas Witkowski committed
527
    /// Maps local dof indices to real dof indices.
528
    DofMapping mapLocalDofIndex;  
529
530
531
532
533
534

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
535
536
    DofIndexToBool isRankDof;

537
538
    /** \brief
     * Maps every dof pointer in ranks macro mesh to a boolean variable indicating 
539
     * whether this dof is a vertex dof (true) or not (false).
540
     */
541
    DofToBool vertexDof;
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
542

543
    /** \brief
544
545
546
547
     * If periodic boundaries are used, this map stores, for each periodic boundary
     * type, for all DOFs in rank's partition (that are on periodic boundaries), the 
     * corresponding mapped periodic DOFs. The mapping is defined by using global 
     * dof indices.
548
     */
549
    PeriodicDofMap periodicDof;
550
551
552
553
554
555
556
557
    
    /** \brief
     * If periodic boundaries are used, this map stores to each periodic DOF in rank's
     * partition the set of periodic boundaries the DOF is associated to. In 2D, most
     * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
     * being periodic, the for corners are associated by two different boundaries.     
     */
    std::map<int, std::set<BoundaryType> > periodicDofAssociations;
558

559
    /// Is the index of the first row of the linear system, which is owned by the rank.
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
560
    int rstart;
561

562
563
564
565
566
567
568
    /** \brief
     * If the problem definition has been read from a serialization file, this 
     * variable is true, otherwise it is false. This variable is used to stop the
     * initialization function, if the problem definition has already been read from
     * a serialization file.
     */
    bool deserialized;
569

570
571
572
    /// Denotes whether there exists a filewriter for this object.
    bool writeSerializationFile;

573
574
575
576
577
    /** \brief
     * Stores the mesh change index. This is used to recognize changes in the mesh 
     * structure (e.g. through refinement or coarsening managers).
     */
    long lastMeshChangeIndex;
578

579
580
581
582
583
    /// This variable is true, if the macro elements are consistent with all other
    /// data structures. Within the initialization and during mesh redistribution this
    /// may not be the case.
    bool macroElementStructureConsisten;

584
    friend class ParallelDebug;
585
586
587
  };
}

588
#endif // AMDIS_MESHDISTRIBUTOR_H