Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

MeshDistributor.h 18.8 KB
Newer Older
1 2 3 4
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
5
// ==  http://www.amdis-fem.org                                              ==
6 7
// ==                                                                        ==
// ============================================================================
8 9 10 11 12 13 14 15 16 17 18 19
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


20

21
/** \file MeshDistributor.h */
22

23 24
#ifndef AMDIS_MESHDISTRIBUTOR_H
#define AMDIS_MESHDISTRIBUTOR_H
25 26 27


#include <map>
28
#include <set>
29
#include <vector>
Thomas Witkowski's avatar
Thomas Witkowski committed
30
#include <mpi.h>
31

32
#include "parallel/MeshPartitioner.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
33
#include "parallel/InteriorBoundary.h"
34
#include "AMDiS_fwd.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
35
#include "Global.h"
36 37
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
38
#include "FiniteElemSpace.h"
39
#include "Serializer.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
40
#include "BoundaryManager.h"
41
#include "ElementObjectData.h"
42
#include "SystemVector.h"
43

44
namespace AMDiS {
45 46

  using namespace std;
47
  
48
  class MeshDistributor
49
  {
50
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
51
    /// Defines a mapping type from DOFs to rank numbers.
52
    typedef map<const DegreeOfFreedom*, int> DofToRank;
Thomas Witkowski's avatar
Thomas Witkowski committed
53 54

    /// Defines a mapping type from DOFs to a set of rank numbers.
55
    typedef map<const DegreeOfFreedom*, std::set<int> > DofToPartitions;
Thomas Witkowski's avatar
Thomas Witkowski committed
56 57

    /// Defines a mapping type from rank numbers to sets of DOFs.
58
    typedef map<int, DofContainer> RankToDofContainer;
Thomas Witkowski's avatar
Thomas Witkowski committed
59 60

    /// Defines a mapping type from DOF indices to DOF indices.
61
    typedef map<DegreeOfFreedom, DegreeOfFreedom> DofMapping;
Thomas Witkowski's avatar
Thomas Witkowski committed
62

63
    /// Defines a mapping type from DOFs to boolean values.
64
    typedef map<const DegreeOfFreedom*, bool> DofToBool;
65

Thomas Witkowski's avatar
Thomas Witkowski committed
66
    /// Defines a mapping type from DOF indices to boolean values.
67
    typedef map<DegreeOfFreedom, bool> DofIndexToBool;
Thomas Witkowski's avatar
Thomas Witkowski committed
68

Thomas Witkowski's avatar
Thomas Witkowski committed
69 70 71
    /// Forward type (it maps rank numbers to the interior boundary objects).
    typedef InteriorBoundary::RankToBoundMap RankToBoundMap;

72
    typedef map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;
Thomas Witkowski's avatar
Thomas Witkowski committed
73

74 75
    /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic 
    /// dof mapping.
76 77 78
    typedef map<BoundaryType, DofMapping> PeriodicDofMap;

    typedef vector<MeshStructure> MeshCodeVec;
79

80
  public:
81
    MeshDistributor(string str);
82 83
		          
    virtual ~MeshDistributor() {}
84

85
    void initParallelization();
86

87
    void exitParallelization();
88 89

    void addProblemStat(ProblemVec *probVec);
90

91 92 93 94 95 96 97
    /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
    /// be automatically interchanged between ranks when mesh is repartitioned.
    void addInterchangeVector(DOFVector<double> *vec)
    {
      interchangeVectors.push_back(vec);
    }

98 99 100 101 102 103 104
    /// Adds all DOFVectors of a SystemVector to \ref interchangeVecs.
    void addInterchangeVector(SystemVector *vec)
    {
      for (int i = 0; i < vec->getSize(); i++)
	interchangeVectors.push_back(vec->getDOFVector(i));
    }
    
105 106 107 108 109 110 111 112 113
    /** \brief
     * This function checks if the mesh has changed on at least on rank. In this case,
     * the interior boundaries are adapted on all ranks such that they fit together on
     * all ranks. Furthermore the function \ref updateLocalGlobalNumbering() is called
     * to update the dof numberings and mappings on all rank due to the new mesh
     * structure.
     */
    void checkMeshChange();

114 115 116 117 118 119 120
    /** \brief
     * Test, if the mesh consists of macro elements only. The mesh partitioning of
     * the parallelization works for macro meshes only and would fail, if the mesh
     * is already refined in some way. Therefore, this function will exit the program
     * if it finds a non macro element in the mesh.
     */
    void testForMacroMesh();
121 122

    /// Set for each element on the partitioning level the number of leaf elements.
123
    void setInitialElementWeights();
124

125
    inline virtual string getName() 
126 127 128
    { 
      return name; 
    }
129

Thomas Witkowski's avatar
Thomas Witkowski committed
130 131 132 133 134
    inline Mesh* getMesh()
    {
      return mesh;
    }

135 136 137 138 139
    /// Returns \ref feSpace.
    inline const FiniteElemSpace* getFeSpace()
    {
      return feSpace;
    }
140 141 142
    
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
    inline int getNumberRankDofs() 
143
    {
144
      return nRankDofs;
145
    }
146

147
    /// Returns \ref nOverallDofs, the global number of DOFs.
148
    inline int getNumberOverallDofs()
149
    {
150
      return nOverallDofs;
151
    }
152

Thomas Witkowski's avatar
Thomas Witkowski committed
153 154 155 156 157
    inline DofMapping& getMapLocalGlobalDofs()
    {
      return mapLocalGlobalDofs;
    }

158
    /// Maps a local dof to its global index.
159
    inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
160
    {
161
      return mapLocalGlobalDofs[dof];
162
    }
163

164 165
    DegreeOfFreedom mapGlobalToLocal(DegreeOfFreedom dof);

166
    /// Maps a local dof to its local index.
167 168 169 170 171
    inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
    {
      return mapLocalDofIndex[dof];
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
172 173 174 175 176 177
    /// Returns the periodic mapping for all boundary DOFs in rank.
    inline PeriodicDofMap& getPeriodicMapping()
    {
      return periodicDof;
    }

178
    /// Returns for a global dof index its periodic mapping for a given boundary type.
179
    inline int getPeriodicMapping(int globalDofIndex, BoundaryType type)
180
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
181 182
      FUNCNAME("MeshDistributor::getPeriodicMapping()");

183
      TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
Thomas Witkowski's avatar
Thomas Witkowski committed
184 185
	("There is no periodic association for global DOF %d for boundary type %d!\n",
	 globalDofIndex, type);
186 187 188 189

      return periodicDof[type][globalDofIndex];
    }

190 191 192
    /// For a given global DOF index, this function returns the set of periodic
    /// associations, i.e., the boundary types the DOF is associated to, for this DOF.
    inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
193
    {      
194 195 196
      TEST_EXIT_DBG(periodicDofAssociations.count(globalDofIndex)) 
 	("Should not happen!\n"); 

197
      return periodicDofAssociations[globalDofIndex];
198
    }
199

200
    /// Returns true, if the DOF (global index) is a periodic DOF.
201
    inline bool isPeriodicDof(int globalDofIndex)
202
    {
203 204
      return (periodicDofAssociations.count(globalDofIndex) > 0 &&
	       periodicDofAssociations[globalDofIndex].size() > 0);
205 206
    }

207 208 209
    /// Returns true, if the DOF (global index) is a periodic DOF for the given
    /// boundary type.
    inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
210 211
    {
      return (periodicDof[type].count(globalDofIndex) > 0);
212 213
    }

214 215
    /// Return true, if the given DOF is owned by the rank. If false, the DOF is in
    /// rank's partition, but is owned by some other rank.
216 217
    inline bool getIsRankDof(DegreeOfFreedom dof)
    {
218 219 220 221
      if (isRankDof.count(dof))
	return isRankDof[dof];

      return false;
222
    }
223

224
    inline long getLastMeshChangeIndex()
225
    {
226
      return lastMeshChangeIndex;
227
    }
228

229 230 231 232
    inline int getRstart()
    {
      return rstart;
    }
233

234
    inline int getMpiRank()
235
    {
236
      return mpiRank;
237
    }
238

Thomas Witkowski's avatar
Thomas Witkowski committed
239 240 241 242 243
    inline int getMpiSize()
    {
      return mpiSize;
    }

244 245 246
    inline MPI::Intracomm& getMpiComm()
    {
      return mpiComm;
247 248
    }

249
    inline RankToDofContainer& getSendDofs()
250
    {
251
      return sendDofs;
252 253
    }

254
    inline RankToDofContainer& getRecvDofs()
255
    {
256
      return recvDofs;
257
    }
258

259
    // Writes all data of this object to an output stream.
260
    void serialize(ostream &out);
261

262
    // Reads the object data from an input stream.
263
    void deserialize(istream &in);
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283

    /** \brief
     * This function must be used if the values of a DOFVector must be synchronised
     * over all ranks. That means, that each rank sends the values of the DOFs, which
     * are owned by the rank and lie on an interior bounday, to all other ranks also
     * having these DOFs.
     *
     * This function must be used, for example, after the lineary system is solved, or
     * after the DOFVector is set by some user defined functions, e.g., initial
     * solution functions.
     */    
    void synchVector(DOFVector<double> &vec);

    /** \brief
     * Works in the same way as the function above defined for DOFVectors. Due to
     * performance, this function does not call \ref synchVector for each DOFVector,
     * but instead sends all values of all DOFVectors all at once.
     */
    void synchVector(SystemVector &vec);

284 285
    void check3dValidMesh();

286
  protected:
287
    /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
288
     * Determines the interior boundaries, i.e. boundaries between ranks, and stores
289 290
     * all information about them in \ref interiorBoundary.
     */
291
    void createInteriorBoundaryInfo();
292

Thomas Witkowski's avatar
Thomas Witkowski committed
293 294 295 296 297
    void updateInteriorBoundaryInfo();

    void createMeshElementData();

    void createBoundaryData();
Thomas Witkowski's avatar
Thomas Witkowski committed
298

299 300 301
    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

302
    /// Updates the local and global DOF numbering after the mesh has been changed.
303
    void updateLocalGlobalNumbering();
304

305 306 307 308 309
    /** \brief
     * Creates to all dofs in rank's partition that are on a periodic boundary the
     * mapping from dof index to the other periodic dof indices. This information
     * is stored in \ref periodicDof.
     */
310 311
    void createPeriodicMap();

312 313
    void createMacroElementInfo();

314 315
    void updateMacroElementInfo();

316 317 318 319 320 321 322 323 324
    /** \brief
     * Checks for all given interior boundaries if the elements fit together on both
     * sides of the boundaries. If this is not the case, the mesh is adapted. Because
     * refinement of a certain element may forces the refinement of other elements,
     * it is not guaranteed that all rank's meshes fit together after this function
     * terminates. Hence, it must be called until a stable mesh refinement is reached.
     *
     * \param[in] allBound   Defines a map from rank to interior boundaries which 
     *                       should be checked.
325 326 327 328
     *
     * \return    If the mesh has  been changed by this function, it returns true. 
     *            Otherwise, it returns false, i.e., the given interior boundaries 
     *            fit together on both sides.
329 330
     */
    bool checkAndAdaptBoundary(RankToBoundMap &allBound);
331 332 333 334 335 336 337 338
  
    /** \brief
     * Checks if is required to repartition the mesh. If this is the case, a new
     * partition will be created and the mesh will be redistributed between the
     * ranks.
     */
    void repartitionMesh();

339 340 341
    /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
    void setRankDofs();

342 343 344 345
    /// Removes all periodic boundary condition information from all matrices and
    /// vectors of all stationary problems and from the mesh itself.
    void removePeriodicBoundaryConditions();

Thomas Witkowski's avatar
Thomas Witkowski committed
346
    // Removes all periodic boundaries from a given boundary map.
347
    void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
348

349
    /// Writes a vector of dof pointers to an output stream.
350
    void serialize(ostream &out, DofContainer &data);
351 352

    /// Reads a vector of dof pointers from an input stream.
353 354
    void deserialize(istream &in, DofContainer &data,
		     map<int, const DegreeOfFreedom*> &dofMap);
355 356

    /// Writes a \ref RankToDofContainer to an output stream.
357
    void serialize(ostream &out, RankToDofContainer &data);
358 359

    /// Reads a \ref RankToDofContainer from an input stream.
360 361
    void deserialize(istream &in, RankToDofContainer &data,
		     map<int, const DegreeOfFreedom*> &dofMap);
362

363
    /// Writes a periodic dof mapping to an output stream.
364
    void serialize(ostream &out, PeriodicDofMap &data);
365

366
    void serialize(ostream &out, map<int, std::set<int> >& data);
367

368
    /// Reads a periodic dof mapping from an input stream.
369
    void deserialize(istream &in, PeriodicDofMap &data);
370

371
    void deserialize(istream &in, map<int, std::set<int> >& data);
372

373 374
    /// Writes a mapping from dof pointers to some values to an output stream.
    template<typename T>
375
    void serialize(ostream &out, map<const DegreeOfFreedom*, T> &data)
376
    {
377 378
      FUNCNAME("ParallelDomainBase::serialize()");

379
      int mapSize = data.size();
380
      SerUtil::serialize(out, mapSize);
381
      for (typename map<const DegreeOfFreedom*, T>::iterator it = data.begin();
382 383 384
	   it != data.end(); ++it) {
	int v1 = (*(it->first));
	T v2 = it->second;
385 386
	SerUtil::serialize(out, v1);
	SerUtil::serialize(out, v2);
387 388 389 390 391
      }
    }

    /// Reads a mapping from dof pointer to some values from an input stream.
    template<typename T>
392 393
    void deserialize(istream &in, map<const DegreeOfFreedom*, T> &data,
		     map<int, const DegreeOfFreedom*> &dofMap)
394
    {
395 396
      FUNCNAME("ParallelDomainBase::deserialize()");

397
      int mapSize = 0;
398
      SerUtil::deserialize(in, mapSize);
399 400 401
      for (int i = 0; i < mapSize; i++) {
	int v1 = 0;
	T v2;
402 403
	SerUtil::deserialize(in, v1);
	SerUtil::deserialize(in, v2);
404 405 406

	TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);

407 408 409
	data[dofMap[v1]] = v2;
      }
    }
410

411
  protected:
Thomas Witkowski's avatar
Thomas Witkowski committed
412
    ///
413
    vector<ProblemVec*> probStat;
Thomas Witkowski's avatar
Thomas Witkowski committed
414

415 416 417 418 419 420 421 422 423 424 425 426 427 428
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
429
    string name;
430

431 432 433
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

434 435 436
    /// Mesh of the problem.
    Mesh *mesh;

437 438 439 440 441 442 443
    /** \brief
     * A refinement manager that should be used on the mesh. It is used to refine
     * elements at interior boundaries in order to fit together with elements on the
     * other side of the interior boundary.
     */    
    RefinementManager *refineManager;

444 445 446
    /// Info level.
    int info;

447 448
    /// Pointer to a mesh partitioner that is used to partition the mesh to the ranks.
    MeshPartitioner *partitioner;
449 450

    /// Weights for the elements, i.e., the number of leaf elements within this element.
451
    map<int, double> elemWeights;
452 453

    /** \brief
454 455
     * Stores to every macro element index the number of the rank that owns this
     * macro element.
456
     */
457
    map<int, int> partitionMap;
458

459
    /// Number of DOFs in the rank mesh.
460
    int nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
461

462
    /// Number of DOFs in the whole domain.
463 464
    int nOverallDofs;

Thomas Witkowski's avatar
Thomas Witkowski committed
465 466 467 468
    // Data structure to store all sub-objects of all elements of the macro mesh.
    ElementObjects elObjects;

    // Maps to each macro element index a pointer to the corresponding element.
469
    map<int, Element*> macroElIndexMap;
Thomas Witkowski's avatar
Thomas Witkowski committed
470 471
    
    // Maps to each macro element index the type of this element.
472
    map<int, int> macroElIndexTypeMap;
Thomas Witkowski's avatar
Thomas Witkowski committed
473

Thomas Witkowski's avatar
Thomas Witkowski committed
474
    /** \brief 
Thomas Witkowski's avatar
Thomas Witkowski committed
475 476 477 478 479 480 481 482 483 484 485 486
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
     * the object gives for every neighbour rank i the boundaries this rank owns and 
     * shares with rank i.
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are not owned by the rank,
     * i.e., the object gives for every neighbour rank i the boundaries that are
     * owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
487
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
488
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
489

490
    /** \brief
491 492
     * Defines the periodic boundaries with other ranks. Periodic boundaries have
     * no owner, as it is the case of interior boundaries.
493 494 495
     */
    InteriorBoundary periodicBoundary;

496 497 498 499
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
500
    RankToDofContainer sendDofs;
501 502

    /** \brief
503 504 505
     * This map contains for each rank the list of DOFs from which the current rank 
     * will receive DOF values (i.e., this are all DOFs at an interior boundary). The
     * DOF indices are given in rank's local numbering.
506
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
507
    RankToDofContainer recvDofs;
508 509

    /// Maps local to global dof indices.
510
    DofMapping mapLocalGlobalDofs;
511

Thomas Witkowski's avatar
Thomas Witkowski committed
512
    /// Maps local dof indices to real dof indices.
513
    DofMapping mapLocalDofIndex;  
514 515 516 517 518 519

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
520 521
    DofIndexToBool isRankDof;

522
    /** \brief
523 524 525 526
     * If periodic boundaries are used, this map stores, for each periodic boundary
     * type, for all DOFs in rank's partition (that are on periodic boundaries), the 
     * corresponding mapped periodic DOFs. The mapping is defined by using global 
     * dof indices.
527
     */
528
    PeriodicDofMap periodicDof;
529 530 531 532 533
    
    /** \brief
     * If periodic boundaries are used, this map stores to each periodic DOF in rank's
     * partition the set of periodic boundaries the DOF is associated to. In 2D, most
     * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
534
     * being periodic, the four corners are associated by two different boundaries.
535
     */
536
    map<int, std::set<BoundaryType> > periodicDofAssociations;
537

538 539 540 541 542
    
    /// This set of values must be interchanged between ranks when the mesh is 
    /// repartitioned.
    vector<DOFVector<double>*> interchangeVectors;
		        
543
    /// Is the index of the first row of the linear system, which is owned by the rank.
Thomas Witkowski's avatar
n  
Thomas Witkowski committed
544
    int rstart;
545

546 547 548 549 550 551 552
    /** \brief
     * If the problem definition has been read from a serialization file, this 
     * variable is true, otherwise it is false. This variable is used to stop the
     * initialization function, if the problem definition has already been read from
     * a serialization file.
     */
    bool deserialized;
553

554 555 556
    /// Denotes whether there exists a filewriter for this object.
    bool writeSerializationFile;

557 558 559
    /// If true, it is possible to repartition the mesh during computations.
    bool repartitioningAllowed;

560
    /// Stores the number of mesh changes that must lie in between to repartitionings.
561 562
    int repartitionIthChange;

563 564
    /// Counts the number of mesh changes after the last mesh repartitioning was done.
    int nMeshChangesAfterLastRepartitioning;
565

566 567 568
    /// Countes the number of mesh repartitions that were done. Till now, this 
    /// variable is used only for debug outputs.
    int repartitioningCounter;
569

570
    /// Directory name where all debug output files should be written to.
571
    string debugOutputDir;
572

573 574 575 576 577
    /** \brief
     * Stores the mesh change index. This is used to recognize changes in the mesh 
     * structure (e.g. through refinement or coarsening managers).
     */
    long lastMeshChangeIndex;
578

579
    map<int, vector<int> > macroElementNeighbours;
580

581 582
    /// Store all macro elements of the overall mesh, i.e., before the macro mesh is
    /// redistributed for the first time.
583
    vector<MacroElement*> allMacroElements;
584

585
    friend class ParallelDebug;
586 587 588
  };
}

589
#endif // AMDIS_MESHDISTRIBUTOR_H