Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

MeshDistributor.h 26.2 KB
Newer Older
1 2 3 4
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
5
// ==  http://www.amdis-fem.org                                              ==
6 7
// ==                                                                        ==
// ============================================================================
8 9 10 11 12 13 14 15 16 17 18 19
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


20

21
/** \file MeshDistributor.h */
22

23 24
#ifndef AMDIS_MESHDISTRIBUTOR_H
#define AMDIS_MESHDISTRIBUTOR_H
25 26


Thomas Witkowski's avatar
Thomas Witkowski committed
27
#include <mpi.h>
28
#include "parallel/DofComm.h"
29
#include "parallel/ElementObjectData.h"
30
#include "parallel/ParallelTypes.h"
31
#include "parallel/MeshPartitioner.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
32
#include "parallel/InteriorBoundary.h"
33
#include "parallel/StdMpi.h"
34
#include "AMDiS_fwd.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
35
#include "Global.h"
36 37
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
38
#include "FiniteElemSpace.h"
39
#include "Serializer.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
40
#include "BoundaryManager.h"
41
#include "SystemVector.h"
42

43
namespace AMDiS {
44 45

  using namespace std;
Thomas Witkowski's avatar
Thomas Witkowski committed
46 47 48 49


  struct BoundaryDofInfo
  {
50
    map<GeoIndex, DofContainerSet> geoDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
51 52
  };

53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77

  struct DofData
  {
    /// Number of DOFs in the rank mesh.
    int nRankDofs;

    /// Is the index of the first global DOF index, which is owned by the rank.
    int rStartDofs;

    /// Number of DOFs in the whole domain.
    int nOverallDofs;

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF 
     * is owned by the rank. Otherwise, its an interior boundary DOF that is 
     * owned by another rank.
     */
    DofIndexToBool isRankDof;

    /// Maps local to global dof indices.
    DofMapping mapLocalGlobalDofs;

    /// Maps local dof indices to real dof indices.
    DofMapping mapLocalDofIndex;  
  };
78 79


80
  class MeshDistributor
81
  {
82
  private:
83
    MeshDistributor();
84
	          
85
    virtual ~MeshDistributor() {}
86

87
  public:
88
    void initParallelization();
89

90
    void exitParallelization();
91

92 93 94
    /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector 
    /// will be automatically interchanged between ranks when mesh is 
    /// repartitioned.
95 96 97 98 99
    void addInterchangeVector(DOFVector<double> *vec)
    {
      interchangeVectors.push_back(vec);
    }

100 101 102 103 104 105 106
    /// Adds all DOFVectors of a SystemVector to \ref interchangeVecs.
    void addInterchangeVector(SystemVector *vec)
    {
      for (int i = 0; i < vec->getSize(); i++)
	interchangeVectors.push_back(vec->getDOFVector(i));
    }
    
107
    /** \brief
108 109 110 111 112
     * This function checks if the mesh has changed on at least on rank. In 
     * this case, the interior boundaries are adapted on all ranks such that 
     * they fit together on all ranks. Furthermore the function 
     * \ref updateLocalGlobalNumbering() is called to update the DOF numberings 
     * and mappings on all rank due to the new mesh structure.
113
     *
114 115 116 117 118
     * \param[in]  tryRepartition   If this parameter is true, repartitioning 
     *                              may be done. This depends on several other 
     *                              parameters. If the parameter is false, the 
     *                              mesh is only checked and adapted but never 
     *                              repartitioned.
119
     */
120
    void checkMeshChange(bool tryRepartition = true);
121

122 123 124 125 126 127 128 129 130 131
    /** \brief
     * Checks if is required to repartition the mesh. If this is the case, a new
     * partition will be created and the mesh will be redistributed between the
     * ranks.
     */
    void repartitionMesh();
    
    /// Calculates the imbalancing factor and prints it to screen.
    void printImbalanceFactor();

132
    /** \brief
133 134 135 136
     * Test, if the mesh consists of macro elements only. The mesh partitioning 
     * of the parallelization works for macro meshes only and would fail, if the 
     * mesh is already refined in some way. Therefore, this function will exit
     * the program if it finds a non macro element in the mesh.
137 138
     */
    void testForMacroMesh();
139

140 141
    /// Set for each element on the partitioning level the number of 
    /// leaf elements.
142
    void setInitialElementWeights();
143

144
    inline virtual string getName() 
145 146 147
    { 
      return name; 
    }
148

Thomas Witkowski's avatar
Thomas Witkowski committed
149 150 151 152 153
    inline Mesh* getMesh()
    {
      return mesh;
    }

154 155
    /// Returns an FE space from \ref feSpaces.
    inline const FiniteElemSpace* getFeSpace(unsigned int i = 0)
156
    {
157 158 159 160 161
      FUNCNAME("MeshDistributor::getFeSpace()");

      TEST_EXIT_DBG(i < feSpaces.size())("Should not happen!\n");

      return feSpaces[i];
162
    }
163 164 165 166 167 168 169

    /// Returns all FE spaces, thus \ref feSpaces.
    inline vector<const FiniteElemSpace*>& getFeSpaces()
    {
      return feSpaces;
    }

170
    /// Returns the number of DOFs in rank's domain for a given FE space.
171
    inline int getNumberRankDofs(const FiniteElemSpace *feSpace) 
172
    {
173 174 175 176
      FUNCNAME("MeshDistributor::getNumberRankDofs()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");

177
      return dofFeData[feSpace].nRankDofs;
178
    }
179

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
    /// Returns the number of DOFs in rank's domain for a set of FE spaces.
    inline int getNumberRankDofs(vector<const FiniteElemSpace*>& feSpaces)
    {
      FUNCNAME("MeshDistributor::getNumberRankDofs()");

      int result = 0;
      for (unsigned int i = 0; i < feSpaces.size(); i++) {
	TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n");
	result += dofFeData[feSpaces[i]].nRankDofs;
      }

      return result;
    }
    
    /// Returns the first global DOF index of an FE space, owned by rank.
195
    inline int getStartDofs(const FiniteElemSpace *feSpace)
196
    {
197 198 199 200
      FUNCNAME("MeshDistributor::getStartDofs()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");

201
      return dofFeData[feSpace].rStartDofs;
202 203
    }

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
    /// Returns the first global DOF index for a set of FE spaces, owned by rank.
    inline int getStartDofs(vector<const FiniteElemSpace*>& feSpaces)
    {
      FUNCNAME("MeshDistributor::getStartDofs()");

      int result = 0;
      for (unsigned int i = 0; i < feSpaces.size(); i++) {
	TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n");

	result += dofFeData[feSpaces[i]].rStartDofs;
      }

      return result;
    }

    /// Returns the global number of DOFs for a given FE space.
220
    inline int getNumberOverallDofs(const FiniteElemSpace *feSpace)
221
    {
222 223 224 225
      FUNCNAME("MeshDistributor::getNumberOverallDofs()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");

226
      return dofFeData[feSpace].nOverallDofs;
227
    }
228

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
    /// Returns the global number of DOFs for a set of FE spaces.
    inline int getNumberOverallDofs(vector<const FiniteElemSpace*>& feSpaces)
    {
      FUNCNAME("MeshDistributor::getNumberOverallDofs()");

      int result = 0;
      for (unsigned int i = 0; i < feSpaces.size(); i++) {
	TEST_EXIT_DBG(dofFeData.count(feSpaces[i]))("Should not happen!\n");

	result += dofFeData[feSpaces[i]].nOverallDofs;
      }

      return result;
    }

244
    inline DofMapping& getMapLocalGlobalDofs(const FiniteElemSpace *feSpace)
Thomas Witkowski's avatar
Thomas Witkowski committed
245
    {
246 247 248 249
      FUNCNAME("MeshDistributor::getMapLocalGlobalDofs()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");

250
      return dofFeData[feSpace].mapLocalGlobalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
251 252
    }

253
    /// Maps a local DOF to its global index.
254 255
    inline DegreeOfFreedom mapLocalToGlobal(const FiniteElemSpace *feSpace,
					    DegreeOfFreedom dof)
256
    {
257 258 259 260 261
      FUNCNAME("MeshDistributor::mapLocalToGlobal()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))
	("No DOF data for FE space at addr %p!\n", feSpace);

262
      return dofFeData[feSpace].mapLocalGlobalDofs[dof];
263
    }
264

265 266
    DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
				     DegreeOfFreedom dof);
267

268
    /// Maps a local DOF to its local index.
269 270
    inline DegreeOfFreedom mapLocalToDofIndex(const FiniteElemSpace *feSpace,
					      DegreeOfFreedom dof)
271
    {
272 273 274 275 276
      FUNCNAME("MeshDistributor::mapLocalToDofIndex()");

      TEST_EXIT_DBG(dofFeData.count(feSpace))
	("No DOF data for FE space at addr %p!\n", feSpace);

277
      return dofFeData[feSpace].mapLocalDofIndex[dof];
278 279
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
280
    /// Returns the periodic mapping for all boundary DOFs in rank.
281
    inline PeriodicDofMap& getPeriodicMapping(const FiniteElemSpace *feSpace)
Thomas Witkowski's avatar
Thomas Witkowski committed
282
    {
283
      return periodicDofMap[feSpace];
Thomas Witkowski's avatar
Thomas Witkowski committed
284 285
    }

286 287 288 289 290
    /// Returns for a global DOF index of a given FE space its periodic mapping
    /// for a given  boundary type.
    inline int getPeriodicMapping(const FiniteElemSpace *feSpace,
				  BoundaryType type,
				  int globalDofIndex) 
291
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
292 293
      FUNCNAME("MeshDistributor::getPeriodicMapping()");

294 295
      TEST_EXIT_DBG(periodicDofMap.count(feSpace))("Should not happen!\n");
      TEST_EXIT_DBG(periodicDofMap[feSpace][type].count(globalDofIndex) == 1)
Thomas Witkowski's avatar
Thomas Witkowski committed
296 297
	("There is no periodic association for global DOF %d for boundary type %d!\n",
	 globalDofIndex, type);
298

299
      return periodicDofMap[feSpace][type][globalDofIndex];
300 301
    }

302
    /// For a given global DOF index, this function returns the set of periodic
303 304
    /// associations, i.e., the boundary types the DOF is associated to, for 
    /// this DOF.
305 306 307 308 309 310 311 312
    inline std::set<BoundaryType>& getPerDofAssociations(const FiniteElemSpace* feSpace,
							 int globalDofIndex)
    {
      FUNCNAME("MeshDistributor::getPerDofAssociations()");

      TEST_EXIT_DBG(periodicDofAssociations.count(feSpace))
	("Should not happen!\n");
      TEST_EXIT_DBG(periodicDofAssociations[feSpace].count(globalDofIndex)) 
313 314
 	("Should not happen!\n"); 

315
      return periodicDofAssociations[feSpace][globalDofIndex];
316
    }
317

318
    /// Returns true, if the DOF (global index) is a periodic DOF.
319
    inline bool isPeriodicDof(const FiniteElemSpace *feSpace, int globalDofIndex)
320
    {
321 322
      return (periodicDofAssociations[feSpace].count(globalDofIndex) > 0 &&
	       periodicDofAssociations[feSpace][globalDofIndex].size() > 0);
323 324
    }

325 326 327 328 329
    /// Returns true, if the DOF (global index) of a given FE space is a 
    /// periodic DOF for the given boundary type.
    inline bool isPeriodicDof(const FiniteElemSpace *feSpace,
			      BoundaryType type,
			      int globalDofIndex)
330
    {
331
      return (periodicDofMap[feSpace][type].count(globalDofIndex) > 0);
332 333
    }

334
    DofComm& getSendDofs()
335 336 337 338
    {
      return sendDofs;
    }

339
    DofComm& getRecvDofs()
340 341 342 343
    {
      return recvDofs;
    }

344 345 346 347 348
    DofComm& getPeriodicDofs()
    {
      return periodicDofs;
    }

349 350
    /// Return true, if the given DOF is owned by the rank. If false, the DOF
    /// is in rank's partition, but is owned by some other rank.
351
    inline bool getIsRankDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)
352
    {
353 354
      if (dofFeData[feSpace].isRankDof.count(dof))
	return dofFeData[feSpace].isRankDof[dof];
355 356

      return false;
357
    }
358

359
    inline DofIndexToBool& getIsRankDof(const FiniteElemSpace *feSpace)
360
    {
361
      return dofFeData[feSpace].isRankDof;
362 363
    }

364
    inline long getLastMeshChangeIndex()
365
    {
366
      return lastMeshChangeIndex;
367
    }
368

369
    inline int getMpiRank()
370
    {
371
      return mpiRank;
372
    }
373

Thomas Witkowski's avatar
Thomas Witkowski committed
374 375 376 377 378
    inline int getMpiSize()
    {
      return mpiSize;
    }

379 380 381
    inline MPI::Intracomm& getMpiComm()
    {
      return mpiComm;
382 383
    }

384 385
    /// Creates a set of all DOFs that are on interior boundaries of rank's
    /// domain. Thus, it creates the union of \ref sendDofs and \ref recvDofs.
386 387
    void createBoundaryDofs(const FiniteElemSpace *feSpace,
			    std::set<DegreeOfFreedom> &boundaryDofs);
388

389
    // Writes all data of this object to an output stream.
390
    void serialize(ostream &out);
391

392
    // Reads the object data from an input stream.
393
    void deserialize(istream &in);
394 395

    /** \brief
396 397 398 399
     * This function must be used if the values of a DOFVector must be 
     * synchronised over all ranks. That means, that each rank sends the 
     * values of the DOFs, which are owned by the rank and lie on an interior 
     * bounday, to all other ranks also having these DOFs.
400
     *
401 402 403
     * This function must be used, for example, after the lineary system is 
     * solved, or after the DOFVector is set by some user defined functions, 
     * e.g., initial solution functions.
404
     */    
405 406 407 408 409
    template<typename T>
    void synchVector(DOFVector<T> &vec) 
    {
      StdMpi<vector<T> > stdMpi(mpiComm);

410 411
      const FiniteElemSpace *fe = vec.getFeSpace();

412
      for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) {
413
	vector<T> dofs;
414
	dofs.reserve(it.getDofs().size());
415
	
416 417
	for (; !it.endDofIter(); it.nextDof())
	  dofs.push_back(vec[it.getDofIndex()]);
418
	
419
	stdMpi.send(it.getRank(), dofs);
420
      }
421 422 423 424
	     
      for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank())
        stdMpi.recv(it.getRank());
	     
425
      stdMpi.startCommunication();
426 427 428 429 430

      for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank())
	for (; !it.endDofIter(); it.nextDof())
	  vec[it.getDofIndex()] = 
	     stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
431 432
    }
    
433
    /** \brief
434 435 436
     * Works in the same way as the function above defined for DOFVectors. Due
     * to performance, this function does not call \ref synchVector for each 
     * DOFVector, but instead sends all values of all DOFVectors all at once.
437 438 439
     */
    void synchVector(SystemVector &vec);

440 441
    void check3dValidMesh();

Thomas Witkowski's avatar
Thomas Witkowski committed
442 443 444 445 446
    void setBoundaryDofRequirement(Flag flag)
    {
      createBoundaryDofFlag = flag;
    }

447
    BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace)
448
    {
449
      return boundaryDofInfo[feSpace];
450 451
    }

452 453
    void getAllBoundaryDofs(const FiniteElemSpace *feSpace,
			    DofContainer& dofs);
454

455 456 457 458 459 460

  public:
    /// Adds a stationary problem to the global mesh distributor objects.
    static void addProblemStatGlobal(ProblemStatSeq *probStat);

    
461
  protected:
462 463
    void addProblemStat(ProblemStatSeq *probStat);

464 465
    /// Determines the interior boundaries, i.e. boundaries between ranks, and
    /// stores all information about them in \ref interiorBoundary.
466
    void createInteriorBoundaryInfo();
467

Thomas Witkowski's avatar
Thomas Witkowski committed
468 469 470 471 472
    void updateInteriorBoundaryInfo();

    void createMeshElementData();

    void createBoundaryData();
Thomas Witkowski's avatar
Thomas Witkowski committed
473

Thomas Witkowski's avatar
Thomas Witkowski committed
474 475
    void createBoundaryDofs();

476 477
    void createBoundaryDofs(const FiniteElemSpace *feSpace);

478 479
    /// Removes all macro elements from the mesh that are not part of ranks 
    /// partition.
480 481
    void removeMacroElements();

482 483
    void updateLocalGlobalNumbering();

484 485
    /// Updates the local and global DOF numbering after the mesh has been 
    /// changed.
486
    void updateLocalGlobalNumbering(const FiniteElemSpace *feSpace);
487

488 489 490 491
    /// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are
    /// handled by the mesh distributor.
    void createPeriodicMap();

492
    /** \brief
493 494 495 496
     * Creates, for a specific FE space, to all DOFs in rank's partition that 
     * are on a periodic boundary the mapping from dof index to the other 
     * periodic dof indices. This information is stored in \ref periodicDofMap.
     */    
497
    void createPeriodicMap(const FiniteElemSpace *feSpace);
498

499 500 501 502 503 504 505 506
    /** \brief
     * This function is called only once during the initialization when the
     * whole macro mesh is available on all cores. It copies the pointers of all
     * macro elements to \ref allMacroElements and stores all neighbour 
     * information based on macro element indices (and not pointer based) in 
     * \ref macroElementNeighbours. These information are then used to 
     * reconstruct macro elements during mesh redistribution.
     */
507 508
    void createMacroElementInfo();

509 510
    void updateMacroElementInfo();

511
    /** \brief
512 513 514 515 516 517
     * Checks for all given interior boundaries if the elements fit together on
     * both sides of the boundaries. If this is not the case, the mesh is 
     * adapted. Because refinement of a certain element may forces the 
     * refinement of other elements, it is not guaranteed that all rank's meshes
     * fit together after this function terminates. Hence, it must be called 
     * until a stable mesh refinement is reached.
518
     *
519 520
     * \param[in] allBound   Defines a map from rank to interior boundaries 
     *                       which should be checked.
521
     *
522 523 524
     * \return    If the mesh has  been changed by this function, it returns 
     *            true. Otherwise, it returns false, i.e., the given interior 
     *            boundaries fit together on both sides.
525 526
     */
    bool checkAndAdaptBoundary(RankToBoundMap &allBound);
527
  
528 529
    /// Sets \ref isRankDof to all matrices and rhs vectors in a given 
    /// stationary problem.
530
    void setRankDofs(ProblemStatSeq *probStat);
531

532 533
    /// Sets \ref isRankDof to all matrices and rhs vectors in all 
    /// stationary problems.
534 535
    void setRankDofs();

536 537 538 539
    /// Removes all periodic boundary condition information from all matrices and
    /// vectors of all stationary problems and from the mesh itself.
    void removePeriodicBoundaryConditions();

Thomas Witkowski's avatar
Thomas Witkowski committed
540
    // Removes all periodic boundaries from a given boundary map.
541
    void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
542

543
    /// Writes a vector of dof pointers to an output stream.
544
    void serialize(ostream &out, DofContainer &data);
545

546 547 548 549
    /// Writes a \ref RankToDofContainer to an output stream.
    void serialize(ostream &out, 
		   map<int, map<const FiniteElemSpace*, DofContainer> > &data);

550
    /// Reads a vector of dof pointers from an input stream.
551 552
    void deserialize(istream &in, DofContainer &data,
		     map<int, const DegreeOfFreedom*> &dofMap);
553 554

    /// Reads a \ref RankToDofContainer from an input stream.
555 556 557
    void deserialize(istream &in, 
		     map<int, map<const FiniteElemSpace*, DofContainer> > &data,
		     map<const FiniteElemSpace*, map<int, const DegreeOfFreedom*> > &dofMap);
558

559
    /// Writes a periodic dof mapping to an output stream.
560
    void serialize(ostream &out, PeriodicDofMap &data);
561

562
    void serialize(ostream &out, map<int, std::set<int> >& data);
563

564
    /// Reads a periodic dof mapping from an input stream.
565
    void deserialize(istream &in, PeriodicDofMap &data);
566

567
    void deserialize(istream &in, map<int, std::set<int> >& data);
568

569 570
    /// Writes a mapping from dof pointers to some values to an output stream.
    template<typename T>
571
    void serialize(ostream &out, map<const DegreeOfFreedom*, T> &data)
572
    {
573 574
      FUNCNAME("ParallelDomainBase::serialize()");

575
      int mapSize = data.size();
576
      SerUtil::serialize(out, mapSize);
577
      for (typename map<const DegreeOfFreedom*, T>::iterator it = data.begin();
578 579 580
	   it != data.end(); ++it) {
	int v1 = (*(it->first));
	T v2 = it->second;
581 582
	SerUtil::serialize(out, v1);
	SerUtil::serialize(out, v2);
583 584 585 586 587
      }
    }

    /// Reads a mapping from dof pointer to some values from an input stream.
    template<typename T>
588 589
    void deserialize(istream &in, map<const DegreeOfFreedom*, T> &data,
		     map<int, const DegreeOfFreedom*> &dofMap)
590
    {
591 592
      FUNCNAME("ParallelDomainBase::deserialize()");

593
      int mapSize = 0;
594
      SerUtil::deserialize(in, mapSize);
595 596 597
      for (int i = 0; i < mapSize; i++) {
	int v1 = 0;
	T v2;
598 599
	SerUtil::deserialize(in, v1);
	SerUtil::deserialize(in, v2);
600 601 602

	TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);

603 604 605
	data[dofMap[v1]] = v2;
      }
    }
606

607
  protected:
608 609
    /// List of all stationary problems that are managed by this mesh 
    /// distributor.
610
    vector<ProblemStatSeq*> problemStat;
Thomas Witkowski's avatar
Thomas Witkowski committed
611

612 613 614
    /// If true, the mesh distributor is already initialized;
    bool initialized;

615 616 617 618 619 620
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

621 622
    /// MPI communicator collected all processes, which should be used for
    /// calculation. The Debug procces is not included in this communicator.
623 624 625
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
626
    string name;
627

628 629
    /// Finite element spaces of the problem.
    vector<const FiniteElemSpace*> feSpaces;
630

631 632 633
    /// Mesh of the problem.
    Mesh *mesh;

634
    /** \brief
635 636 637
     * A refinement manager that should be used on the mesh. It is used to 
     * refine elements at interior boundaries in order to fit together with 
     * elements on the other side of the interior boundary.
638 639 640
     */    
    RefinementManager *refineManager;

641 642 643
    /// Info level.
    int info;

644 645
    /// Pointer to a mesh partitioner that is used to partition the mesh to 
    /// the ranks.
646
    MeshPartitioner *partitioner;
647

648 649
    /// Weights for the elements, i.e., the number of leaf elements within 
    /// this element.
650
    map<int, double> elemWeights;
651 652

    /** \brief
653 654
     * Stores to every macro element index the number of the rank that owns this
     * macro element.
655
     */
656
    map<int, int> partitionMap;
657

658
    map<const FiniteElemSpace*, DofData> dofFeData;
659

660 661
    /// Data structure to store all sub-objects of all elements of the 
    /// macro mesh.
Thomas Witkowski's avatar
Thomas Witkowski committed
662 663
    ElementObjects elObjects;

664
    /// Maps to each macro element index a pointer to the corresponding element.
665
    map<int, Element*> macroElIndexMap;
Thomas Witkowski's avatar
Thomas Witkowski committed
666
    
667
    /// Maps to each macro element index the type of this element.
668
    map<int, int> macroElIndexTypeMap;
Thomas Witkowski's avatar
Thomas Witkowski committed
669

Thomas Witkowski's avatar
Thomas Witkowski committed
670
    /** \brief 
671 672 673 674
     * Defines the interior boundaries of the domain that result from 
     * partitioning the whole mesh. Contains only the boundaries, which are 
     * owned by the rank, i.e., the object gives for every neighbour rank i 
     * the boundaries this rank owns and shares with rank i.
Thomas Witkowski's avatar
Thomas Witkowski committed
675 676 677 678
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
679 680 681 682
     * Defines the interior boundaries of the domain that result from 
     * partitioning the whole mesh. Contains only the boundaries, which are 
     * not owned by the rank, i.e., the object gives for every neighbour rank 
     * i the boundaries that are owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
683
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
684
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
685

686
    /** \brief
687 688
     * Defines the periodic boundaries with other ranks. Periodic boundaries
     * have no owner, as it is the case of interior boundaries.
689 690 691
     */
    InteriorBoundary periodicBoundary;

692
    /** \brief
693 694
     * This map contains for each rank the list of DOFs the current rank must 
     * send to exchange solution DOFs at the interior boundaries.
695
     */
696
    DofComm sendDofs;
697 698

    /** \brief
699 700 701
     * This map contains on each rank the list of DOFs from which the current 
     * rank will receive DOF values (i.e., this are all DOFs at an interior 
     * boundary). The DOF indices are given in rank's local numbering.
702
     */
703
    DofComm recvDofs;
704

705 706 707 708 709 710 711 712
    /** \brief
     * This map contains on each rank a list of DOFs along the interior bound-
     * aries to communicate with other ranks. The DOF indices are given in rank's
     * local numbering. Periodic boundaries within one subdomain are not 
     * considered here. 
     */
    DofComm periodicDofs;

713
    /** \brief
714 715 716 717
     * If periodic boundaries are used, this map stores, for each periodic 
     * boundary type, for all DOFs in rank's partition (that are on periodic 
     * boundaries), the corresponding mapped periodic DOFs. The mapping is 
     * defined by using global DOF indices.
718
     */
719
    PeriodicDofMapFeSpace periodicDofMap;
720 721
    
    /** \brief
722 723 724 725 726
     * If periodic boundaries are used, this map stores to each periodic DOF in 
     * rank's partition the set of periodic boundaries the DOF is associated to.
     * In 2D, most DOFs are only on one periodic boundary. Only, e.g., in a box 
     * with all boundaries being periodic, the four corners are associated by 
     * two different boundaries.
727
     */
728
    map<const FiniteElemSpace*, map<DegreeOfFreedom, std::set<BoundaryType> > > periodicDofAssociations;
729

730 731 732 733 734
    
    /// This set of values must be interchanged between ranks when the mesh is 
    /// repartitioned.
    vector<DOFVector<double>*> interchangeVectors;
		        
735 736 737
    /** \brief
     * If the problem definition has been read from a serialization file, this 
     * variable is true, otherwise it is false. This variable is used to stop the
738 739
     * initialization function, if the problem definition has already been read
     * from a serialization file.
740 741
     */
    bool deserialized;
742

743 744 745
    /// Denotes whether there exists a filewriter for this object.
    bool writeSerializationFile;

746 747 748
    /// If true, it is possible to repartition the mesh during computations.
    bool repartitioningAllowed;

749 750
    /// Stores the number of mesh changes that must lie in between to 
    /// repartitionings.
751 752
    int repartitionIthChange;

753 754
    /// Counts the number of mesh changes after the last mesh repartitioning 
    /// was done.
755
    int nMeshChangesAfterLastRepartitioning;
756

757 758 759
    /// Countes the number of mesh repartitions that were done. Till now, this 
    /// variable is used only for debug outputs.
    int repartitioningCounter;
760

761
    /// Directory name where all debug output files should be written to.
762
    string debugOutputDir;
763

764
    /** \brief
765 766
     * Stores the mesh change index. This is used to recognize changes in the
     * mesh structure (e.g. through refinement or coarsening managers).
767 768
     */
    long lastMeshChangeIndex;
769

770 771 772 773
    /// Stores for all macro elements of the original macro mesh the
    /// neighbourhood information based on element indices. Thus, each macro
    /// element index is mapped to a vector containing all indices of 
    /// neighbouring macro elements.
774
    map<int, vector<int> > macroElementNeighbours;
775

776 777
    /// Store all macro elements of the overall mesh, i.e., before the
    /// mesh is redistributed for the first time.
778
    vector<MacroElement*> allMacroElements;
779

Thomas Witkowski's avatar
Thomas Witkowski committed
780 781
    Flag createBoundaryDofFlag;

782
    map<const FiniteElemSpace*, BoundaryDofInfo> boundaryDofInfo;
783

Thomas Witkowski's avatar
Thomas Witkowski committed
784
  public:
785 786 787
    /// The boundary DOFs are sorted by subobject entities, i.e., first all
    /// face DOFs, edge DOFs and to the last vertex DOFs will be set to
    /// communication structure vectors, \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
Thomas Witkowski committed
788 789
    static const Flag BOUNDARY_SUBOBJ_SORTED;

790 791 792 793 794 795 796 797 798
    /// When boundary DOFs are created, \ref boundaryDofInfo is filled for
    /// all DOFs that this rank will send to other ranks (thus, rank 
    /// owned DOFs.
    static const Flag BOUNDARY_FILL_INFO_SEND_DOFS;

    /// When boundary DOFs are created, \ref boundaryDofInfo is filled for
    /// all DOFs that this rank will receive from other ranks (thus, DOFs
    /// that are owned by another rank).
    static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
Thomas Witkowski's avatar
Thomas Witkowski committed
799

800 801
    static MeshDistributor *globalMeshDistributor;

802
    friend class ParallelDebug;
803 804 805
  };
}

806
#endif // AMDIS_MESHDISTRIBUTOR_H