// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// == http://www.amdis-fem.org ==
// == ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.
/** \file MeshDistributor.h */
#ifndef AMDIS_MESHDISTRIBUTOR_H
#define AMDIS_MESHDISTRIBUTOR_H
#include
#include "parallel/ParallelTypes.h"
#include "parallel/MeshPartitioner.h"
#include "parallel/InteriorBoundary.h"
#include "AMDiS_fwd.h"
#include "Global.h"
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "FiniteElemSpace.h"
#include "Serializer.h"
#include "BoundaryManager.h"
#include "ElementObjectData.h"
#include "SystemVector.h"
namespace AMDiS {
using namespace std;
struct BoundaryDofInfo
{
map geoDofs;
};
class MeshDistributor
{
public:
MeshDistributor();
virtual ~MeshDistributor() {}
void initParallelization();
void exitParallelization();
void addProblemStat(ProblemStatSeq *probStat);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector *vec)
{
interchangeVectors.push_back(vec);
}
/// Adds all DOFVectors of a SystemVector to \ref interchangeVecs.
void addInterchangeVector(SystemVector *vec)
{
for (int i = 0; i < vec->getSize(); i++)
interchangeVectors.push_back(vec->getDOFVector(i));
}
/** \brief
* This function checks if the mesh has changed on at least on rank. In this case,
* the interior boundaries are adapted on all ranks such that they fit together on
* all ranks. Furthermore the function \ref updateLocalGlobalNumbering() is called
* to update the DOF numberings and mappings on all rank due to the new mesh
* structure.
*
* \param[in] tryRepartition If this parameter is true, repartitioning may be
* done. This depends on several other parameters. If
* the parameter is false, the mesh is only checked
* and adapted but never repartitioned.
*/
void checkMeshChange(bool tryRepartition = true);
/** \brief
* Test, if the mesh consists of macro elements only. The mesh partitioning of
* the parallelization works for macro meshes only and would fail, if the mesh
* is already refined in some way. Therefore, this function will exit the program
* if it finds a non macro element in the mesh.
*/
void testForMacroMesh();
/// Set for each element on the partitioning level the number of leaf elements.
void setInitialElementWeights();
inline virtual string getName()
{
return name;
}
inline Mesh* getMesh()
{
return mesh;
}
/// Returns \ref feSpace.
inline const FiniteElemSpace* getFeSpace()
{
return feSpace;
}
/// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
inline int getNumberRankDofs()
{
return nRankDofs;
}
/// Returns \ref nOverallDofs, the global number of DOFs.
inline int getNumberOverallDofs()
{
return nOverallDofs;
}
inline DofMapping& getMapLocalGlobalDofs()
{
return mapLocalGlobalDofs;
}
/// Maps a local DOF to its global index.
inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
{
return mapLocalGlobalDofs[dof];
}
DegreeOfFreedom mapGlobalToLocal(DegreeOfFreedom dof);
/// Maps a local DOF to its local index.
inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
{
return mapLocalDofIndex[dof];
}
/// Returns the periodic mapping for all boundary DOFs in rank.
inline PeriodicDofMap& getPeriodicMapping()
{
return periodicDof;
}
/// Returns for a global dof index its periodic mapping for a given boundary type.
inline int getPeriodicMapping(int globalDofIndex, BoundaryType type)
{
FUNCNAME("MeshDistributor::getPeriodicMapping()");
TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
("There is no periodic association for global DOF %d for boundary type %d!\n",
globalDofIndex, type);
return periodicDof[type][globalDofIndex];
}
/// For a given global DOF index, this function returns the set of periodic
/// associations, i.e., the boundary types the DOF is associated to, for this DOF.
inline std::set& getPerDofAssociations(int globalDofIndex)
{
TEST_EXIT_DBG(periodicDofAssociations.count(globalDofIndex))
("Should not happen!\n");
return periodicDofAssociations[globalDofIndex];
}
/// Returns true, if the DOF (global index) is a periodic DOF.
inline bool isPeriodicDof(int globalDofIndex)
{
return (periodicDofAssociations.count(globalDofIndex) > 0 &&
periodicDofAssociations[globalDofIndex].size() > 0);
}
/// Returns true, if the DOF (global index) is a periodic DOF for the given
/// boundary type.
inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
{
return (periodicDof[type].count(globalDofIndex) > 0);
}
/// Return true, if the given DOF is owned by the rank. If false, the DOF is in
/// rank's partition, but is owned by some other rank.
inline bool getIsRankDof(DegreeOfFreedom dof)
{
if (isRankDof.count(dof))
return isRankDof[dof];
return false;
}
inline DofIndexToBool& getIsRankDof()
{
return isRankDof;
}
inline long getLastMeshChangeIndex()
{
return lastMeshChangeIndex;
}
inline int getRstart()
{
return rstart;
}
inline int getMpiRank()
{
return mpiRank;
}
inline int getMpiSize()
{
return mpiSize;
}
inline MPI::Intracomm& getMpiComm()
{
return mpiComm;
}
inline RankToDofContainer& getSendDofs()
{
return sendDofs;
}
inline RankToDofContainer& getRecvDofs()
{
return recvDofs;
}
// Writes all data of this object to an output stream.
void serialize(ostream &out);
// Reads the object data from an input stream.
void deserialize(istream &in);
/** \brief
* This function must be used if the values of a DOFVector must be synchronised
* over all ranks. That means, that each rank sends the values of the DOFs, which
* are owned by the rank and lie on an interior bounday, to all other ranks also
* having these DOFs.
*
* This function must be used, for example, after the lineary system is solved, or
* after the DOFVector is set by some user defined functions, e.g., initial
* solution functions.
*/
void synchVector(DOFVector &vec);
/** \brief
* Works in the same way as the function above defined for DOFVectors. Due to
* performance, this function does not call \ref synchVector for each DOFVector,
* but instead sends all values of all DOFVectors all at once.
*/
void synchVector(SystemVector &vec);
void check3dValidMesh();
void setBoundaryDofRequirement(Flag flag)
{
createBoundaryDofFlag = flag;
}
BoundaryDofInfo& getBoundaryDofInfo()
{
return boundaryDofInfo;
}
void getAllBoundaryDofs(DofContainer& dofs);
protected:
/** \brief
* Determines the interior boundaries, i.e. boundaries between ranks, and stores
* all information about them in \ref interiorBoundary.
*/
void createInteriorBoundaryInfo();
void updateInteriorBoundaryInfo();
void createMeshElementData();
void createBoundaryData();
void createBoundaryDofs();
/// Removes all macro elements from the mesh that are not part of ranks partition.
void removeMacroElements();
/// Updates the local and global DOF numbering after the mesh has been changed.
void updateLocalGlobalNumbering();
/** \brief
* Creates to all dofs in rank's partition that are on a periodic boundary the
* mapping from dof index to the other periodic dof indices. This information
* is stored in \ref periodicDof.
*/
void createPeriodicMap();
void createMacroElementInfo();
void updateMacroElementInfo();
/** \brief
* Checks for all given interior boundaries if the elements fit together on both
* sides of the boundaries. If this is not the case, the mesh is adapted. Because
* refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached.
*
* \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/
bool checkAndAdaptBoundary(RankToBoundMap &allBound);
/** \brief
* Checks if is required to repartition the mesh. If this is the case, a new
* partition will be created and the mesh will be redistributed between the
* ranks.
*/
void repartitionMesh();
/// Sets \ref isRankDof to all matrices and rhs vectors in a given
/// stationary problem.
void setRankDofs(ProblemStatSeq *probStat);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs();
/// Removes all periodic boundary condition information from all matrices and
/// vectors of all stationary problems and from the mesh itself.
void removePeriodicBoundaryConditions();
// Removes all periodic boundaries from a given boundary map.
void removePeriodicBoundaryConditions(BoundaryIndexMap& boundaryMap);
/// Writes a vector of dof pointers to an output stream.
void serialize(ostream &out, DofContainer &data);
/// Reads a vector of dof pointers from an input stream.
void deserialize(istream &in, DofContainer &data,
map &dofMap);
/// Writes a \ref RankToDofContainer to an output stream.
void serialize(ostream &out, RankToDofContainer &data);
/// Reads a \ref RankToDofContainer from an input stream.
void deserialize(istream &in, RankToDofContainer &data,
map &dofMap);
/// Writes a periodic dof mapping to an output stream.
void serialize(ostream &out, PeriodicDofMap &data);
void serialize(ostream &out, map >& data);
/// Reads a periodic dof mapping from an input stream.
void deserialize(istream &in, PeriodicDofMap &data);
void deserialize(istream &in, map >& data);
/// Writes a mapping from dof pointers to some values to an output stream.
template
void serialize(ostream &out, map &data)
{
FUNCNAME("ParallelDomainBase::serialize()");
int mapSize = data.size();
SerUtil::serialize(out, mapSize);
for (typename map::iterator it = data.begin();
it != data.end(); ++it) {
int v1 = (*(it->first));
T v2 = it->second;
SerUtil::serialize(out, v1);
SerUtil::serialize(out, v2);
}
}
/// Reads a mapping from dof pointer to some values from an input stream.
template
void deserialize(istream &in, map &data,
map &dofMap)
{
FUNCNAME("ParallelDomainBase::deserialize()");
int mapSize = 0;
SerUtil::deserialize(in, mapSize);
for (int i = 0; i < mapSize; i++) {
int v1 = 0;
T v2;
SerUtil::deserialize(in, v1);
SerUtil::deserialize(in, v2);
TEST_EXIT_DBG(dofMap.count(v1) != 0)("Cannot find DOF %d in map!\n", v1);
data[dofMap[v1]] = v2;
}
}
protected:
/// List of all stationary problems that are managed by this mesh distributor.
vector problemStat;
/// If true, the mesh distributor is already initialized;
bool initialized;
/// The rank of the current process.
int mpiRank;
/// Overall number of processes.
int mpiSize;
/** \brief
* MPI communicator collected all processes, which should
* be used for calculation. The Debug procces is not included
* in this communicator.
*/
MPI::Intracomm mpiComm;
/// Name of the problem (as used in the init files)
string name;
/// Finite element space of the problem.
FiniteElemSpace *feSpace;
/// Mesh of the problem.
Mesh *mesh;
/** \brief
* A refinement manager that should be used on the mesh. It is used to refine
* elements at interior boundaries in order to fit together with elements on the
* other side of the interior boundary.
*/
RefinementManager *refineManager;
/// Info level.
int info;
/// Pointer to a mesh partitioner that is used to partition the mesh to the ranks.
MeshPartitioner *partitioner;
/// Weights for the elements, i.e., the number of leaf elements within this element.
map elemWeights;
/** \brief
* Stores to every macro element index the number of the rank that owns this
* macro element.
*/
map partitionMap;
/// Number of DOFs in the rank mesh.
int nRankDofs;
/// Number of DOFs in the whole domain.
int nOverallDofs;
// Data structure to store all sub-objects of all elements of the macro mesh.
ElementObjects elObjects;
// Maps to each macro element index a pointer to the corresponding element.
map macroElIndexMap;
// Maps to each macro element index the type of this element.
map macroElIndexTypeMap;
/** \brief
* Defines the interior boundaries of the domain that result from partitioning
* the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
* the object gives for every neighbour rank i the boundaries this rank owns and
* shares with rank i.
*/
InteriorBoundary myIntBoundary;
/** \brief
* Defines the interior boundaries of the domain that result from partitioning
* the whole mesh. Contains only the boundaries, which are not owned by the rank,
* i.e., the object gives for every neighbour rank i the boundaries that are
* owned by rank i and are shared with this rank.
*/
InteriorBoundary otherIntBoundary;
/** \brief
* Defines the periodic boundaries with other ranks. Periodic boundaries have
* no owner, as it is the case of interior boundaries.
*/
InteriorBoundary periodicBoundary;
/** \brief
* This map contains for each rank the list of DOFs the current rank must send
* to exchange solution DOFs at the interior boundaries.
*/
RankToDofContainer sendDofs;
/** \brief
* This map contains on each rank the list of DOFs from which the current rank
* will receive DOF values (i.e., this are all DOFs at an interior boundary). The
* DOF indices are given in rank's local numbering.
*/
RankToDofContainer recvDofs;
/// Maps local to global dof indices.
DofMapping mapLocalGlobalDofs;
/// Maps local dof indices to real dof indices.
DofMapping mapLocalDofIndex;
/** \brief
* Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
* owned by the rank. Otherwise, its an interior boundary DOF that is owned by
* another rank.
*/
DofIndexToBool isRankDof;
/** \brief
* If periodic boundaries are used, this map stores, for each periodic boundary
* type, for all DOFs in rank's partition (that are on periodic boundaries), the
* corresponding mapped periodic DOFs. The mapping is defined by using global
* dof indices.
*/
PeriodicDofMap periodicDof;
/** \brief
* If periodic boundaries are used, this map stores to each periodic DOF in rank's
* partition the set of periodic boundaries the DOF is associated to. In 2D, most
* DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
* being periodic, the four corners are associated by two different boundaries.
*/
map > periodicDofAssociations;
/// This set of values must be interchanged between ranks when the mesh is
/// repartitioned.
vector*> interchangeVectors;
/// Is the index of the first row of the linear system, which is owned by the rank.
int rstart;
/** \brief
* If the problem definition has been read from a serialization file, this
* variable is true, otherwise it is false. This variable is used to stop the
* initialization function, if the problem definition has already been read from
* a serialization file.
*/
bool deserialized;
/// Denotes whether there exists a filewriter for this object.
bool writeSerializationFile;
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange;
/// Counts the number of mesh changes after the last mesh repartitioning was done.
int nMeshChangesAfterLastRepartitioning;
/// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int repartitioningCounter;
/// Directory name where all debug output files should be written to.
string debugOutputDir;
/** \brief
* Stores the mesh change index. This is used to recognize changes in the mesh
* structure (e.g. through refinement or coarsening managers).
*/
long lastMeshChangeIndex;
map > macroElementNeighbours;
/// Store all macro elements of the overall mesh, i.e., before the macro mesh is
/// redistributed for the first time.
vector allMacroElements;
Flag createBoundaryDofFlag;
BoundaryDofInfo boundaryDofInfo;
public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
/// communication structure vectors, \ref sendDofs and \ref recvDofs.
static const Flag BOUNDARY_SUBOBJ_SORTED;
/// When boundary DOFs are created, \ref boundaryDofInfo is filled for
/// all DOFs that this rank will send to other ranks (thus, rank
/// owned DOFs.
static const Flag BOUNDARY_FILL_INFO_SEND_DOFS;
/// When boundary DOFs are created, \ref boundaryDofInfo is filled for
/// all DOFs that this rank will receive from other ranks (thus, DOFs
/// that are owned by another rank).
static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
friend class ParallelDebug;
};
}
#endif // AMDIS_MESHDISTRIBUTOR_H