Commit 859d3918 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Added comments to the parallel DOF mapping class.

parent 21017269
...@@ -17,7 +17,7 @@ namespace AMDiS { ...@@ -17,7 +17,7 @@ namespace AMDiS {
using namespace std; using namespace std;
void GlobalDofMap::clear() void FeSpaceDofMap::clear()
{ {
dofMap.clear(); dofMap.clear();
nRankDofs = 0; nRankDofs = 0;
...@@ -26,37 +26,47 @@ namespace AMDiS { ...@@ -26,37 +26,47 @@ namespace AMDiS {
} }
void GlobalDofMap::update() void FeSpaceDofMap::update()
{ {
// === Compute local indices for all rank owned DOFs. ===
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0) if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++; it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0; nOverallDofs = 0;
rStartDofs = 0; rStartDofs = 0;
mpi::getDofNumbering(*mpiComm, nRankDofs, rStartDofs, nOverallDofs); mpi::getDofNumbering(*mpiComm, nRankDofs, rStartDofs, nOverallDofs);
if (needGlobalMapping) // === If required, compute also the global indices. ===
computeGlobalMapping(rStartDofs);
if (hasNonLocalDofs) if (needGlobalMapping) {
computeNonLocalIndices(); computeGlobalMapping();
if (hasNonLocalDofs)
computeNonLocalIndices();
}
} }
void GlobalDofMap::computeGlobalMapping(int offset) void FeSpaceDofMap::computeGlobalMapping()
{ {
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
it->second.global = it->second.local + offset; it->second.global = it->second.local + rStartDofs;
} }
void GlobalDofMap::computeNonLocalIndices() void FeSpaceDofMap::computeNonLocalIndices()
{ {
FUNCNAME("GlobalDofMap::computeNonLocalIndices()"); FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;
// === Send all global indices of DOFs that are owned by the rank to all ===
// === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(*mpiComm); StdMpi<vector<int> > stdMpi(*mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
...@@ -65,6 +75,9 @@ namespace AMDiS { ...@@ -65,6 +75,9 @@ namespace AMDiS {
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
// === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) { for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
bool recvFromRank = false; bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
...@@ -78,8 +91,14 @@ namespace AMDiS { ...@@ -78,8 +91,14 @@ namespace AMDiS {
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
} }
// === Start communication to exchange global indices. ===
stdMpi.startCommunication(); stdMpi.startCommunication();
// === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(*recvDofs, feSpace); for (DofComm::Iterator it(*recvDofs, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int i = 0; int i = 0;
...@@ -90,20 +109,48 @@ namespace AMDiS { ...@@ -90,20 +109,48 @@ namespace AMDiS {
} }
void GlobalDofMap::print() void ParallelDofMapping::init(MPI::Intracomm *m,
vector<const FiniteElemSpace*> &fe,
bool needGlobalMapping,
bool bNonLocalDofs)
{ {
FUNCNAME("GlobalDofMap::print()"); FUNCNAME("ParallelDofMapping::init()");
mpiComm = m;
feSpaces = fe;
hasNonLocalDofs = bNonLocalDofs;
MSG("Local to global mapping on this rank: \n"); // === Create a set of unique FE spaces. ===
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (unsigned int i = 0; i < feSpaces.size(); i++)
if (nonRankDofs.count(it->first) == 0) feSpacesUnique.insert(feSpaces[i]);
MSG(" %d -> %d (rank-dof)\n", it->first, it->second.local);
else
MSG(" %d -> %d \n", it->first, it->second.local); // === Init the mapping for all different FE spaces. ===
for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it) {
addFeSpace(*it);
data[*it].setNeedGlobalMapping(needGlobalMapping);
data[*it].setNonLocalDofs(hasNonLocalDofs);
}
} }
void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
{
FUNCNAME("ParallelDofMapping::setDofComm()");
sendDofs = &pSend;
recvDofs = &pRecv;
// Add the DOF communicator also to all FE space DOF mappings.
for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it)
data[*it].setDofComm(pSend, pRecv);
}
void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace) void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
{ {
FUNCNAME("ParallelDofMapping::addFeSpace()"); FUNCNAME("ParallelDofMapping::addFeSpace()");
...@@ -111,43 +158,43 @@ namespace AMDiS { ...@@ -111,43 +158,43 @@ namespace AMDiS {
if (data.count(feSpace)) if (data.count(feSpace))
data.find(feSpace)->second.clear(); data.find(feSpace)->second.clear();
else else
data.insert(make_pair(feSpace, GlobalDofMap(mpiComm))); data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
data.find(feSpace)->second.setFeSpace(feSpace); data.find(feSpace)->second.setFeSpace(feSpace);
} }
int ParallelDofMapping::getRankDofs(vector<const FiniteElemSpace*> &fe) int ParallelDofMapping::computeRankDofs()
{ {
FUNCNAME("ParallelDofMapping::getRankDofs()"); FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < fe.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(fe[i]))("Cannot find FE space: %p\n", fe[i]); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[fe[i]].nRankDofs; result += data[feSpaces[i]].nRankDofs;
} }
return result; return result;
} }
int ParallelDofMapping::getLocalDofs(vector<const FiniteElemSpace*> &fe) int ParallelDofMapping::computeLocalDofs()
{ {
FUNCNAME("ParallelDofMapping::getLocalDofs()"); FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < fe.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(fe[i]))("Cannot find FE space: %p\n", fe[i]); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[fe[i]].nLocalDofs; result += data[feSpaces[i]].nLocalDofs;
} }
return result; return result;
} }
int ParallelDofMapping::getOverallDofs(vector<const FiniteElemSpace*> &feSpaces) int ParallelDofMapping::computeOverallDofs()
{ {
FUNCNAME("ParallelDofMapping::getOverallDofs()"); FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
...@@ -159,9 +206,9 @@ namespace AMDiS { ...@@ -159,9 +206,9 @@ namespace AMDiS {
} }
int ParallelDofMapping::getStartDofs(vector<const FiniteElemSpace*> &feSpaces) int ParallelDofMapping::computeStartDofs()
{ {
FUNCNAME("ParallelDofMapping::getStartDofs()"); FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
...@@ -173,39 +220,22 @@ namespace AMDiS { ...@@ -173,39 +220,22 @@ namespace AMDiS {
} }
void ParallelDofMapping::init(MPI::Intracomm *m,
vector<const FiniteElemSpace*> &fe,
bool needGlobalMapping,
bool bNonLocalDofs)
{
FUNCNAME("ParallelDofMapping::init()");
mpiComm = m;
feSpaces = fe;
hasNonLocalDofs = bNonLocalDofs;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
feSpacesUnique.insert(feSpaces[i]);
addFeSpace(feSpaces[i]);
data[feSpaces[i]].setNeedGlobalMapping(needGlobalMapping);
data[feSpaces[i]].setNonLocalDofs(hasNonLocalDofs);
}
}
void ParallelDofMapping::update() void ParallelDofMapping::update()
{ {
FUNCNAME("ParallelDofMapping::update()"); FUNCNAME("ParallelDofMapping::update()");
// First, update all FE space DOF mappings.
for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin(); for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].update(); data[*it].update();
nRankDofs = getRankDofs(feSpaces); // Compute all numbers from this mappings.
nLocalDofs = getLocalDofs(feSpaces); nRankDofs = computeRankDofs();
nOverallDofs = getOverallDofs(feSpaces); nLocalDofs = computeLocalDofs();
rStartDofs = getStartDofs(feSpaces); nOverallDofs = computeOverallDofs();
rStartDofs = computeStartDofs();
// And finally, compute the matrix indices.
computeMatIndex(); computeMatIndex();
} }
...@@ -216,10 +246,18 @@ namespace AMDiS { ...@@ -216,10 +246,18 @@ namespace AMDiS {
dofToMatIndex.clear(); dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank.
int offset = rStartDofs; int offset = rStartDofs;
// === Create the matrix indices for all component FE spaces. ===
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
// Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index.
map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap(); map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap();
typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType; typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType;
for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) { for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) {
...@@ -229,14 +267,20 @@ namespace AMDiS { ...@@ -229,14 +267,20 @@ namespace AMDiS {
} }
} }
// Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs; offset += data[feSpaces[i]].nRankDofs;
// If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs) if (!hasNonLocalDofs)
continue; continue;
TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL) TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
("No communicator given!\n"); ("No communicator given!\n");
// === Communicate the matrix indices for all DOFs that are on some ===
// === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm); StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]); for (DofComm::Iterator it(*sendDofs, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
...@@ -270,17 +314,4 @@ namespace AMDiS { ...@@ -270,17 +314,4 @@ namespace AMDiS {
} }
} }
void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
{
FUNCNAME("ParallelDofMapping::setDofComm()");
sendDofs = &pSend;
recvDofs = &pRecv;
for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it)
data[*it].setDofComm(pSend, pRecv);
}
} }
...@@ -34,17 +34,18 @@ ...@@ -34,17 +34,18 @@
namespace AMDiS { namespace AMDiS {
using namespace std; using namespace std;
/// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
/// and a global one.
struct MultiIndex struct MultiIndex
{ {
int local, global; int local, global;
}; };
/** \brief /** \brief
* Defines for each system component a mapping for sets of global DOF indices * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
* to global matrix indices. The mapping is defined for all DOFs in rank's * class defines a mapping from component number and DOF index to a global
* subdomain. When periodic boundary conditions are used, then the mapping * matrix index. This class does not calculate the indices by itself!
* stores also information for the periodic associations of rank's DOF on
* periodic boundaries.
*/ */
class DofToMatIndex class DofToMatIndex
{ {
...@@ -57,13 +58,19 @@ namespace AMDiS { ...@@ -57,13 +58,19 @@ namespace AMDiS {
data.clear(); data.clear();
} }
/// Add a new mapping. /** Add a new mapping for a given DOF.
*
* \param[in] component Component number for which the mapping
* is defined.
* \param[in] dof DOF index
* \param[in] globalMatIndex Global matrix index.
*/
inline void add(int component, DegreeOfFreedom dof, int globalMatIndex) inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
{ {
data[component][dof] = globalMatIndex; data[component][dof] = globalMatIndex;
} }
/// Map a global DOF index to the global matrix index for a specific /// Maps a global DOF index to the global matrix index for a specific
/// system component number. /// system component number.
inline int get(int component, DegreeOfFreedom dof) inline int get(int component, DegreeOfFreedom dof)
{ {
...@@ -86,31 +93,44 @@ namespace AMDiS { ...@@ -86,31 +93,44 @@ namespace AMDiS {
}; };
class GlobalDofMap /**
* This class defines the parallel mapping of DOFs for one FE space. It is used
* by the class \ref ParallelDofMapping to specifiy the mapping for a set of
* FE spaces.
*/
class FeSpaceDofMap
{ {
public: public:
/// This constructor exists only to create std::map of this class and make /// This constructor exists only to create std::map of this class and make
/// use of the operator [] for read access. Should never be called. /// use of the operator [] for read access. Should never be called.
GlobalDofMap() FeSpaceDofMap()
{ {
ERROR_EXIT("Should not be called!\n"); ERROR_EXIT("Should not be called!\n");
} }
GlobalDofMap(MPI::Intracomm* m) /// This is the only valid constructur to be used.
FeSpaceDofMap(MPI::Intracomm* m)
: mpiComm(m), : mpiComm(m),
sendDofs(NULL), sendDofs(NULL),
recvDofs(NULL), recvDofs(NULL),
feSpace(NULL), feSpace(NULL),
needGlobalMapping(false), needGlobalMapping(false),
hasNonLocalDofs(false),
nRankDofs(0), nRankDofs(0),
nLocalDofs(0), nLocalDofs(0),
nOverallDofs(0), nOverallDofs(0),
rStartDofs(0), rStartDofs(0)
hasNonLocalDofs(false) {
{} FUNCNAME("FeSpaceDofMap::FeSpaceDofMap()");
TEST_EXIT(mpiComm)("No MPI Communicator specified!\n");
}
/// Clears all data of the mapping.
void clear(); void clear();
/// Maps a DOF index to both, the local and global index of the mapping. The
/// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d) MultiIndex& operator[](DegreeOfFreedom d)
{ {
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n"); TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");
...@@ -118,9 +138,11 @@ namespace AMDiS { ...@@ -118,9 +138,11 @@ namespace AMDiS {
return dofMap[d]; return dofMap[d];
} }
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank.
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1) void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{ {
FUNCNAME("GlobalDofMap::insertRankDof()"); FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
...@@ -130,9 +152,11 @@ namespace AMDiS { ...@@ -130,9 +152,11 @@ namespace AMDiS {
nRankDofs++; nRankDofs++;
} }
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary.
void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1) void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{ {
FUNCNAME("GlobalDofMap::insert()"); FUNCNAME("FeSpaceDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
...@@ -141,49 +165,55 @@ namespace AMDiS { ...@@ -141,49 +165,55 @@ namespace AMDiS {
nonRankDofs.insert(dof0); nonRankDofs.insert(dof0);
} }
/// Checks if a given DOF is in the DOF mapping.
bool isSet(DegreeOfFreedom dof) bool isSet(DegreeOfFreedom dof)
{ {
return static_cast<bool>(dofMap.count(dof)); return static_cast<bool>(dofMap.count(dof));
} }
/// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
/// a DOF of the mapping (this is not checked here), otherwise the result is