Commit 3e2d84dc authored by Thomas Witkowski's avatar Thomas Witkowski

Setup for multilevel parallel dof mapping.

parent a6892347
...@@ -1200,7 +1200,7 @@ namespace AMDiS { ...@@ -1200,7 +1200,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. === // === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap())); partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
bool partitioningSucceed = bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART); partitioner->partition(elemWeights, ADAPTIVE_REPART);
...@@ -1979,9 +1979,9 @@ namespace AMDiS { ...@@ -1979,9 +1979,9 @@ namespace AMDiS {
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("FE space %d:\n", i); MSG("FE space %d:\n", i);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs); MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs); MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs); MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
} }
stringstream oss; stringstream oss;
...@@ -2259,20 +2259,6 @@ namespace AMDiS { ...@@ -2259,20 +2259,6 @@ namespace AMDiS {
} }
DegreeOfFreedom MeshDistributor::mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
FUNCNAME("MeshDistributor::mapGlobalToLocal()");
for (DofMap::iterator it = dofMap[feSpace].getMap().begin();
it != dofMap[feSpace].getMap().end(); ++it)
if (it->second.global == dof)
return it->first;
return -1;
}
void MeshDistributor::serialize(ostream &out) void MeshDistributor::serialize(ostream &out)
{ {
FUNCNAME("MeshDistributor::serialize()"); FUNCNAME("MeshDistributor::serialize()");
......
...@@ -157,13 +157,6 @@ namespace AMDiS { ...@@ -157,13 +157,6 @@ namespace AMDiS {
return periodicMap; return periodicMap;
} }
/// Returns for a global index the DOF index in rank's subdomain. As there
/// is no direct data structure that stores this information, we have to
/// search for it in \ref dofMap. This is not very efficient and this
/// function should thus be used for debugging only.
DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof);
DofComm& getSendDofs() DofComm& getSendDofs()
{ {
return sendDofs; return sendDofs;
......
...@@ -634,7 +634,12 @@ namespace AMDiS { ...@@ -634,7 +634,12 @@ namespace AMDiS {
void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank) void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank)
{ {
FUNCNAME("ParallelDebug::printMapLocalGlobal()");
ERROR_EXIT("Rewrite this function!\n");
#if 0
if (rank == -1 || pdb.mpiRank == rank) { if (rank == -1 || pdb.mpiRank == rank) {
const FiniteElemSpace *feSpace = pdb.feSpaces[0]; const FiniteElemSpace *feSpace = pdb.feSpaces[0];
...@@ -662,6 +667,7 @@ namespace AMDiS { ...@@ -662,6 +667,7 @@ namespace AMDiS {
cout << "------" << endl; cout << "------" << endl;
} }
} }
#endif
} }
......
...@@ -20,47 +20,61 @@ namespace AMDiS { ...@@ -20,47 +20,61 @@ namespace AMDiS {
void FeSpaceDofMap::clear() void FeSpaceDofMap::clear()
{ {
dofMap.clear(); dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear(); nonRankDofs.clear();
nRankDofs = 0; nonRankDofs.resize(nLevel);
nLocalDofs = 0;
nOverallDofs = 0; for (int i = 0; i < nLevel; i++) {
rStartDofs = 0; nRankDofs[i] = 0;
nLocalDofs[i] = 0;
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
}
} }
void FeSpaceDofMap::update() void FeSpaceDofMap::update()
{ {
// === Compute local indices for all rank owned DOFs. === FUNCNAME("FeSpaceDofMap::update()");
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (int i = 0; i < nLevel; i++) {
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0;
rStartDofs = 0;
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
// === If required, compute also the global indices. ===
if (needGlobalMapping) { // === Compute local indices for all rank owned DOFs. ===
computeGlobalMapping();
if (hasNonLocalDofs) for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
computeNonLocalIndices(); if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
it->second.local = nRankDofs[i]++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
if (needGlobalMapping) {
computeGlobalMapping(i);
if (hasNonLocalDofs)
computeNonLocalIndices(i);
}
} }
} }
void FeSpaceDofMap::computeGlobalMapping() void FeSpaceDofMap::computeGlobalMapping(int level)
{ {
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
it->second.global = it->second.local + rStartDofs;
for (DofMap::iterator it = dofMap[level].begin();
it != dofMap[level].end(); ++it)
it->second.global = it->second.local + rStartDofs[level];
} }
void FeSpaceDofMap::computeNonLocalIndices() void FeSpaceDofMap::computeNonLocalIndices(int level)
{ {
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()"); FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
...@@ -70,20 +84,22 @@ namespace AMDiS { ...@@ -70,20 +84,22 @@ namespace AMDiS {
// === other ranks that also include this DOF. === // === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(mpiComm); StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(*sendDofs, level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex())) if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global); stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global);
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
// === Check from which ranks this rank must receive some data. === // === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) { for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false; bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs.count(it.getDofIndex())) { if (nonRankDofs[level].count(it.getDofIndex())) {
recvFromRank = true; recvFromRank = true;
break; break;
} }
...@@ -101,12 +117,12 @@ namespace AMDiS { ...@@ -101,12 +117,12 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. === // === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(*recvDofs, feSpace); for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int i = 0; int i = 0;
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs.count(it.getDofIndex())) if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++]; dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
} }
} }
...@@ -144,10 +160,12 @@ namespace AMDiS { ...@@ -144,10 +160,12 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].clear(); data[*it].clear();
nRankDofs = -1; for (int i = 0; i < nLevel; i++) {
nLocalDofs = -1; nRankDofs[i] = -1;
nOverallDofs = -1; nLocalDofs[i] = -1;
rStartDofs = -1; nOverallDofs[i] = -1;
rStartDofs[i] = -1;
}
dofToMatIndex.clear(); dofToMatIndex.clear();
} }
...@@ -180,56 +198,56 @@ namespace AMDiS { ...@@ -180,56 +198,56 @@ namespace AMDiS {
} }
int ParallelDofMapping::computeRankDofs() int ParallelDofMapping::computeRankDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeRankDofs()"); FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs; result += data[feSpaces[i]].nRankDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeLocalDofs() int ParallelDofMapping::computeLocalDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeLocalDofs()"); FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs; result += data[feSpaces[i]].nLocalDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeOverallDofs() int ParallelDofMapping::computeOverallDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeOverallDofs()"); FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs; result += data.find(feSpaces[i])->second.nOverallDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeStartDofs() int ParallelDofMapping::computeStartDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeStartDofs()"); FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs; result += data.find(feSpaces[i])->second.rStartDofs[level];
} }
return result; return result;
...@@ -245,15 +263,18 @@ namespace AMDiS { ...@@ -245,15 +263,18 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].update(); data[*it].update();
// Compute all numbers from this mappings. for (int i = 0; i < nLevel; i++) {
nRankDofs = computeRankDofs(); // Compute all numbers from this mappings.
nLocalDofs = computeLocalDofs();
nOverallDofs = computeOverallDofs(); nRankDofs[i] = computeRankDofs(i);
rStartDofs = computeStartDofs(); nLocalDofs[i] = computeLocalDofs(i);
nOverallDofs[i] = computeOverallDofs(i);
// And finally, compute the matrix indices. rStartDofs[i] = computeStartDofs(i);
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal); // And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal, i);
}
} }
...@@ -272,16 +293,18 @@ namespace AMDiS { ...@@ -272,16 +293,18 @@ namespace AMDiS {
} }
void ParallelDofMapping::computeMatIndex(bool globalIndex) void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
{ {
FUNCNAME("ParallelDofMapping::computeMatIndex()"); FUNCNAME("ParallelDofMapping::computeMatIndex()");
TEST_EXIT(level == 0)("Not yet implemented for non-zero level!\n");
dofToMatIndex.clear(); dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the // The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is // DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank. // owned by the rank.
int offset = rStartDofs; int offset = rStartDofs[level];
// === Create the matrix indices for all component FE spaces. === // === Create the matrix indices for all component FE spaces. ===
...@@ -290,7 +313,7 @@ namespace AMDiS { ...@@ -290,7 +313,7 @@ namespace AMDiS {
// Traverse all DOFs of the FE space and create for all rank owned DOFs // Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index. // a matrix index.
DofMap& dofMap = data[feSpaces[i]].getMap(); DofMap& dofMap = data[feSpaces[i]].getMap(level);
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) { for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first)) { if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex = it->second.local + offset; int globalMatIndex = it->second.local + offset;
...@@ -303,7 +326,7 @@ namespace AMDiS { ...@@ -303,7 +326,7 @@ namespace AMDiS {
// Increase the offset for the next FE space by the number of DOFs owned // Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space. // by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs; offset += data[feSpaces[i]].nRankDofs[level];
// If there are no non local DOFs, continue with the next FE space. // If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs) if (!hasNonLocalDofs)
...@@ -316,7 +339,7 @@ namespace AMDiS { ...@@ -316,7 +339,7 @@ namespace AMDiS {
// === interior boundaries. === // === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm); StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]); for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs; vector<DegreeOfFreedom> sendGlobalDofs;
...@@ -330,14 +353,14 @@ namespace AMDiS { ...@@ -330,14 +353,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), sendGlobalDofs); stdMpi.send(it.getRank(), sendGlobalDofs);
} }
for (DofComm::Iterator it(*recvDofs, feSpaces[i]); for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) !it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
{ {
for (DofComm::Iterator it(*recvDofs, feSpaces[i]); for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
......
...@@ -107,13 +107,17 @@ namespace AMDiS { ...@@ -107,13 +107,17 @@ namespace AMDiS {
sendDofs(NULL), sendDofs(NULL),
recvDofs(NULL), recvDofs(NULL),
feSpace(NULL), feSpace(NULL),
nLevel(1),
dofMap(1),
needGlobalMapping(false), needGlobalMapping(false),
hasNonLocalDofs(false), hasNonLocalDofs(false),
nRankDofs(0), nRankDofs(1),
nLocalDofs(0), nLocalDofs(1),
nOverallDofs(0), nOverallDofs(1),
rStartDofs(0) rStartDofs(1)
{} {
clear();
}
/// Clears all data of the mapping. /// Clears all data of the mapping.
void clear(); void clear();
...@@ -122,9 +126,9 @@ namespace AMDiS { ...@@ -122,9 +126,9 @@ namespace AMDiS {
/// global index must not be set. /// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d) MultiIndex& operator[](DegreeOfFreedom d)
{ {
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
return dofMap[d]; return dofMap[0][d];
} }
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
...@@ -133,12 +137,12 @@ namespace AMDiS { ...@@ -133,12 +137,12 @@ namespace AMDiS {
{ {
FUNCNAME("FeSpaceDofMap::insertRankDof()"); FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1; dofMap[0][dof0].local = dof1;
nLocalDofs++; nLocalDofs[0]++;
if (dof1 != -1) if (dof1 != -1)
nRankDofs++; nRankDofs[0]++;
} }
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
...@@ -147,17 +151,17 @@ namespace AMDiS { ...@@ -147,17 +151,17 @@ namespace AMDiS {
{ {
FUNCNAME("FeSpaceDofMap::insert()"); FUNCNAME("FeSpaceDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1; dofMap[0][dof0].local = dof1;
nLocalDofs++; nLocalDofs[0]++;
nonRankDofs.insert(dof0); nonRankDofs[0].insert(dof0);
} }
/// Checks if a given DOF is in the DOF mapping. /// Checks if a given DOF is in the DOF mapping.
bool isSet(DegreeOfFreedom dof) bool isSet(DegreeOfFreedom dof, int level = 0)
{ {
return static_cast<bool>(dofMap.count(dof)); return static_cast<bool>(dofMap[level].count(dof));
} }
/// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
...@@ -165,19 +169,19 @@ namespace AMDiS { ...@@ -165,19 +169,19 @@ namespace AMDiS {
/// meaningsless. /// meaningsless.
bool isRankDof(DegreeOfFreedom dof) bool isRankDof(DegreeOfFreedom dof)