Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit 3e2d84dc authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Setup for multilevel parallel dof mapping.

parent a6892347
...@@ -1200,7 +1200,7 @@ namespace AMDiS { ...@@ -1200,7 +1200,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. === // === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap())); partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
bool partitioningSucceed = bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART); partitioner->partition(elemWeights, ADAPTIVE_REPART);
...@@ -1979,9 +1979,9 @@ namespace AMDiS { ...@@ -1979,9 +1979,9 @@ namespace AMDiS {
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("FE space %d:\n", i); MSG("FE space %d:\n", i);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs); MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs); MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs); MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
} }
stringstream oss; stringstream oss;
...@@ -2259,20 +2259,6 @@ namespace AMDiS { ...@@ -2259,20 +2259,6 @@ namespace AMDiS {
} }
DegreeOfFreedom MeshDistributor::mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
FUNCNAME("MeshDistributor::mapGlobalToLocal()");
for (DofMap::iterator it = dofMap[feSpace].getMap().begin();
it != dofMap[feSpace].getMap().end(); ++it)
if (it->second.global == dof)
return it->first;
return -1;
}
void MeshDistributor::serialize(ostream &out) void MeshDistributor::serialize(ostream &out)
{ {
FUNCNAME("MeshDistributor::serialize()"); FUNCNAME("MeshDistributor::serialize()");
......
...@@ -157,13 +157,6 @@ namespace AMDiS { ...@@ -157,13 +157,6 @@ namespace AMDiS {
return periodicMap; return periodicMap;
} }
/// Returns for a global index the DOF index in rank's subdomain. As there
/// is no direct data structure that stores this information, we have to
/// search for it in \ref dofMap. This is not very efficient and this
/// function should thus be used for debugging only.
DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof);
DofComm& getSendDofs() DofComm& getSendDofs()
{ {
return sendDofs; return sendDofs;
......
...@@ -634,7 +634,12 @@ namespace AMDiS { ...@@ -634,7 +634,12 @@ namespace AMDiS {
void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank) void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank)
{ {
FUNCNAME("ParallelDebug::printMapLocalGlobal()");
ERROR_EXIT("Rewrite this function!\n");
#if 0
if (rank == -1 || pdb.mpiRank == rank) { if (rank == -1 || pdb.mpiRank == rank) {
const FiniteElemSpace *feSpace = pdb.feSpaces[0]; const FiniteElemSpace *feSpace = pdb.feSpaces[0];
...@@ -662,6 +667,7 @@ namespace AMDiS { ...@@ -662,6 +667,7 @@ namespace AMDiS {
cout << "------" << endl; cout << "------" << endl;
} }
} }
#endif
} }
......
...@@ -20,47 +20,61 @@ namespace AMDiS { ...@@ -20,47 +20,61 @@ namespace AMDiS {
void FeSpaceDofMap::clear() void FeSpaceDofMap::clear()
{ {
dofMap.clear(); dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear(); nonRankDofs.clear();
nRankDofs = 0; nonRankDofs.resize(nLevel);
nLocalDofs = 0;
nOverallDofs = 0; for (int i = 0; i < nLevel; i++) {
rStartDofs = 0; nRankDofs[i] = 0;
nLocalDofs[i] = 0;
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
}
} }
void FeSpaceDofMap::update() void FeSpaceDofMap::update()
{ {
// === Compute local indices for all rank owned DOFs. === FUNCNAME("FeSpaceDofMap::update()");
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (int i = 0; i < nLevel; i++) {
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0;
rStartDofs = 0;
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
// === If required, compute also the global indices. ===
if (needGlobalMapping) { // === Compute local indices for all rank owned DOFs. ===
computeGlobalMapping();
if (hasNonLocalDofs) for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
computeNonLocalIndices(); if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
it->second.local = nRankDofs[i]++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
if (needGlobalMapping) {
computeGlobalMapping(i);
if (hasNonLocalDofs)
computeNonLocalIndices(i);
}
} }
} }
void FeSpaceDofMap::computeGlobalMapping() void FeSpaceDofMap::computeGlobalMapping(int level)
{ {
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
it->second.global = it->second.local + rStartDofs;
for (DofMap::iterator it = dofMap[level].begin();
it != dofMap[level].end(); ++it)
it->second.global = it->second.local + rStartDofs[level];
} }
void FeSpaceDofMap::computeNonLocalIndices() void FeSpaceDofMap::computeNonLocalIndices(int level)
{ {
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()"); FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
...@@ -70,20 +84,22 @@ namespace AMDiS { ...@@ -70,20 +84,22 @@ namespace AMDiS {
// === other ranks that also include this DOF. === // === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(mpiComm); StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(*sendDofs, level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex())) if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global); stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global);
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
// === Check from which ranks this rank must receive some data. === // === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) { for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false; bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs.count(it.getDofIndex())) { if (nonRankDofs[level].count(it.getDofIndex())) {
recvFromRank = true; recvFromRank = true;
break; break;
} }
...@@ -101,12 +117,12 @@ namespace AMDiS { ...@@ -101,12 +117,12 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. === // === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(*recvDofs, feSpace); for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int i = 0; int i = 0;
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs.count(it.getDofIndex())) if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++]; dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
} }
} }
...@@ -144,10 +160,12 @@ namespace AMDiS { ...@@ -144,10 +160,12 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].clear(); data[*it].clear();
nRankDofs = -1; for (int i = 0; i < nLevel; i++) {
nLocalDofs = -1; nRankDofs[i] = -1;
nOverallDofs = -1; nLocalDofs[i] = -1;
rStartDofs = -1; nOverallDofs[i] = -1;
rStartDofs[i] = -1;
}
dofToMatIndex.clear(); dofToMatIndex.clear();
} }
...@@ -180,56 +198,56 @@ namespace AMDiS { ...@@ -180,56 +198,56 @@ namespace AMDiS {
} }
int ParallelDofMapping::computeRankDofs() int ParallelDofMapping::computeRankDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeRankDofs()"); FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs; result += data[feSpaces[i]].nRankDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeLocalDofs() int ParallelDofMapping::computeLocalDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeLocalDofs()"); FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs; result += data[feSpaces[i]].nLocalDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeOverallDofs() int ParallelDofMapping::computeOverallDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeOverallDofs()"); FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs; result += data.find(feSpaces[i])->second.nOverallDofs[level];
} }
return result; return result;
} }
int ParallelDofMapping::computeStartDofs() int ParallelDofMapping::computeStartDofs(int level)
{ {
FUNCNAME("ParallelDofMapping::computeStartDofs()"); FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs; result += data.find(feSpaces[i])->second.rStartDofs[level];
} }
return result; return result;
...@@ -245,15 +263,18 @@ namespace AMDiS { ...@@ -245,15 +263,18 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].update(); data[*it].update();
// Compute all numbers from this mappings. for (int i = 0; i < nLevel; i++) {
nRankDofs = computeRankDofs(); // Compute all numbers from this mappings.
nLocalDofs = computeLocalDofs();
nOverallDofs = computeOverallDofs(); nRankDofs[i] = computeRankDofs(i);
rStartDofs = computeStartDofs(); nLocalDofs[i] = computeLocalDofs(i);
nOverallDofs[i] = computeOverallDofs(i);
// And finally, compute the matrix indices. rStartDofs[i] = computeStartDofs(i);
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal); // And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal, i);
}
} }
...@@ -272,16 +293,18 @@ namespace AMDiS { ...@@ -272,16 +293,18 @@ namespace AMDiS {
} }
void ParallelDofMapping::computeMatIndex(bool globalIndex) void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
{ {
FUNCNAME("ParallelDofMapping::computeMatIndex()"); FUNCNAME("ParallelDofMapping::computeMatIndex()");
TEST_EXIT(level == 0)("Not yet implemented for non-zero level!\n");
dofToMatIndex.clear(); dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the // The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is // DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank. // owned by the rank.
int offset = rStartDofs; int offset = rStartDofs[level];
// === Create the matrix indices for all component FE spaces. === // === Create the matrix indices for all component FE spaces. ===
...@@ -290,7 +313,7 @@ namespace AMDiS { ...@@ -290,7 +313,7 @@ namespace AMDiS {
// Traverse all DOFs of the FE space and create for all rank owned DOFs // Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index. // a matrix index.
DofMap& dofMap = data[feSpaces[i]].getMap(); DofMap& dofMap = data[feSpaces[i]].getMap(level);
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) { for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first)) { if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex = it->second.local + offset; int globalMatIndex = it->second.local + offset;
...@@ -303,7 +326,7 @@ namespace AMDiS { ...@@ -303,7 +326,7 @@ namespace AMDiS {
// Increase the offset for the next FE space by the number of DOFs owned // Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space. // by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs; offset += data[feSpaces[i]].nRankDofs[level];
// If there are no non local DOFs, continue with the next FE space. // If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs) if (!hasNonLocalDofs)
...@@ -316,7 +339,7 @@ namespace AMDiS { ...@@ -316,7 +339,7 @@ namespace AMDiS {
// === interior boundaries. === // === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm); StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]); for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs; vector<DegreeOfFreedom> sendGlobalDofs;
...@@ -330,14 +353,14 @@ namespace AMDiS { ...@@ -330,14 +353,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), sendGlobalDofs); stdMpi.send(it.getRank(), sendGlobalDofs);
} }
for (DofComm::Iterator it(*recvDofs, feSpaces[i]); for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) !it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
{ {
for (DofComm::Iterator it(*recvDofs, feSpaces[i]); for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
......
...@@ -107,13 +107,17 @@ namespace AMDiS { ...@@ -107,13 +107,17 @@ namespace AMDiS {
sendDofs(NULL), sendDofs(NULL),
recvDofs(NULL), recvDofs(NULL),
feSpace(NULL), feSpace(NULL),
nLevel(1),
dofMap(1),
needGlobalMapping(false), needGlobalMapping(false),
hasNonLocalDofs(false), hasNonLocalDofs(false),
nRankDofs(0), nRankDofs(1),
nLocalDofs(0), nLocalDofs(1),
nOverallDofs(0), nOverallDofs(1),
rStartDofs(0) rStartDofs(1)
{} {
clear();
}
/// Clears all data of the mapping. /// Clears all data of the mapping.
void clear(); void clear();
...@@ -122,9 +126,9 @@ namespace AMDiS { ...@@ -122,9 +126,9 @@ namespace AMDiS {
/// global index must not be set. /// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d) MultiIndex& operator[](DegreeOfFreedom d)
{ {
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
return dofMap[d]; return dofMap[0][d];
} }
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
...@@ -133,12 +137,12 @@ namespace AMDiS { ...@@ -133,12 +137,12 @@ namespace AMDiS {
{ {
FUNCNAME("FeSpaceDofMap::insertRankDof()"); FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1; dofMap[0][dof0].local = dof1;
nLocalDofs++; nLocalDofs[0]++;
if (dof1 != -1) if (dof1 != -1)
nRankDofs++; nRankDofs[0]++;
} }
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
...@@ -147,17 +151,17 @@ namespace AMDiS { ...@@ -147,17 +151,17 @@ namespace AMDiS {
{ {
FUNCNAME("FeSpaceDofMap::insert()"); FUNCNAME("FeSpaceDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1; dofMap[0][dof0].local = dof1;