Commit 235f1f39 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Removed level hierarchie in parallel dof mapping, is not necessary.

parent 346f9163
...@@ -238,7 +238,7 @@ namespace AMDiS { ...@@ -238,7 +238,7 @@ namespace AMDiS {
if (condition->applyBoundaryCondition()) { if (condition->applyBoundaryCondition()) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (dofMap->isRankDof(rowIndices[i], 0)) if (dofMap->isRankDof(rowIndices[i]))
applyDBCs.insert(static_cast<int>(row)); applyDBCs.insert(static_cast<int>(row));
#else #else
applyDBCs.insert(static_cast<int>(row)); applyDBCs.insert(static_cast<int>(row));
......
...@@ -248,7 +248,7 @@ namespace AMDiS { ...@@ -248,7 +248,7 @@ namespace AMDiS {
{ {
TEST_EXIT_DBG(dofMap)("No rank dofs set!\n"); TEST_EXIT_DBG(dofMap)("No rank dofs set!\n");
return dofMap->isRankDof(dof, 0); return dofMap->isRankDof(dof);
} }
#endif #endif
......
...@@ -1246,7 +1246,7 @@ namespace AMDiS { ...@@ -1246,7 +1246,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. === // === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0))); partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap()));
bool partitioningSucceed = bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART); partitioner->partition(elemWeights, ADAPTIVE_REPART);
...@@ -1654,23 +1654,19 @@ namespace AMDiS { ...@@ -1654,23 +1654,19 @@ namespace AMDiS {
MSG("| number of levels: %d\n", nLevels); MSG("| number of levels: %d\n", nLevels);
MSG("| number of FE spaces: %d\n", feSpaces.size()); MSG("| number of FE spaces: %d\n", feSpaces.size());
for (int level = 0; level < nLevels; level++) {
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i); MSG("| FE space = %d:\n", i);
MSG("| nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[level]); MSG("| nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs);
MSG("| nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[level]); MSG("| nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs);
MSG("| rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[level]); MSG("| rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs);
}
} }
if (nLevels > 1) { if (nLevels > 1) {
for (int level = 0; level < nLevels; level++) {
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i); MSG("| FE space = %d:\n", i);
MSG("| nRankDofs = %d\n", dofMapSd[feSpaces[i]].nRankDofs[level]); MSG("| nRankDofs = %d\n", dofMapSd[feSpaces[i]].nRankDofs);
MSG("| nOverallDofs = %d\n", dofMapSd[feSpaces[i]].nOverallDofs[level]); MSG("| nOverallDofs = %d\n", dofMapSd[feSpaces[i]].nOverallDofs);
MSG("| rStartDofs = %d\n", dofMapSd[feSpaces[i]].rStartDofs[level]); MSG("| rStartDofs = %d\n", dofMapSd[feSpaces[i]].rStartDofs);
}
} }
} }
...@@ -1704,23 +1700,19 @@ namespace AMDiS { ...@@ -1704,23 +1700,19 @@ namespace AMDiS {
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue); sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
// === Traverse interior boundaries and get all DOFs on them. === // === Traverse interior boundaries and get all DOFs on them. ===
int nLevels = levelData.getLevelNumber();
for (int level = 0; level < nLevels; level++) {
DofContainerSet nonRankDofs; DofContainerSet nonRankDofs;
for (DofComm::Iterator it(dcom.getRecvDofs(), level, feSpace); for (DofComm::Iterator it(dcom.getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank()) !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof()); nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++) for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0) if (nonRankDofs.count(rankDofs[i]) == 0)
dmap[feSpace].insertRankDof(level, *(rankDofs[i])); dmap[feSpace].insertRankDof(*(rankDofs[i]));
for (DofContainerSet::iterator it = nonRankDofs.begin(); for (DofContainerSet::iterator it = nonRankDofs.begin();
it != nonRankDofs.end(); ++it) it != nonRankDofs.end(); ++it)
dmap[feSpace].insertNonRankDof(level, **it); dmap[feSpace].insertNonRankDof(**it);
}
} }
...@@ -1782,8 +1774,8 @@ namespace AMDiS { ...@@ -1782,8 +1774,8 @@ namespace AMDiS {
BoundaryType type = bound.type; BoundaryType type = bound.type;
for (unsigned int j = 0; j < dofs0.size(); j++) { for (unsigned int j = 0; j < dofs0.size(); j++) {
DegreeOfFreedom globalDof0 = dofMap[feSpace][0][*(dofs0[j])].global; DegreeOfFreedom globalDof0 = dofMap[feSpace][*(dofs0[j])].global;
DegreeOfFreedom globalDof1 = dofMap[feSpace][0][*(dofs1[j])].global; DegreeOfFreedom globalDof1 = dofMap[feSpace][*(dofs1[j])].global;
if (!periodicMap.isPeriodicOnBound(feSpace, type, globalDof0)) if (!periodicMap.isPeriodicOnBound(feSpace, type, globalDof0))
periodicMap.add(feSpace, type, globalDof0, globalDof1); periodicMap.add(feSpace, type, globalDof0, globalDof1);
...@@ -1808,7 +1800,7 @@ namespace AMDiS { ...@@ -1808,7 +1800,7 @@ namespace AMDiS {
// Send the global indices to the rank on the other side. // Send the global indices to the rank on the other side.
stdMpi.getSendData(it->first).reserve(dofs.size()); stdMpi.getSendData(it->first).reserve(dofs.size());
for (unsigned int i = 0; i < dofs.size(); i++) for (unsigned int i = 0; i < dofs.size(); i++)
stdMpi.getSendData(it->first).push_back(dofMap[feSpace][0][*(dofs[i])].global); stdMpi.getSendData(it->first).push_back(dofMap[feSpace][*(dofs[i])].global);
// Receive from this rank the same number of dofs. // Receive from this rank the same number of dofs.
stdMpi.recv(it->first, dofs.size()); stdMpi.recv(it->first, dofs.size());
...@@ -1834,7 +1826,7 @@ namespace AMDiS { ...@@ -1834,7 +1826,7 @@ namespace AMDiS {
// Added the received DOFs to the mapping. // Added the received DOFs to the mapping.
for (unsigned int i = 0; i < dofs.size(); i++) { for (unsigned int i = 0; i < dofs.size(); i++) {
int globalDofIndex = dofMap[feSpace][0][*(dofs[i])].global; int globalDofIndex = dofMap[feSpace][*(dofs[i])].global;
int mapGlobalDofIndex = stdMpi.getRecvData(it->first)[i]; int mapGlobalDofIndex = stdMpi.getRecvData(it->first)[i];
BoundaryType type = types[i]; BoundaryType type = types[i];
...@@ -1867,7 +1859,7 @@ namespace AMDiS { ...@@ -1867,7 +1859,7 @@ namespace AMDiS {
boundIt->rankObj.el->getAllDofs(feSpace, boundIt->rankObj, dofs); boundIt->rankObj.el->getAllDofs(feSpace, boundIt->rankObj, dofs);
for (unsigned int i = 0; i < dofs.size(); i++) { for (unsigned int i = 0; i < dofs.size(); i++) {
DegreeOfFreedom globalDof = dofMap[feSpace][0][*dofs[i]].global; DegreeOfFreedom globalDof = dofMap[feSpace][*dofs[i]].global;
std::set<BoundaryType>& assoc = std::set<BoundaryType>& assoc =
periodicMap.getAssociations(feSpace, globalDof); periodicMap.getAssociations(feSpace, globalDof);
......
...@@ -151,6 +151,11 @@ namespace AMDiS { ...@@ -151,6 +151,11 @@ namespace AMDiS {
return dofMap; return dofMap;
} }
inline ParallelDofMapping& getDofMapSd()
{
return dofMapSd;
}
/// Returns the periodic mapping handler, \ref periodicMap. /// Returns the periodic mapping handler, \ref periodicMap.
inline PeriodicMap& getPeriodicMap() inline PeriodicMap& getPeriodicMap()
{ {
......
...@@ -480,7 +480,7 @@ namespace AMDiS { ...@@ -480,7 +480,7 @@ namespace AMDiS {
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS); DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) { for (it.reset(); !it.end(); ++it) {
coordsToIndex[(*it)] = pdb.dofMap[feSpace][0][it.getDOFIndex()].global; coordsToIndex[(*it)] = pdb.dofMap[feSpace][it.getDOFIndex()].global;
// MSG(" CHECK FOR DOF %d AT COORDS %f %f %f\n", // MSG(" CHECK FOR DOF %d AT COORDS %f %f %f\n",
// coordsToIndex[(*it)], (*it)[0], (*it)[1], (*it)[2]); // coordsToIndex[(*it)], (*it)[0], (*it)[1], (*it)[2]);
} }
...@@ -794,8 +794,8 @@ namespace AMDiS { ...@@ -794,8 +794,8 @@ namespace AMDiS {
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS); DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) { for (it.reset(); !it.end(); ++it) {
file << it.getDOFIndex() << " " file << it.getDOFIndex() << " "
<< pdb.dofMap[feSpace][0][it.getDOFIndex()].global << " " << pdb.dofMap[feSpace][it.getDOFIndex()].global << " "
<< pdb.dofMap[feSpace].isRankDof(it.getDOFIndex(), 0); << pdb.dofMap[feSpace].isRankDof(it.getDOFIndex());
for (int i = 0; i < pdb.mesh->getDim(); i++) for (int i = 0; i < pdb.mesh->getDim(); i++)
file << " " << (*it)[i]; file << " " << (*it)[i];
file << "\n"; file << "\n";
......
...@@ -19,22 +19,14 @@ namespace AMDiS { ...@@ -19,22 +19,14 @@ namespace AMDiS {
void FeSpaceDofMap::clear() void FeSpaceDofMap::clear()
{ {
int nLevel = levelData->getLevelNumber();
dofMap.clear(); dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear(); nonRankDofs.clear();
nonRankDofs.resize(nLevel);
nRankDofs.clear(); nRankDofs = 0;
nLocalDofs.clear(); nLocalDofs = 0;
nOverallDofs.clear(); nOverallDofs = 0;
rStartDofs.clear(); rStartDofs = 0;
nRankDofs.resize(nLevel, 0);
nLocalDofs.resize(nLevel, 0);
nOverallDofs.resize(nLevel, 0);
rStartDofs.resize(nLevel, 0);
} }
...@@ -42,48 +34,44 @@ namespace AMDiS { ...@@ -42,48 +34,44 @@ namespace AMDiS {
{ {
FUNCNAME("FeSpaceDofMap::update()"); FUNCNAME("FeSpaceDofMap::update()");
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// === Compute local indices for all rank owned DOFs. === // === Compute local indices for all rank owned DOFs. ===
for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it) for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0) if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs[i]++; it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. === // === Compute number of local and global DOFs in the mapping. ===
nOverallDofs[i] = 0; nOverallDofs = 0;
rStartDofs[i] = 0; rStartDofs = 0;
mpi::getDofNumbering(mpiComm, mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. === // === If required, compute also the global indices. ===
if (needGlobalMapping) { if (needGlobalMapping) {
computeGlobalMapping(i); computeGlobalMapping();
if (hasNonLocalDofs) if (hasNonLocalDofs)
computeNonLocalIndices(i); computeNonLocalIndices();
}
} }
} }
void FeSpaceDofMap::computeGlobalMapping(int level) void FeSpaceDofMap::computeGlobalMapping()
{ {
FUNCNAME("FeSpaceDofMap::computeGlobalMapping()"); FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
for (DofMap::iterator it = dofMap[level].begin(); for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
it != dofMap[level].end(); ++it) it->second.global = it->second.local + rStartDofs;
it->second.global = it->second.local + rStartDofs[level];
} }
void FeSpaceDofMap::computeNonLocalIndices(int level) void FeSpaceDofMap::computeNonLocalIndices()
{ {
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()"); FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
TEST_EXIT_DBG(dofComm)("No DOF communicator defined!\n");
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;
// === Send all global indices of DOFs that are owned by the rank to all === // === Send all global indices of DOFs that are owned by the rank to all ===
...@@ -91,17 +79,17 @@ namespace AMDiS { ...@@ -91,17 +79,17 @@ namespace AMDiS {
StdMpi<vector<int> > stdMpi(mpiComm); StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpace); for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int rank = it.getRank(); int rank = it.getRank();
if (meshLevel > 0) if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel); rank = levelData->mapRank(rank, 0, meshLevel);
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (dofMap[level].count(it.getDofIndex()) && if (dofMap.count(it.getDofIndex()) &&
!nonRankDofs[level].count(it.getDofIndex())) !nonRankDofs.count(it.getDofIndex()))
stdMpi.getSendData(rank). stdMpi.getSendData(rank).
push_back(dofMap[level][it.getDofIndex()].global); push_back(dofMap[it.getDofIndex()].global);
} }
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
...@@ -109,11 +97,11 @@ namespace AMDiS { ...@@ -109,11 +97,11 @@ namespace AMDiS {
// === Check from which ranks this rank must receive some data. === // === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace); for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
bool recvFromRank = false; bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs[level].count(it.getDofIndex())) { if (nonRankDofs.count(it.getDofIndex())) {
recvFromRank = true; recvFromRank = true;
break; break;
} }
...@@ -136,7 +124,7 @@ namespace AMDiS { ...@@ -136,7 +124,7 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. === // === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace); for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int rank = it.getRank(); int rank = it.getRank();
if (meshLevel > 0) if (meshLevel > 0)
...@@ -144,8 +132,8 @@ namespace AMDiS { ...@@ -144,8 +132,8 @@ namespace AMDiS {
int i = 0; int i = 0;
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs[level].count(it.getDofIndex())) if (nonRankDofs.count(it.getDofIndex()))
dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++]; dofMap[it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
} }
} }
...@@ -185,20 +173,11 @@ namespace AMDiS { ...@@ -185,20 +173,11 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].clear(); data[*it].clear();
int nLevel = levelData->getLevelNumber(); nRankDofs = -1;
nRankDofs.resize(nLevel); nLocalDofs = -1;
nLocalDofs.resize(nLevel); nOverallDofs = -1;
nOverallDofs.resize(nLevel); rStartDofs = -1;
rStartDofs.resize(nLevel); dofToMatIndex.clear();
dofToMatIndex.resize(nLevel);
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = -1;
nLocalDofs[i] = -1;
nOverallDofs[i] = -1;
rStartDofs[i] = -1;
dofToMatIndex[i].clear();
}
} }
...@@ -239,56 +218,56 @@ namespace AMDiS { ...@@ -239,56 +218,56 @@ namespace AMDiS {
} }
int ParallelDofMapping::computeRankDofs(int level) int ParallelDofMapping::computeRankDofs()
{ {
FUNCNAME("ParallelDofMapping::computeRankDofs()"); FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs[level]; result += data[feSpaces[i]].nRankDofs;
} }
return result; return result;
} }
int ParallelDofMapping::computeLocalDofs(int level) int ParallelDofMapping::computeLocalDofs()
{ {
FUNCNAME("ParallelDofMapping::computeLocalDofs()"); FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs[level]; result += data[feSpaces[i]].nLocalDofs;
} }
return result; return result;
} }
int ParallelDofMapping::computeOverallDofs(int level) int ParallelDofMapping::computeOverallDofs()
{ {
FUNCNAME("ParallelDofMapping::computeOverallDofs()"); FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs[level]; result += data.find(feSpaces[i])->second.nOverallDofs;
} }
return result; return result;
} }
int ParallelDofMapping::computeStartDofs(int level) int ParallelDofMapping::computeStartDofs()
{ {
FUNCNAME("ParallelDofMapping::computeStartDofs()"); FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0; int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) { for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n"); TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs[level]; result += data.find(feSpaces[i])->second.rStartDofs;
} }
return result; return result;
...@@ -304,19 +283,15 @@ namespace AMDiS { ...@@ -304,19 +283,15 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it) it != feSpacesUnique.end(); ++it)
data[*it].update(); data[*it].update();
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// Compute all numbers from this mappings. // Compute all numbers from this mappings.
nRankDofs = computeRankDofs();
nRankDofs[i] = computeRankDofs(i); nLocalDofs = computeLocalDofs();
nLocalDofs[i] = computeLocalDofs(i); nOverallDofs = computeOverallDofs();
nOverallDofs[i] = computeOverallDofs(i); rStartDofs = computeStartDofs();
rStartDofs[i] = computeStartDofs(i);
// And finally, compute the matrix indices. // And finally, compute the matrix indices.
if (needMatIndex) if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal, i); computeMatIndex(needMatIndexFromGlobal);
}
} }
...@@ -335,16 +310,16 @@ namespace AMDiS { ...@@ -335,16 +310,16 @@ namespace AMDiS {
} }