Commit 346f9163 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Checkpoint for code refactoring in parallel multilevel methods.

parent 6f8a7433
...@@ -46,10 +46,7 @@ namespace AMDiS { ...@@ -46,10 +46,7 @@ namespace AMDiS {
{ {
FUNCNAME("DofComm::create()"); FUNCNAME("DofComm::create()");
MSG("---> senddofs\n");
createContainer(boundary.getOwn(), sendDofs); createContainer(boundary.getOwn(), sendDofs);
MSG("---> recvdofs\n");
createContainer(boundary.getOther(), recvDofs); createContainer(boundary.getOther(), recvDofs);
} }
......
...@@ -1536,7 +1536,6 @@ namespace AMDiS { ...@@ -1536,7 +1536,6 @@ namespace AMDiS {
dofComm.create(intBoundary); dofComm.create(intBoundary);
if (levelData.getLevelNumber() > 1) { if (levelData.getLevelNumber() > 1) {
MSG("START CREATE DOF-COMM-SD\n");
dofCommSd.init(0, levelData, feSpaces); dofCommSd.init(0, levelData, feSpaces);
dofCommSd.create(intBoundarySd); dofCommSd.create(intBoundarySd);
} }
...@@ -1609,6 +1608,8 @@ namespace AMDiS { ...@@ -1609,6 +1608,8 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()"); FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()");
mesh->dofCompress();
#if (DEBUG != 0) #if (DEBUG != 0)
debug::ElementIdxToDofs elMap; debug::ElementIdxToDofs elMap;
debug::createSortedDofs(mesh, elMap); debug::createSortedDofs(mesh, elMap);
...@@ -1618,17 +1619,37 @@ namespace AMDiS { ...@@ -1618,17 +1619,37 @@ namespace AMDiS {
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
dofMap.init(levelData, feSpaces, feSpaces, true, true); dofMap.init(levelData, feSpaces, feSpaces, true, true);
dofMap.setMpiComm(levelData.getMpiComm(0), 0);
dofMap.setDofComm(dofComm); dofMap.setDofComm(dofComm);
dofMap.clear(); dofMap.clear();
if (nLevels > 1) {
dofMapSd.init(levelData, feSpaces, feSpaces, true, true);
dofMapSd.setMpiComm(levelData.getMpiComm(1), 1);
dofMapSd.setDofComm(dofCommSd);
dofMapSd.clear();
}
createBoundaryDofs(); createBoundaryDofs();
for (unsigned int i = 0; i < feSpaces.size(); i++) for (unsigned int i = 0; i < feSpaces.size(); i++)
updateLocalGlobalNumbering(feSpaces[i]); updateLocalGlobalNumbering(dofMap, dofComm, feSpaces[i]);
dofMap.update(); dofMap.update();
if (nLevels > 1) {
for (unsigned int i = 0; i < feSpaces.size(); i++)
updateLocalGlobalNumbering(dofMapSd, dofCommSd, feSpaces[i]);
dofMapSd.update();
}
// === Update DOF admins due to new number of DOFs. ===
lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this);
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
MSG("| number of levels: %d\n", nLevels); MSG("| number of levels: %d\n", nLevels);
MSG("| number of FE spaces: %d\n", feSpaces.size()); MSG("| number of FE spaces: %d\n", feSpaces.size());
...@@ -1642,9 +1663,19 @@ namespace AMDiS { ...@@ -1642,9 +1663,19 @@ namespace AMDiS {
} }
} }
stringstream oss; if (nLevels > 1) {
oss << debugOutputDir << "elementIndex-" << mpiRank << ".vtu"; for (int level = 0; level < nLevels; level++) {
debug::writeElementIndexMesh(mesh, oss.str()); for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i);
MSG("| nRankDofs = %d\n", dofMapSd[feSpaces[i]].nRankDofs[level]);
MSG("| nOverallDofs = %d\n", dofMapSd[feSpaces[i]].nOverallDofs[level]);
MSG("| rStartDofs = %d\n", dofMapSd[feSpaces[i]].rStartDofs[level]);
}
}
}
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex-" +
lexical_cast<string>(mpiRank) + ".vtu");
ParallelDebug::writeDebugFile(*this, debugOutputDir + "mpi-dbg", "dat"); ParallelDebug::writeDebugFile(*this, debugOutputDir + "mpi-dbg", "dat");
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
...@@ -1658,12 +1689,12 @@ namespace AMDiS { ...@@ -1658,12 +1689,12 @@ namespace AMDiS {
} }
void MeshDistributor::updateLocalGlobalNumbering(const FiniteElemSpace *feSpace) void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dmap,
DofComm &dcom,
const FiniteElemSpace *feSpace)
{ {
FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()"); FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()");
mesh->dofCompress();
// === Get all DOFs in ranks partition. === // === Get all DOFs in ranks partition. ===
std::set<const DegreeOfFreedom*> rankDofSet; std::set<const DegreeOfFreedom*> rankDofSet;
...@@ -1677,29 +1708,19 @@ namespace AMDiS { ...@@ -1677,29 +1708,19 @@ namespace AMDiS {
int nLevels = levelData.getLevelNumber(); int nLevels = levelData.getLevelNumber();
for (int level = 0; level < nLevels; level++) { for (int level = 0; level < nLevels; level++) {
DofContainerSet nonRankDofs; DofContainerSet nonRankDofs;
for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace); for (DofComm::Iterator it(dcom.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank()) !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof()); nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++) for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0) if (nonRankDofs.count(rankDofs[i]) == 0)
dofMap[feSpace].insertRankDof(level, *(rankDofs[i])); dmap[feSpace].insertRankDof(level, *(rankDofs[i]));
for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace); for (DofContainerSet::iterator it = nonRankDofs.begin();
!it.end(); it.nextRank()) it != nonRankDofs.end(); ++it)
for (; !it.endDofIter(); it.nextDof()) dmap[feSpace].insertNonRankDof(level, **it);
dofMap[feSpace].insertNonRankDof(level, it.getDofIndex());
} }
// === Update DOF admins due to new number of DOFs. ===
lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this);
#endif
} }
......
...@@ -309,7 +309,9 @@ namespace AMDiS { ...@@ -309,7 +309,9 @@ namespace AMDiS {
/// Updates the local and global DOF numbering after the mesh has been /// Updates the local and global DOF numbering after the mesh has been
/// changed. /// changed.
void updateLocalGlobalNumbering(const FiniteElemSpace *feSpace); void updateLocalGlobalNumbering(ParallelDofMapping &dmap,
DofComm &dcom,
const FiniteElemSpace *feSpace);
/// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are /// Calls \ref createPeriodicMap(feSpace) for all FE spaces that are
/// handled by the mesh distributor. /// handled by the mesh distributor.
...@@ -473,6 +475,8 @@ namespace AMDiS { ...@@ -473,6 +475,8 @@ namespace AMDiS {
/// Mapping object to map from local DOF indices to global ones. /// Mapping object to map from local DOF indices to global ones.
ParallelDofMapping dofMap; ParallelDofMapping dofMap;
ParallelDofMapping dofMapSd;
/// Database to store and query all sub-objects of all elements of the /// Database to store and query all sub-objects of all elements of the
/// macro mesh. /// macro mesh.
ElementObjectDatabase elObjDb; ElementObjectDatabase elObjDb;
......
...@@ -55,7 +55,7 @@ namespace AMDiS { ...@@ -55,7 +55,7 @@ namespace AMDiS {
nOverallDofs[i] = 0; nOverallDofs[i] = 0;
rStartDofs[i] = 0; rStartDofs[i] = 0;
mpi::getDofNumbering(levelData->getMpiComm(0), mpi::getDofNumbering(mpiComm,
nRankDofs[i], rStartDofs[i], nOverallDofs[i]); nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. === // === If required, compute also the global indices. ===
...@@ -89,13 +89,20 @@ namespace AMDiS { ...@@ -89,13 +89,20 @@ namespace AMDiS {
// === Send all global indices of DOFs that are owned by the rank to all === // === Send all global indices of DOFs that are owned by the rank to all ===
// === other ranks that also include this DOF. === // === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(levelData->getMpiComm(0)); StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpace); for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpace);
!it.end(); it.nextRank()) !it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex())) if (dofMap[level].count(it.getDofIndex()) &&
stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global); !nonRankDofs[level].count(it.getDofIndex()))
stdMpi.getSendData(rank).
push_back(dofMap[level][it.getDofIndex()].global);
}
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
...@@ -112,8 +119,13 @@ namespace AMDiS { ...@@ -112,8 +119,13 @@ namespace AMDiS {
} }
} }
if (recvFromRank) if (recvFromRank) {
stdMpi.recv(it.getRank()); int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
stdMpi.recv(rank);
}
} }
...@@ -126,10 +138,14 @@ namespace AMDiS { ...@@ -126,10 +138,14 @@ namespace AMDiS {
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace); for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
int i = 0; int i = 0;
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs[level].count(it.getDofIndex())) if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++]; dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
} }
} }
...@@ -185,6 +201,17 @@ namespace AMDiS { ...@@ -185,6 +201,17 @@ namespace AMDiS {
} }
} }
void ParallelDofMapping::setMpiComm(MPI::Intracomm &m, int l)
{
mpiComm = m;
meshLevel = l;
for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it)
data[*it].setMpiComm(m, l);
}
void ParallelDofMapping::setDofComm(DofComm &dc) void ParallelDofMapping::setDofComm(DofComm &dc)
{ {
...@@ -350,7 +377,7 @@ namespace AMDiS { ...@@ -350,7 +377,7 @@ namespace AMDiS {
// === Communicate the matrix indices for all DOFs that are on some === // === Communicate the matrix indices for all DOFs that are on some ===
// === interior boundaries. === // === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(levelData->getMpiComm(0)); StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpaces[i]); for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs; vector<DegreeOfFreedom> sendGlobalDofs;
...@@ -362,22 +389,35 @@ namespace AMDiS { ...@@ -362,22 +389,35 @@ namespace AMDiS {
else else
sendGlobalDofs.push_back(dofToMatIndex[level].get(i, it.getDofIndex())); sendGlobalDofs.push_back(dofToMatIndex[level].get(i, it.getDofIndex()));
stdMpi.send(it.getRank(), sendGlobalDofs); int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
stdMpi.send(rank, sendGlobalDofs);
} }
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]); for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]);
!it.end(); it.nextRank()) !it.end(); it.nextRank()) {
stdMpi.recv(it.getRank()); int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
stdMpi.recv(rank);
}
stdMpi.startCommunication(); stdMpi.startCommunication();
{ {
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]); for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]);
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
int counter = 0; int counter = 0;
for (; !it.endDofIter(); it.nextDof()) { for (; !it.endDofIter(); it.nextDof()) {
if (dofMap.count(it.getDofIndex())) { if (dofMap.count(it.getDofIndex())) {
DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++]; DegreeOfFreedom d = stdMpi.getRecvData(rank)[counter++];
if (globalIndex) if (globalIndex)
dofToMatIndex[level].add(i, dofMap[it.getDofIndex()].global, d); dofToMatIndex[level].add(i, dofMap[it.getDofIndex()].global, d);
else else
......
...@@ -216,6 +216,12 @@ namespace AMDiS { ...@@ -216,6 +216,12 @@ namespace AMDiS {
dofComm = &dc; dofComm = &dc;
} }
void setMpiComm(MPI::Intracomm &m, int l)
{
mpiComm = m;
meshLevel = l;
}
private: private:
/// Computes a global mapping from the local one. /// Computes a global mapping from the local one.
void computeGlobalMapping(int level); void computeGlobalMapping(int level);
...@@ -230,6 +236,10 @@ namespace AMDiS { ...@@ -230,6 +236,10 @@ namespace AMDiS {
/// DOF communicator for all DOFs on interior boundaries. /// DOF communicator for all DOFs on interior boundaries.
DofComm *dofComm; DofComm *dofComm;
MPI::Intracomm mpiComm;
int meshLevel;
/// The FE space this mapping belongs to. This is used only the get the /// The FE space this mapping belongs to. This is used only the get the
/// correct DOF communicator in \ref dofComm. /// correct DOF communicator in \ref dofComm.
const FiniteElemSpace *feSpace; const FiniteElemSpace *feSpace;
...@@ -297,6 +307,8 @@ namespace AMDiS { ...@@ -297,6 +307,8 @@ namespace AMDiS {
vector<const FiniteElemSpace*> &uniqueFe, vector<const FiniteElemSpace*> &uniqueFe,
bool needGlobalMapping, bool needGlobalMapping,
bool bNonLocalDofs); bool bNonLocalDofs);
void setMpiComm(MPI::Intracomm &m, int l);
/// Clear all data. /// Clear all data.
void clear(); void clear();
...@@ -412,6 +424,10 @@ namespace AMDiS { ...@@ -412,6 +424,10 @@ namespace AMDiS {
int computeStartDofs(int level); int computeStartDofs(int level);
private: private:
MPI::Intracomm mpiComm;
int meshLevel;
MeshLevelData *levelData; MeshLevelData *levelData;
/// DOF communicator for all DOFs on interior boundaries. /// DOF communicator for all DOFs on interior boundaries.
......
...@@ -363,7 +363,7 @@ namespace AMDiS { ...@@ -363,7 +363,7 @@ namespace AMDiS {
// === create local indices of the primals starting at zero. === // === create local indices of the primals starting at zero. ===
for (DofIndexSet::iterator it = primals.begin(); it != primals.end(); ++it) for (DofIndexSet::iterator it = primals.begin(); it != primals.end(); ++it)
if (meshDistributor->getDofMap()[feSpace].isRankDof(*it, meshLevel)) if (meshDistributor->getDofMap()[feSpace].isRankDof(*it, 0))
primalDofMap[feSpace].insertRankDof(meshLevel, *it); primalDofMap[feSpace].insertRankDof(meshLevel, *it);
else else
primalDofMap[feSpace].insertNonRankDof(meshLevel, *it); primalDofMap[feSpace].insertNonRankDof(meshLevel, *it);
...@@ -453,7 +453,7 @@ namespace AMDiS { ...@@ -453,7 +453,7 @@ namespace AMDiS {
DofMap& dualMap = dualDofMap[feSpace].getMap(meshLevel); DofMap& dualMap = dualDofMap[feSpace].getMap(meshLevel);
for (DofMap::iterator it = dualMap.begin(); it != dualMap.end(); ++it) { for (DofMap::iterator it = dualMap.begin(); it != dualMap.end(); ++it) {
if (meshDistributor->getDofMap()[feSpace].isRankDof(it->first, meshLevel)) { if (meshDistributor->getDofMap()[feSpace].isRankDof(it->first, 0)) {
lagrangeMap[feSpace].insertRankDof(meshLevel, it->first, nRankLagrange); lagrangeMap[feSpace].insertRankDof(meshLevel, it->first, nRankLagrange);
int degree = boundaryDofRanks[feSpace][it->first].size(); int degree = boundaryDofRanks[feSpace][it->first].size();
nRankLagrange += (degree * (degree - 1)) / 2; nRankLagrange += (degree * (degree - 1)) / 2;
......
#define BOOST_TEST_DYN_LINK #define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE 0005 #define BOOST_TEST_MODULE 0006
#define BOOST_TEST_NO_MAIN #define BOOST_TEST_NO_MAIN
#include <boost/test/unit_test.hpp> #include <boost/test/unit_test.hpp>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment