Commit 37e0d38c authored by Thomas Witkowski's avatar Thomas Witkowski

And more changes on the DofCommunicator.

parent 78d5dc6b
...@@ -18,26 +18,15 @@ namespace AMDiS { ...@@ -18,26 +18,15 @@ namespace AMDiS {
using namespace std; using namespace std;
int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace) void DofComm::create(InteriorBoundary &boundary)
{ {
FUNCNAME("DofComm::getNumberDofs()"); createContainer(boundary.getOwn(), sendDofs);
createContainer(boundary.getOther(), recvDofs);
TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofs;
for (DataIter rankIt = data[level].begin();
rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace)
dofs.insert(feIt->second.begin(), feIt->second.end());
return static_cast<int>(dofs.size());
} }
void DofComm::create(RankToBoundMap &boundary) void DofComm::createContainer(RankToBoundMap &boundary,
LevelDataType &data)
{ {
// === Fill data. === // === Fill data. ===
...@@ -72,11 +61,32 @@ namespace AMDiS { ...@@ -72,11 +61,32 @@ namespace AMDiS {
} }
int DofComm::getNumberDofs(LevelDataType &data,
int level,
const FiniteElemSpace *feSpace)
{
FUNCNAME("DofComm::getNumberDofs()");
TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofs;
for (DataIter rankIt = data[level].begin();
rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace)
dofs.insert(feIt->second.begin(), feIt->second.end());
return static_cast<int>(dofs.size());
}
bool DofComm::Iterator::setNextFeMap() bool DofComm::Iterator::setNextFeMap()
{ {
FUNCNAME("DofComm::Iterator::setNextFeMap()"); FUNCNAME("DofComm::Iterator::setNextFeMap()");
if (dataIter != dofComm.data[traverseLevel].end()) { if (dataIter != data[traverseLevel].end()) {
TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n"); TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
feMapIter = dataIter->second.begin(); feMapIter = dataIter->second.begin();
......
...@@ -36,7 +36,9 @@ namespace AMDiS { ...@@ -36,7 +36,9 @@ namespace AMDiS {
{ {
public: public:
DofComm() DofComm()
: data(1) : recvDofs(1),
sendDofs(1),
periodicDofs(0)
{} {}
typedef map<const FiniteElemSpace*, DofContainer> FeMapType; typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
...@@ -46,33 +48,76 @@ namespace AMDiS { ...@@ -46,33 +48,76 @@ namespace AMDiS {
// meshLevel: map[rank -> map[feSpace -> DofContainer]] // meshLevel: map[rank -> map[feSpace -> DofContainer]]
typedef vector<DataType> LevelDataType; typedef vector<DataType> LevelDataType;
inline DofContainer& getDofContainer(int rank,
const FiniteElemSpace *feSpace,
int level = 0)
{
return data[level][rank][feSpace];
}
void init(int n, vector<const FiniteElemSpace*> &fe) void init(int n, vector<const FiniteElemSpace*> &fe)
{ {
FUNCNAME("DofComm::init()");
TEST_EXIT_DBG(n >= 1)("Should not happen!\n");
nLevel = n; nLevel = n;
feSpaces = fe; feSpaces = fe;
data.clear(); sendDofs.clear();
data.resize(nLevel); recvDofs.clear();
periodicDofs.clear();
sendDofs.resize(nLevel);
recvDofs.resize(nLevel);
periodicDofs.resize(nLevel);
}
void create(InteriorBoundary &boundary);
LevelDataType& getSendDofs()
{
return sendDofs;
}
LevelDataType& getRecvDofs()
{
return recvDofs;
} }
DataType& getData(int level = 0) LevelDataType& getPeriodicDofs()
{ {
return data[level]; return periodicDofs;
} }
int getNumberDofs(int level, const FiniteElemSpace *feSpace); // Writes all data of this object to an output stream.
void serialize(ostream &out)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
void create(RankToBoundMap &boundary); // Reads the object data from an input stream.
void deserialize(istream &in,
map<const FiniteElemSpace*, map<int, const DegreeOfFreedom*> > dofIndexMap)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
int getNumberDofs(LevelDataType &data,
int level,
const FiniteElemSpace *feSpace);
protected:
void createContainer(RankToBoundMap &boundary, LevelDataType &data);
protected: protected:
LevelDataType data; /// This map contains for each rank the list of DOFs the current rank must
/// end to exchange solution DOFs at the interior boundaries.
LevelDataType sendDofs;
/// This map contains on each rank the list of DOFs from which the current
/// rank will receive DOF values (i.e., this are all DOFs at an interior
/// boundary). The DOF indices are given in rank's local numbering.
LevelDataType recvDofs;
/// This map contains on each rank a list of DOFs along the interior bound-
/// aries to communicate with other ranks. The DOF indices are given in rank's
/// local numbering. Periodic boundaries within one subdomain are not
/// considered here.
LevelDataType periodicDofs;
int nLevel; int nLevel;
...@@ -84,9 +129,9 @@ namespace AMDiS { ...@@ -84,9 +129,9 @@ namespace AMDiS {
class Iterator class Iterator
{ {
public: public:
Iterator(DofComm &dc, Iterator(LevelDataType &d,
const FiniteElemSpace *fe = NULL) const FiniteElemSpace *fe = NULL)
: dofComm(dc), : data(d),
dofCounter(-1), dofCounter(-1),
traverseFeSpace(fe), traverseFeSpace(fe),
traverseLevel(0) traverseLevel(0)
...@@ -94,10 +139,10 @@ namespace AMDiS { ...@@ -94,10 +139,10 @@ namespace AMDiS {
goFirst(); goFirst();
} }
Iterator(DofComm &dc, Iterator(LevelDataType &d,
int level, int level,
const FiniteElemSpace *fe = NULL) const FiniteElemSpace *fe = NULL)
: dofComm(dc), : data(d),
dofCounter(-1), dofCounter(-1),
traverseFeSpace(fe), traverseFeSpace(fe),
traverseLevel(level) traverseLevel(level)
...@@ -107,7 +152,7 @@ namespace AMDiS { ...@@ -107,7 +152,7 @@ namespace AMDiS {
inline bool end() inline bool end()
{ {
return (dataIter == dofComm.data[traverseLevel].end()); return (dataIter == data[traverseLevel].end());
} }
inline void nextRank() inline void nextRank()
...@@ -200,7 +245,7 @@ namespace AMDiS { ...@@ -200,7 +245,7 @@ namespace AMDiS {
protected: protected:
void goFirst() void goFirst()
{ {
dataIter = dofComm.data[traverseLevel].begin(); dataIter = data[traverseLevel].begin();
while (setNextFeMap() == false) while (setNextFeMap() == false)
++dataIter; ++dataIter;
...@@ -209,7 +254,7 @@ namespace AMDiS { ...@@ -209,7 +254,7 @@ namespace AMDiS {
bool setNextFeMap(); bool setNextFeMap();
protected: protected:
DofComm &dofComm; LevelDataType &data;
DofComm::DataIter dataIter; DofComm::DataIter dataIter;
......
...@@ -511,7 +511,8 @@ namespace AMDiS { ...@@ -511,7 +511,8 @@ namespace AMDiS {
StdMpi<vector<double> > stdMpi(mpiComm); StdMpi<vector<double> > stdMpi(mpiComm);
for (DofComm::Iterator it(sendDofs); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getSendDofs());
!it.end(); it.nextRank()) {
vector<double> dofs; vector<double> dofs;
for (int i = 0; i < vec.getSize(); i++) { for (int i = 0; i < vec.getSize(); i++) {
...@@ -524,12 +525,12 @@ namespace AMDiS { ...@@ -524,12 +525,12 @@ namespace AMDiS {
stdMpi.send(it.getRank(), dofs); stdMpi.send(it.getRank(), dofs);
} }
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (int i = 0; i < vec.getSize(); i++) { for (int i = 0; i < vec.getSize(); i++) {
...@@ -659,9 +660,11 @@ namespace AMDiS { ...@@ -659,9 +660,11 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::getAllBoundaryDofs()"); FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
DofContainerSet dofSet; DofContainerSet dofSet;
for (DofComm::Iterator it(sendDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getSendDofs(), level, feSpace);
!it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end()); dofSet.insert(it.getDofs().begin(), it.getDofs().end());
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end()); dofSet.insert(it.getDofs().begin(), it.getDofs().end());
dofs.clear(); dofs.clear();
...@@ -1533,14 +1536,8 @@ namespace AMDiS { ...@@ -1533,14 +1536,8 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createBoundaryDofs()"); FUNCNAME("MeshDistributor::createBoundaryDofs()");
int nLevels = levelData.getLevelNumber(); dofComm.init(levelData.getLevelNumber(), feSpaces);
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); dofComm.create(intBoundary);
sendDofs.init(nLevels, feSpaces);
sendDofs.create(intBoundary.getOwn());
recvDofs.init(nLevels, feSpaces);
recvDofs.create(intBoundary.getOther());
createBoundaryDofInfo(); createBoundaryDofInfo();
} }
...@@ -1625,7 +1622,7 @@ namespace AMDiS { ...@@ -1625,7 +1622,7 @@ namespace AMDiS {
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
dofMap.init(levelData, feSpaces, feSpaces, true, true); dofMap.init(levelData, feSpaces, feSpaces, true, true);
dofMap.setDofComm(sendDofs, recvDofs); dofMap.setDofComm(dofComm);
dofMap.clear(); dofMap.clear();
createBoundaryDofs(); createBoundaryDofs();
...@@ -1684,7 +1681,8 @@ namespace AMDiS { ...@@ -1684,7 +1681,8 @@ namespace AMDiS {
int nLevels = levelData.getLevelNumber(); int nLevels = levelData.getLevelNumber();
for (int level = 0; level < nLevels; level++) { for (int level = 0; level < nLevels; level++) {
DofContainerSet nonRankDofs; DofContainerSet nonRankDofs;
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof()); nonRankDofs.insert(it.getDof());
...@@ -1692,7 +1690,8 @@ namespace AMDiS { ...@@ -1692,7 +1690,8 @@ namespace AMDiS {
if (nonRankDofs.count(rankDofs[i]) == 0) if (nonRankDofs.count(rankDofs[i]) == 0)
dofMap[feSpace].insertRankDof(level, *(rankDofs[i])); dofMap[feSpace].insertRankDof(level, *(rankDofs[i]));
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
dofMap[feSpace].insertNonRankDof(level, it.getDofIndex()); dofMap[feSpace].insertNonRankDof(level, it.getDofIndex());
} }
...@@ -1702,9 +1701,7 @@ namespace AMDiS { ...@@ -1702,9 +1701,7 @@ namespace AMDiS {
lastMeshChangeIndex = mesh->getChangeIndex(); lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this, ParallelDebug::testDofContainerCommunication(*this);
sendDofs.getData(),
recvDofs.getData());
#endif #endif
} }
...@@ -1714,13 +1711,9 @@ namespace AMDiS { ...@@ -1714,13 +1711,9 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::createPeriodicMap()"); FUNCNAME("MeshDistributor::createPeriodicMap()");
// Clear all periodic DOF mappings calculated before. We do it from scratch. // Clear all periodic DOF mappings calculated before. We do it from scratch.
periodicDofs.init(levelData.getLevelNumber(), feSpaces);
periodicMap.clear(); periodicMap.clear();
// If there are no periodic boundaries, return. Note that periodicDofs and // If there are no periodic boundaries, return.
// periodicMap must be still cleared before: if we do repartitioning and
// there were periodic boundaries in subdomain before and after repartitioning
// there are no more periodic boundaries.
if (!intBoundary.hasPeriodic()) if (!intBoundary.hasPeriodic())
return; return;
...@@ -1736,6 +1729,8 @@ namespace AMDiS { ...@@ -1736,6 +1729,8 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createPeriodicMap()"); FUNCNAME("MeshDistributor::createPeriodicMap()");
DofComm::LevelDataType &periodicDofs = dofComm.getPeriodicDofs();
StdMpi<vector<int> > stdMpi(mpiComm, false); StdMpi<vector<int> > stdMpi(mpiComm, false);
// === Each rank traverse its periodic boundaries and sends the DOF === // === Each rank traverse its periodic boundaries and sends the DOF ===
...@@ -1781,7 +1776,7 @@ namespace AMDiS { ...@@ -1781,7 +1776,7 @@ namespace AMDiS {
// Here we have a periodic boundary between two ranks. // Here we have a periodic boundary between two ranks.
// Create DOF indices on the boundary. // Create DOF indices on the boundary.
DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace); DofContainer& dofs = periodicDofs[0][it->first][feSpace];
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
...@@ -1814,7 +1809,7 @@ namespace AMDiS { ...@@ -1814,7 +1809,7 @@ namespace AMDiS {
for (RankToBoundMap::iterator it = intBoundary.getPeriodic().begin(); for (RankToBoundMap::iterator it = intBoundary.getPeriodic().begin();
it != intBoundary.getPeriodic().end(); ++it) { it != intBoundary.getPeriodic().end(); ++it) {
DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace); DofContainer& dofs = periodicDofs[0][it->first][feSpace];
vector<int>& types = rankToDofType[it->first]; vector<int>& types = rankToDofType[it->first];
TEST_EXIT_DBG(dofs.size() == types.size())("Should not happen!\n"); TEST_EXIT_DBG(dofs.size() == types.size())("Should not happen!\n");
...@@ -1934,8 +1929,7 @@ namespace AMDiS { ...@@ -1934,8 +1929,7 @@ namespace AMDiS {
intBoundary.serialize(out); intBoundary.serialize(out);
serialize(out, sendDofs.getData()); dofComm.serialize(out);
serialize(out, recvDofs.getData());
// === Serialieze FE space dependent data === // === Serialieze FE space dependent data ===
...@@ -1993,8 +1987,7 @@ namespace AMDiS { ...@@ -1993,8 +1987,7 @@ namespace AMDiS {
intBoundary.deserialize(in, elIndexMap); intBoundary.deserialize(in, elIndexMap);
deserialize(in, sendDofs.getData(), dofIndexMap); dofComm.deserialize(in, dofIndexMap);
deserialize(in, recvDofs.getData(), dofIndexMap);
// === Deerialieze FE space dependent data === // === Deerialieze FE space dependent data ===
......
...@@ -157,19 +157,9 @@ namespace AMDiS { ...@@ -157,19 +157,9 @@ namespace AMDiS {
return periodicMap; return periodicMap;
} }
DofComm& getSendDofs() DofComm& getDofComm()
{ {
return sendDofs; return dofComm;
}
DofComm& getRecvDofs()
{
return recvDofs;
}
DofComm& getPeriodicDofs()
{
return periodicDofs;
} }
inline long getLastMeshChangeIndex() inline long getLastMeshChangeIndex()
...@@ -220,7 +210,8 @@ namespace AMDiS { ...@@ -220,7 +210,8 @@ namespace AMDiS {
const FiniteElemSpace *fe = vec.getFeSpace(); const FiniteElemSpace *fe = vec.getFeSpace();
for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getSendDofs(), fe);
!it.end(); it.nextRank()) {
vector<T> dofs; vector<T> dofs;
dofs.reserve(it.getDofs().size()); dofs.reserve(it.getDofs().size());
...@@ -230,12 +221,14 @@ namespace AMDiS { ...@@ -230,12 +221,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), dofs); stdMpi.send(it.getRank(), dofs);
} }
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs());
!it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), fe);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
vec[it.getDofIndex()] = vec[it.getDofIndex()] =
stdMpi.getRecvData(it.getRank())[it.getDofCounter()]; stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
...@@ -477,20 +470,7 @@ namespace AMDiS { ...@@ -477,20 +470,7 @@ namespace AMDiS {
/// partitioning the whole mesh. /// partitioning the whole mesh.
InteriorBoundary intBoundary; InteriorBoundary intBoundary;
/// This map contains for each rank the list of DOFs the current rank must DofComm dofComm;
/// end to exchange solution DOFs at the interior boundaries.
DofComm sendDofs;
/// This map contains on each rank the list of DOFs from which the current
/// rank will receive DOF values (i.e., this are all DOFs at an interior
/// boundary). The DOF indices are given in rank's local numbering.
DofComm recvDofs;
/// This map contains on each rank a list of DOFs along the interior bound-
/// aries to communicate with other ranks. The DOF indices are given in rank's
/// local numbering. Periodic boundaries within one subdomain are not
/// considered here.
DofComm periodicDofs;
PeriodicMap periodicMap; PeriodicMap periodicMap;
......
...@@ -354,11 +354,13 @@ namespace AMDiS { ...@@ -354,11 +354,13 @@ namespace AMDiS {
DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds"); DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds");
pdb.mesh->getDofIndexCoords(feSpace, coords); pdb.mesh->getDofIndexCoords(feSpace, coords);
for (DofComm::Iterator it(pdb.sendDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(pdb.dofComm.getSendDofs(), feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
sendCoords[it.getRank()].push_back(coords[it.getDofIndex()]); sendCoords[it.getRank()].push_back(coords[it.getDofIndex()]);
for (DofComm::Iterator it(pdb.recvDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(pdb.dofComm.getRecvDofs(), feSpace);
!it.end(); it.nextRank())