Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit 37e0d38c authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

And more changes on the DofCommunicator.

parent 78d5dc6b
...@@ -18,26 +18,15 @@ namespace AMDiS { ...@@ -18,26 +18,15 @@ namespace AMDiS {
using namespace std; using namespace std;
int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace) void DofComm::create(InteriorBoundary &boundary)
{ {
FUNCNAME("DofComm::getNumberDofs()"); createContainer(boundary.getOwn(), sendDofs);
createContainer(boundary.getOther(), recvDofs);
TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofs;
for (DataIter rankIt = data[level].begin();
rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace)
dofs.insert(feIt->second.begin(), feIt->second.end());
return static_cast<int>(dofs.size());
} }
void DofComm::create(RankToBoundMap &boundary) void DofComm::createContainer(RankToBoundMap &boundary,
LevelDataType &data)
{ {
// === Fill data. === // === Fill data. ===
...@@ -72,11 +61,32 @@ namespace AMDiS { ...@@ -72,11 +61,32 @@ namespace AMDiS {
} }
int DofComm::getNumberDofs(LevelDataType &data,
int level,
const FiniteElemSpace *feSpace)
{
FUNCNAME("DofComm::getNumberDofs()");
TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofs;
for (DataIter rankIt = data[level].begin();
rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace)
dofs.insert(feIt->second.begin(), feIt->second.end());
return static_cast<int>(dofs.size());
}
bool DofComm::Iterator::setNextFeMap() bool DofComm::Iterator::setNextFeMap()
{ {
FUNCNAME("DofComm::Iterator::setNextFeMap()"); FUNCNAME("DofComm::Iterator::setNextFeMap()");
if (dataIter != dofComm.data[traverseLevel].end()) { if (dataIter != data[traverseLevel].end()) {
TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n"); TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
feMapIter = dataIter->second.begin(); feMapIter = dataIter->second.begin();
......
...@@ -36,7 +36,9 @@ namespace AMDiS { ...@@ -36,7 +36,9 @@ namespace AMDiS {
{ {
public: public:
DofComm() DofComm()
: data(1) : recvDofs(1),
sendDofs(1),
periodicDofs(0)
{} {}
typedef map<const FiniteElemSpace*, DofContainer> FeMapType; typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
...@@ -46,33 +48,76 @@ namespace AMDiS { ...@@ -46,33 +48,76 @@ namespace AMDiS {
// meshLevel: map[rank -> map[feSpace -> DofContainer]] // meshLevel: map[rank -> map[feSpace -> DofContainer]]
typedef vector<DataType> LevelDataType; typedef vector<DataType> LevelDataType;
inline DofContainer& getDofContainer(int rank,
const FiniteElemSpace *feSpace,
int level = 0)
{
return data[level][rank][feSpace];
}
void init(int n, vector<const FiniteElemSpace*> &fe) void init(int n, vector<const FiniteElemSpace*> &fe)
{ {
FUNCNAME("DofComm::init()");
TEST_EXIT_DBG(n >= 1)("Should not happen!\n");
nLevel = n; nLevel = n;
feSpaces = fe; feSpaces = fe;
data.clear(); sendDofs.clear();
data.resize(nLevel); recvDofs.clear();
periodicDofs.clear();
sendDofs.resize(nLevel);
recvDofs.resize(nLevel);
periodicDofs.resize(nLevel);
}
void create(InteriorBoundary &boundary);
LevelDataType& getSendDofs()
{
return sendDofs;
}
LevelDataType& getRecvDofs()
{
return recvDofs;
} }
DataType& getData(int level = 0) LevelDataType& getPeriodicDofs()
{ {
return data[level]; return periodicDofs;
} }
int getNumberDofs(int level, const FiniteElemSpace *feSpace); // Writes all data of this object to an output stream.
void serialize(ostream &out)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
void create(RankToBoundMap &boundary); // Reads the object data from an input stream.
void deserialize(istream &in,
map<const FiniteElemSpace*, map<int, const DegreeOfFreedom*> > dofIndexMap)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
int getNumberDofs(LevelDataType &data,
int level,
const FiniteElemSpace *feSpace);
protected:
void createContainer(RankToBoundMap &boundary, LevelDataType &data);
protected: protected:
LevelDataType data; /// This map contains for each rank the list of DOFs the current rank must
/// end to exchange solution DOFs at the interior boundaries.
LevelDataType sendDofs;
/// This map contains on each rank the list of DOFs from which the current
/// rank will receive DOF values (i.e., this are all DOFs at an interior
/// boundary). The DOF indices are given in rank's local numbering.
LevelDataType recvDofs;
/// This map contains on each rank a list of DOFs along the interior bound-
/// aries to communicate with other ranks. The DOF indices are given in rank's
/// local numbering. Periodic boundaries within one subdomain are not
/// considered here.
LevelDataType periodicDofs;
int nLevel; int nLevel;
...@@ -84,9 +129,9 @@ namespace AMDiS { ...@@ -84,9 +129,9 @@ namespace AMDiS {
class Iterator class Iterator
{ {
public: public:
Iterator(DofComm &dc, Iterator(LevelDataType &d,
const FiniteElemSpace *fe = NULL) const FiniteElemSpace *fe = NULL)
: dofComm(dc), : data(d),
dofCounter(-1), dofCounter(-1),
traverseFeSpace(fe), traverseFeSpace(fe),
traverseLevel(0) traverseLevel(0)
...@@ -94,10 +139,10 @@ namespace AMDiS { ...@@ -94,10 +139,10 @@ namespace AMDiS {
goFirst(); goFirst();
} }
Iterator(DofComm &dc, Iterator(LevelDataType &d,
int level, int level,
const FiniteElemSpace *fe = NULL) const FiniteElemSpace *fe = NULL)
: dofComm(dc), : data(d),
dofCounter(-1), dofCounter(-1),
traverseFeSpace(fe), traverseFeSpace(fe),
traverseLevel(level) traverseLevel(level)
...@@ -107,7 +152,7 @@ namespace AMDiS { ...@@ -107,7 +152,7 @@ namespace AMDiS {
inline bool end() inline bool end()
{ {
return (dataIter == dofComm.data[traverseLevel].end()); return (dataIter == data[traverseLevel].end());
} }
inline void nextRank() inline void nextRank()
...@@ -200,7 +245,7 @@ namespace AMDiS { ...@@ -200,7 +245,7 @@ namespace AMDiS {
protected: protected:
void goFirst() void goFirst()
{ {
dataIter = dofComm.data[traverseLevel].begin(); dataIter = data[traverseLevel].begin();
while (setNextFeMap() == false) while (setNextFeMap() == false)
++dataIter; ++dataIter;
...@@ -209,7 +254,7 @@ namespace AMDiS { ...@@ -209,7 +254,7 @@ namespace AMDiS {
bool setNextFeMap(); bool setNextFeMap();
protected: protected:
DofComm &dofComm; LevelDataType &data;
DofComm::DataIter dataIter; DofComm::DataIter dataIter;
......
...@@ -511,7 +511,8 @@ namespace AMDiS { ...@@ -511,7 +511,8 @@ namespace AMDiS {
StdMpi<vector<double> > stdMpi(mpiComm); StdMpi<vector<double> > stdMpi(mpiComm);
for (DofComm::Iterator it(sendDofs); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getSendDofs());
!it.end(); it.nextRank()) {
vector<double> dofs; vector<double> dofs;
for (int i = 0; i < vec.getSize(); i++) { for (int i = 0; i < vec.getSize(); i++) {
...@@ -524,12 +525,12 @@ namespace AMDiS { ...@@ -524,12 +525,12 @@ namespace AMDiS {
stdMpi.send(it.getRank(), dofs); stdMpi.send(it.getRank(), dofs);
} }
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (int i = 0; i < vec.getSize(); i++) { for (int i = 0; i < vec.getSize(); i++) {
...@@ -659,9 +660,11 @@ namespace AMDiS { ...@@ -659,9 +660,11 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::getAllBoundaryDofs()"); FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
DofContainerSet dofSet; DofContainerSet dofSet;
for (DofComm::Iterator it(sendDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getSendDofs(), level, feSpace);
!it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end()); dofSet.insert(it.getDofs().begin(), it.getDofs().end());
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end()); dofSet.insert(it.getDofs().begin(), it.getDofs().end());
dofs.clear(); dofs.clear();
...@@ -1533,14 +1536,8 @@ namespace AMDiS { ...@@ -1533,14 +1536,8 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createBoundaryDofs()"); FUNCNAME("MeshDistributor::createBoundaryDofs()");
int nLevels = levelData.getLevelNumber(); dofComm.init(levelData.getLevelNumber(), feSpaces);
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); dofComm.create(intBoundary);
sendDofs.init(nLevels, feSpaces);
sendDofs.create(intBoundary.getOwn());
recvDofs.init(nLevels, feSpaces);
recvDofs.create(intBoundary.getOther());
createBoundaryDofInfo(); createBoundaryDofInfo();
} }
...@@ -1625,7 +1622,7 @@ namespace AMDiS { ...@@ -1625,7 +1622,7 @@ namespace AMDiS {
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
dofMap.init(levelData, feSpaces, feSpaces, true, true); dofMap.init(levelData, feSpaces, feSpaces, true, true);
dofMap.setDofComm(sendDofs, recvDofs); dofMap.setDofComm(dofComm);
dofMap.clear(); dofMap.clear();
createBoundaryDofs(); createBoundaryDofs();
...@@ -1684,7 +1681,8 @@ namespace AMDiS { ...@@ -1684,7 +1681,8 @@ namespace AMDiS {
int nLevels = levelData.getLevelNumber(); int nLevels = levelData.getLevelNumber();
for (int level = 0; level < nLevels; level++) { for (int level = 0; level < nLevels; level++) {
DofContainerSet nonRankDofs; DofContainerSet nonRankDofs;
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof()); nonRankDofs.insert(it.getDof());
...@@ -1692,7 +1690,8 @@ namespace AMDiS { ...@@ -1692,7 +1690,8 @@ namespace AMDiS {
if (nonRankDofs.count(rankDofs[i]) == 0) if (nonRankDofs.count(rankDofs[i]) == 0)
dofMap[feSpace].insertRankDof(level, *(rankDofs[i])); dofMap[feSpace].insertRankDof(level, *(rankDofs[i]));
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
dofMap[feSpace].insertNonRankDof(level, it.getDofIndex()); dofMap[feSpace].insertNonRankDof(level, it.getDofIndex());
} }
...@@ -1702,9 +1701,7 @@ namespace AMDiS { ...@@ -1702,9 +1701,7 @@ namespace AMDiS {
lastMeshChangeIndex = mesh->getChangeIndex(); lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this, ParallelDebug::testDofContainerCommunication(*this);
sendDofs.getData(),
recvDofs.getData());
#endif #endif
} }
...@@ -1714,13 +1711,9 @@ namespace AMDiS { ...@@ -1714,13 +1711,9 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::createPeriodicMap()"); FUNCNAME("MeshDistributor::createPeriodicMap()");
// Clear all periodic DOF mappings calculated before. We do it from scratch. // Clear all periodic DOF mappings calculated before. We do it from scratch.
periodicDofs.init(levelData.getLevelNumber(), feSpaces);
periodicMap.clear(); periodicMap.clear();
// If there are no periodic boundaries, return. Note that periodicDofs and // If there are no periodic boundaries, return.
// periodicMap must be still cleared before: if we do repartitioning and
// there were periodic boundaries in subdomain before and after repartitioning
// there are no more periodic boundaries.
if (!intBoundary.hasPeriodic()) if (!intBoundary.hasPeriodic())
return; return;
...@@ -1736,6 +1729,8 @@ namespace AMDiS { ...@@ -1736,6 +1729,8 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createPeriodicMap()"); FUNCNAME("MeshDistributor::createPeriodicMap()");
DofComm::LevelDataType &periodicDofs = dofComm.getPeriodicDofs();
StdMpi<vector<int> > stdMpi(mpiComm, false); StdMpi<vector<int> > stdMpi(mpiComm, false);
// === Each rank traverse its periodic boundaries and sends the DOF === // === Each rank traverse its periodic boundaries and sends the DOF ===
...@@ -1781,7 +1776,7 @@ namespace AMDiS { ...@@ -1781,7 +1776,7 @@ namespace AMDiS {
// Here we have a periodic boundary between two ranks. // Here we have a periodic boundary between two ranks.
// Create DOF indices on the boundary. // Create DOF indices on the boundary.
DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace); DofContainer& dofs = periodicDofs[0][it->first][feSpace];
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
...@@ -1814,7 +1809,7 @@ namespace AMDiS { ...@@ -1814,7 +1809,7 @@ namespace AMDiS {
for (RankToBoundMap::iterator it = intBoundary.getPeriodic().begin(); for (RankToBoundMap::iterator it = intBoundary.getPeriodic().begin();
it != intBoundary.getPeriodic().end(); ++it) { it != intBoundary.getPeriodic().end(); ++it) {
DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace); DofContainer& dofs = periodicDofs[0][it->first][feSpace];
vector<int>& types = rankToDofType[it->first]; vector<int>& types = rankToDofType[it->first];
TEST_EXIT_DBG(dofs.size() == types.size())("Should not happen!\n"); TEST_EXIT_DBG(dofs.size() == types.size())("Should not happen!\n");
...@@ -1934,8 +1929,7 @@ namespace AMDiS { ...@@ -1934,8 +1929,7 @@ namespace AMDiS {
intBoundary.serialize(out); intBoundary.serialize(out);
serialize(out, sendDofs.getData()); dofComm.serialize(out);
serialize(out, recvDofs.getData());
// === Serialieze FE space dependent data === // === Serialieze FE space dependent data ===
...@@ -1993,8 +1987,7 @@ namespace AMDiS { ...@@ -1993,8 +1987,7 @@ namespace AMDiS {
intBoundary.deserialize(in, elIndexMap); intBoundary.deserialize(in, elIndexMap);
deserialize(in, sendDofs.getData(), dofIndexMap); dofComm.deserialize(in, dofIndexMap);
deserialize(in, recvDofs.getData(), dofIndexMap);
// === Deerialieze FE space dependent data === // === Deerialieze FE space dependent data ===
......
...@@ -157,19 +157,9 @@ namespace AMDiS { ...@@ -157,19 +157,9 @@ namespace AMDiS {
return periodicMap; return periodicMap;
} }
DofComm& getSendDofs() DofComm& getDofComm()
{ {
return sendDofs; return dofComm;
}
DofComm& getRecvDofs()
{
return recvDofs;
}
DofComm& getPeriodicDofs()
{
return periodicDofs;
} }
inline long getLastMeshChangeIndex() inline long getLastMeshChangeIndex()
...@@ -220,7 +210,8 @@ namespace AMDiS { ...@@ -220,7 +210,8 @@ namespace AMDiS {
const FiniteElemSpace *fe = vec.getFeSpace(); const FiniteElemSpace *fe = vec.getFeSpace();
for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) { for (DofComm::Iterator it(dofComm.getSendDofs(), fe);
!it.end(); it.nextRank()) {
vector<T> dofs; vector<T> dofs;
dofs.reserve(it.getDofs().size()); dofs.reserve(it.getDofs().size());
...@@ -230,12 +221,14 @@ namespace AMDiS { ...@@ -230,12 +221,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), dofs); stdMpi.send(it.getRank(), dofs);
} }
for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs());
!it.end(); it.nextRank())
stdMpi.recv(it.getRank()); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank()) for (DofComm::Iterator it(dofComm.getRecvDofs(), fe);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
vec[it.getDofIndex()] = vec[it.getDofIndex()] =
stdMpi.getRecvData(it.getRank())[it.getDofCounter()]; stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
...@@ -477,20 +470,7 @@ namespace AMDiS { ...@@ -477,20 +470,7 @@ namespace AMDiS {
/// partitioning the whole mesh. /// partitioning the whole mesh.
InteriorBoundary intBoundary; InteriorBoundary intBoundary;
/// This map contains for each rank the list of DOFs the current rank must DofComm dofComm;
/// end to exchange solution DOFs at the interior boundaries.
DofComm sendDofs;
/// This map contains on each rank the list of DOFs from which the current
/// rank will receive DOF values (i.e., this are all DOFs at an interior
/// boundary). The DOF indices are given in rank's local numbering.
DofComm recvDofs;
/// This map contains on each rank a list of DOFs along the interior bound-
/// aries to communicate with other ranks. The DOF indices are given in rank's
/// local numbering. Periodic boundaries within one subdomain are not
/// considered here.
DofComm periodicDofs;
PeriodicMap periodicMap; PeriodicMap periodicMap;
......
...@@ -354,11 +354,13 @@ namespace AMDiS { ...@@ -354,11 +354,13 @@ namespace AMDiS {
DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds"); DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds");
pdb.mesh->getDofIndexCoords(feSpace, coords); pdb.mesh->getDofIndexCoords(feSpace, coords);
for (DofComm::Iterator it(pdb.sendDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(pdb.dofComm.getSendDofs(), feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
sendCoords[it.getRank()].push_back(coords[it.getDofIndex()]); sendCoords[it.getRank()].push_back(coords[it.getDofIndex()]);
for (DofComm::Iterator it(pdb.recvDofs, feSpace); !it.end(); it.nextRank()) for (DofComm::Iterator it(pdb.dofComm.getRecvDofs(), feSpace);