Skip to content
GitLab
Commit 78d5dc6b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

More changes to get multilevel FETI-DP runing.

parent 531be06c
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/ElementObjectDatabase.cc
View file @ 78d5dc6b
...@@ -15,8 +15,13 @@ ...@@ -15,8 +15,13 @@
namespace AMDiS { namespace AMDiS {
void ElementObjectDatabase::create() void ElementObjectDatabase::create(map<int, int>& rankMap, MeshLevelData& ld)
{ {
FUNCNAME("ElementObjectDatabase::create()");
macroElementRankMap = &rankMap;
levelData = &ld;
// === Fills macro element data structures. === // === Fills macro element data structures. ===
TraverseStack stack; TraverseStack stack;
...@@ -417,104 +422,41 @@ namespace AMDiS { ...@@ -417,104 +422,41 @@ namespace AMDiS {
} }
void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap, void ElementObjectDatabase::updateRankData()
MeshLevelData& levelData)
{ {
FUNCNAME("ElementObjectDatabase::createRankData()"); FUNCNAME("ElementObjectDatabase::updateRankData()");
int nLevel = levelData.getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vertexOwner.clear();
vertexInRank.clear(); vertexInRank.clear();
for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin(); for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) { it != vertexElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin(); for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) { it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex]; int elementInRank =(* macroElementRankMap)[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex) if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elementInRank] = *it2; vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
} }
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
vertexMaxLevel[it->first] = maxLevel;
} }
edgeOwner.clear();
edgeInRank.clear(); edgeInRank.clear();
for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin(); for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) { it != edgeElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin(); for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) { it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex]; int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex) if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elementInRank] = *it2; edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
} }
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
edgeMaxLevel[it->first] = maxLevel;
} }
faceOwner.clear();
faceInRank.clear(); faceInRank.clear();
for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin(); for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) { it != faceElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin(); for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) { it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex]; int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex) if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elementInRank] = *it2; faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
} }
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
faceMaxLevel[it->first] = maxLevel;
} }
} }
...@@ -630,6 +572,98 @@ namespace AMDiS { ...@@ -630,6 +572,98 @@ namespace AMDiS {
} }
int ElementObjectDatabase::getIterateOwner()
{
FUNCNAME("ElementObjectDatabase::getIterateOwner()");
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
{
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
default:
ERROR_EXIT("Should not happen!\n");
}
return owner;
}
int ElementObjectDatabase::getIterateMaxLevel()
{
FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()");
int nLevel = levelData->getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
break;
default:
ERROR_EXIT("Should not happen!\n");
}
for (std::set<int>::iterator it = ranksInLevel[0].begin();
it != ranksInLevel[0].end(); ++it)
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData->getLevelId(level, *it));
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
return maxLevel;
}
void ElementObjectDatabase::serialize(ostream &out) void ElementObjectDatabase::serialize(ostream &out)
{ {
FUNCNAME("ElementObjectDatabase::serialize()"); FUNCNAME("ElementObjectDatabase::serialize()");
...@@ -686,11 +720,6 @@ namespace AMDiS { ...@@ -686,11 +720,6 @@ namespace AMDiS {
ERROR_EXIT("REWRITE SERIALIZATION!\n"); ERROR_EXIT("REWRITE SERIALIZATION!\n");
/*
SerUtil::serialize(out, vertexOwner);
SerUtil::serialize(out, edgeOwner);
SerUtil::serialize(out, faceOwner);
*/
nSize = vertexInRank.size(); nSize = vertexInRank.size();
SerUtil::serialize(out, nSize); SerUtil::serialize(out, nSize);
...@@ -825,12 +854,7 @@ namespace AMDiS { ...@@ -825,12 +854,7 @@ namespace AMDiS {
faceLocalMap[data] = face; faceLocalMap[data] = face;
} }
ERROR_EXIT("REWRITE DESERIALIZATION!\n"); ERROR_EXIT("REWRITE DESERIALIZATION!\n");
SerUtil::deserialize(in, vertexOwner);
SerUtil::deserialize(in, edgeOwner);
SerUtil::deserialize(in, faceOwner);
SerUtil::deserialize(in, nSize); SerUtil::deserialize(in, nSize);
vertexInRank.clear(); vertexInRank.clear();
......
AMDiS/src/parallel/ElementObjectDatabase.h
View file @ 78d5dc6b
...@@ -105,7 +105,9 @@ namespace AMDiS { ...@@ -105,7 +105,9 @@ namespace AMDiS {
ElementObjectDatabase() ElementObjectDatabase()
: feSpace(NULL), : feSpace(NULL),
mesh(NULL), mesh(NULL),
iterGeoPos(CENTER) iterGeoPos(CENTER),
macroElementRankMap(NULL),
levelData(NULL)
{} {}
void setFeSpace(const FiniteElemSpace *fe) void setFeSpace(const FiniteElemSpace *fe)
...@@ -119,7 +121,12 @@ namespace AMDiS { ...@@ -119,7 +121,12 @@ namespace AMDiS {
return mesh; return mesh;
} }
void create(); /*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/
void create(map<int, int>& macroElementRankMap,
MeshLevelData& levelData);
void createMacroElementInfo(vector<MacroElement*> &mel); void createMacroElementInfo(vector<MacroElement*> &mel);
...@@ -127,13 +134,8 @@ namespace AMDiS { ...@@ -127,13 +134,8 @@ namespace AMDiS {
* Create for a filled object database the membership information for all * Create for a filled object database the membership information for all
* element objects. An object is owned by a rank, if the rank has the * element objects. An object is owned by a rank, if the rank has the
* heighest rank number of all ranks where the object is part of. * heighest rank number of all ranks where the object is part of.
*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/ */
void createRankData(map<int, int>& macroElementRankMap, void updateRankData();
MeshLevelData& levelData);
/** \brief /** \brief
* Iterates over all elements for one geometrical index, i.e., over all * Iterates over all elements for one geometrical index, i.e., over all
...@@ -227,67 +229,11 @@ namespace AMDiS { ...@@ -227,67 +229,11 @@ namespace AMDiS {
} }
} }
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateOwner() int getIterateOwner();
{
switch (iterGeoPos) {
case VERTEX:
return vertexOwner[vertexIter->first];
break;
case EDGE:
return edgeOwner[edgeIter->first];
break;
case FACE:
return faceOwner[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateMaxLevel() int getIterateMaxLevel();
{
switch (iterGeoPos) {
case VERTEX:
return vertexMaxLevel[vertexIter->first];
break;
case EDGE:
return edgeMaxLevel[edgeIter->first];
break;
case FACE:
return faceMaxLevel[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
/// Returns the rank owner of a vertex DOF.
int getOwner(DegreeOfFreedom vertex)
{
return vertexOwner[vertex];
}
/// Returns the rank owner of an edge.
int getOwner(DofEdge edge)
{
return edgeOwner[edge];
}
/// Returns the rank owner of an face.
int getOwner(DofFace face, int level)
{
return faceOwner[face];
}
/// Checks if a given vertex DOF is in a given rank. /// Checks if a given vertex DOF is in a given rank.
int isInRank(DegreeOfFreedom vertex, int rank) int isInRank(DegreeOfFreedom vertex, int rank)
...@@ -567,23 +513,7 @@ namespace AMDiS { ...@@ -567,23 +513,7 @@ namespace AMDiS {
/// Maps to an element object the corresponding face. /// Maps to an element object the corresponding face.
map<ElementObjectData, DofFace> faceLocalMap; map<ElementObjectData, DofFace> faceLocalMap;
/// Defines for all vertex DOFs the rank that ownes this vertex DOF.
map<DegreeOfFreedom, int> vertexOwner;
/// Defines for all edges the rank that ownes this edge.
map<DofEdge, int> edgeOwner;
/// Defines for all faces the rank that ownes this face.
map<DofFace, int> faceOwner;
map<DegreeOfFreedom, int> vertexMaxLevel;
map<DofEdge, int> edgeMaxLevel;
map<DofFace, int> faceMaxLevel;
/// Defines to each vertex DOF a map that maps to each rank number the element /// Defines to each vertex DOF a map that maps to each rank number the element
/// objects that have this vertex DOF in common. /// objects that have this vertex DOF in common.
map<DegreeOfFreedom, map<int, ElementObjectData> > vertexInRank; map<DegreeOfFreedom, map<int, ElementObjectData> > vertexInRank;
...@@ -636,11 +566,15 @@ namespace AMDiS { ...@@ -636,11 +566,15 @@ namespace AMDiS {
map<pair<ElementObjectData, ElementObjectData>, bool> faceReverseMode; map<pair<ElementObjectData, ElementObjectData>, bool> faceReverseMode;
map<int, int> *macroElementRankMap;
/// Maps to each macro element index a pointer to the corresponding element. /// Maps to each macro element index a pointer to the corresponding element.
map<int, Element*> macroElIndexMap; map<int, Element*> macroElIndexMap;
/// Maps to each macro element index the type of this element. /// Maps to each macro element index the type of this element.
map<int, int> macroElIndexTypeMap; map<int, int> macroElIndexTypeMap;
MeshLevelData* levelData;
}; };
} }
......
AMDiS/src/parallel/InteriorBoundary.cc
View file @ 78d5dc6b
...@@ -19,28 +19,6 @@ ...@@ -19,28 +19,6 @@
namespace AMDiS { namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewAtomicOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicOther(int rank)
{
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicPer(int rank)
{
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
void InteriorBoundary::create(MPI::Intracomm &mpiComm, void InteriorBoundary::create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb) ElementObjectDatabase &elObjDb)
{ {
...@@ -70,7 +48,6 @@ namespace AMDiS { ...@@ -70,7 +48,6 @@ namespace AMDiS {
AtomicBoundary bound; AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel(); bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex); bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
bound.rankObj.elIndex = rankBoundEl.elIndex; bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex); bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex);
...@@ -101,7 +78,7 @@ namespace AMDiS { ...@@ -101,7 +78,7 @@ namespace AMDiS {
bound.type = INTERIOR; bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOwn(it2->first); AtomicBoundary& b = getNewOwn(it2->first);
b = bound; b = bound;
if (geoIndex == EDGE) if (geoIndex == EDGE)
b.neighObj.reverseMode = b.neighObj.reverseMode =
...@@ -125,7 +102,7 @@ namespace AMDiS { ...@@ -125,7 +102,7 @@ namespace AMDiS {
bound.type = INTERIOR; bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOther(owner); AtomicBoundary& b = getNewOther(owner);
b = bound; b = bound;
if (geoIndex == EDGE) if (geoIndex == EDGE)
b.rankObj.reverseMode = b.rankObj.reverseMode =
...@@ -169,7 +146,7 @@ namespace AMDiS { ...@@ -169,7 +146,7 @@ namespace AMDiS {
bound.type = it->second; bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank); AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound; b = bound;
} }
} }
...@@ -203,7 +180,7 @@ namespace AMDiS { ...@@ -203,7 +180,7 @@ namespace AMDiS {
bound.type = it->second; bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank); AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound; b = bound;
if (mpiRank > otherElementRank) if (mpiRank > otherElementRank)
...@@ -249,7 +226,7 @@ namespace AMDiS { ...@@ -249,7 +226,7 @@ namespace AMDiS {
bound.type = it->second; bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank); AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound; b = bound;
if (mpiRank > otherElementRank) if (mpiRank > otherElementRank)
...@@ -465,6 +442,30 @@ namespace AMDiS { ...@@ -465,6 +442,30 @@ namespace AMDiS {
} }
AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}