Skip to content
GitLab
Commit 78d5dc6b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

More changes to get multilevel FETI-DP runing.

parent 531be06c
Show whitespace changes
Inline Side-by-side
AMDiS/src/parallel/ElementObjectDatabase.cc
View file @ 78d5dc6b
......@@ -15,8 +15,13 @@
namespace AMDiS {
void ElementObjectDatabase::create()
void ElementObjectDatabase::create(map<int, int>& rankMap, MeshLevelData& ld)
{
FUNCNAME("ElementObjectDatabase::create()");
macroElementRankMap = &rankMap;
levelData = &ld;
// === Fills macro element data structures. ===
TraverseStack stack;
......@@ -417,104 +422,41 @@ namespace AMDiS {
}
void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap,
MeshLevelData& levelData)
void ElementObjectDatabase::updateRankData()
{
FUNCNAME("ElementObjectDatabase::createRankData()");
int nLevel = levelData.getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
FUNCNAME("ElementObjectDatabase::updateRankData()");
vertexOwner.clear();
vertexInRank.clear();
for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank =(* macroElementRankMap)[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
vertexMaxLevel[it->first] = maxLevel;
}
edgeOwner.clear();
edgeInRank.clear();
for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
edgeMaxLevel[it->first] = maxLevel;
}
faceOwner.clear();
faceInRank.clear();
for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
faceMaxLevel[it->first] = maxLevel;
}
}
......@@ -630,6 +572,98 @@ namespace AMDiS {
}
int ElementObjectDatabase::getIterateOwner()
{
FUNCNAME("ElementObjectDatabase::getIterateOwner()");
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
{
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
default:
ERROR_EXIT("Should not happen!\n");
}
return owner;
}
int ElementObjectDatabase::getIterateMaxLevel()
{
FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()");
int nLevel = levelData->getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
break;
default:
ERROR_EXIT("Should not happen!\n");
}
for (std::set<int>::iterator it = ranksInLevel[0].begin();
it != ranksInLevel[0].end(); ++it)
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData->getLevelId(level, *it));
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
return maxLevel;
}
void ElementObjectDatabase::serialize(ostream &out)
{
FUNCNAME("ElementObjectDatabase::serialize()");
......@@ -686,11 +720,6 @@ namespace AMDiS {
ERROR_EXIT("REWRITE SERIALIZATION!\n");
/*
SerUtil::serialize(out, vertexOwner);
SerUtil::serialize(out, edgeOwner);
SerUtil::serialize(out, faceOwner);
*/
nSize = vertexInRank.size();
SerUtil::serialize(out, nSize);
......@@ -825,12 +854,7 @@ namespace AMDiS {
faceLocalMap[data] = face;
}
ERROR_EXIT("REWRITE DESERIALIZATION!\n");
SerUtil::deserialize(in, vertexOwner);
SerUtil::deserialize(in, edgeOwner);
SerUtil::deserialize(in, faceOwner);
SerUtil::deserialize(in, nSize);
vertexInRank.clear();
......
AMDiS/src/parallel/ElementObjectDatabase.h
View file @ 78d5dc6b
......@@ -105,7 +105,9 @@ namespace AMDiS {
ElementObjectDatabase()
: feSpace(NULL),
mesh(NULL),
iterGeoPos(CENTER)
iterGeoPos(CENTER),
macroElementRankMap(NULL),
levelData(NULL)
{}
void setFeSpace(const FiniteElemSpace *fe)
......@@ -119,7 +121,12 @@ namespace AMDiS {
return mesh;
}
void create();
/*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/
void create(map<int, int>& macroElementRankMap,
MeshLevelData& levelData);
void createMacroElementInfo(vector<MacroElement*> &mel);
......@@ -127,13 +134,8 @@ namespace AMDiS {
* Create for a filled object database the membership information for all
* element objects. An object is owned by a rank, if the rank has the
* heighest rank number of all ranks where the object is part of.
*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/
void createRankData(map<int, int>& macroElementRankMap,
MeshLevelData& levelData);
void updateRankData();
/** \brief
* Iterates over all elements for one geometrical index, i.e., over all
......@@ -227,67 +229,11 @@ namespace AMDiS {
}
}
/// Returns the rank owner of the current iterator position.
int getIterateOwner()
{
switch (iterGeoPos) {
case VERTEX:
return vertexOwner[vertexIter->first];
break;
case EDGE:
return edgeOwner[edgeIter->first];
break;
case FACE:
return faceOwner[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
int getIterateOwner();
/// Returns the rank owner of the current iterator position.
int getIterateMaxLevel()
{
switch (iterGeoPos) {
case VERTEX:
return vertexMaxLevel[vertexIter->first];
break;
case EDGE:
return edgeMaxLevel[edgeIter->first];
break;
case FACE:
return faceMaxLevel[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
/// Returns the rank owner of a vertex DOF.
int getOwner(DegreeOfFreedom vertex)
{
return vertexOwner[vertex];
}
/// Returns the rank owner of an edge.
int getOwner(DofEdge edge)
{
return edgeOwner[edge];
}
/// Returns the rank owner of an face.
int getOwner(DofFace face, int level)
{
return faceOwner[face];
}
int getIterateMaxLevel();
/// Checks if a given vertex DOF is in a given rank.
int isInRank(DegreeOfFreedom vertex, int rank)
......@@ -567,22 +513,6 @@ namespace AMDiS {
/// Maps to an element object the corresponding face.
map<ElementObjectData, DofFace> faceLocalMap;
/// Defines for all vertex DOFs the rank that ownes this vertex DOF.
map<DegreeOfFreedom, int> vertexOwner;
/// Defines for all edges the rank that ownes this edge.
map<DofEdge, int> edgeOwner;
/// Defines for all faces the rank that ownes this face.
map<DofFace, int> faceOwner;
map<DegreeOfFreedom, int> vertexMaxLevel;
map<DofEdge, int> edgeMaxLevel;
map<DofFace, int> faceMaxLevel;
/// Defines to each vertex DOF a map that maps to each rank number the element
/// objects that have this vertex DOF in common.
......@@ -636,11 +566,15 @@ namespace AMDiS {
map<pair<ElementObjectData, ElementObjectData>, bool> faceReverseMode;
map<int, int> *macroElementRankMap;
/// Maps to each macro element index a pointer to the corresponding element.
map<int, Element*> macroElIndexMap;
/// Maps to each macro element index the type of this element.
map<int, int> macroElIndexTypeMap;
MeshLevelData* levelData;
};
}
......
AMDiS/src/parallel/InteriorBoundary.cc
View file @ 78d5dc6b
......@@ -19,28 +19,6 @@
namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewAtomicOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicOther(int rank)
{
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicPer(int rank)
{
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
void InteriorBoundary::create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb)
{
......@@ -70,7 +48,6 @@ namespace AMDiS {
AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex);
......@@ -101,7 +78,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOwn(it2->first);
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
if (geoIndex == EDGE)
b.neighObj.reverseMode =
......@@ -125,7 +102,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOther(owner);
AtomicBoundary& b = getNewOther(owner);
b = bound;
if (geoIndex == EDGE)
b.rankObj.reverseMode =
......@@ -169,7 +146,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
}
......@@ -203,7 +180,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -249,7 +226,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -465,6 +442,30 @@ namespace AMDiS {
}
AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewOther(int rank)
{
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
{
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
void InteriorBoundary::serializeExcludeList(std::ostream &out,
ExcludeList &list)
{
......
AMDiS/src/parallel/InteriorBoundary.h
View file @ 78d5dc6b
......@@ -44,26 +44,47 @@ namespace AMDiS {
void create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb);
AtomicBoundary& getNewAtomicOwn(int rank);
AtomicBoundary& getNewAtomicOther(int rank);
AtomicBoundary& getNewAtomicPer(int rank);
/// Writes this object to a file.
void serialize(ostream &out);
/// Reads the state of an interior boundary from a file.
void deserialize(istream &in, map<int, Element*> &elIndexMap);
RankToBoundMap& getOwn()
{
return own;
}
RankToBoundMap& getOther()
{
return other;
}
RankToBoundMap& getPeriodic()
{
return periodic;
}
bool hasPeriodic()
{
return static_cast<bool>(periodic.size());
}
private:
AtomicBoundary& getNewOwn(int rank);
AtomicBoundary& getNewOther(int rank);
AtomicBoundary& getNewPeriodic(int rank);
void serializeExcludeList(ostream &out, ExcludeList &list);
void deserializeExcludeList(istream &in, ExcludeList &list);
public:
private:
RankToBoundMap own, other, periodic;
friend class ParallelDebug;
public:
/// Iterator for the interior boundary object.
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 78d5dc6b
......@@ -907,17 +907,19 @@ namespace AMDiS {
// important. Therefore, we add all boundaries to one boundary container.
RankToBoundMap allBound;
for (InteriorBoundary::iterator it(intBoundary.own); !it.end(); ++it)
for (InteriorBoundary::iterator it(intBoundary.getOwn()); !it.end(); ++it)
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE))
allBound[it.getRank()].push_back(*it);
for (InteriorBoundary::iterator it(intBoundary.other); !it.end(); ++it)
for (InteriorBoundary::iterator it(intBoundary.getOther());
!it.end(); ++it)
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE))
allBound[it.getRank()].push_back(*it);
for (InteriorBoundary::iterator it(intBoundary.periodic); !it.end(); ++it) {
for (InteriorBoundary::iterator it(intBoundary.getPeriodic());
!it.end(); ++it) {
if (it.getRank() == mpiRank) {
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE)) {
......@@ -1508,8 +1510,8 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
elObjDb.create();
elObjDb.createRankData(partitionMap, levelData);
elObjDb.create(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
}
......@@ -1518,7 +1520,7 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
elObjDb.createRankData(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
#if (DEBUG != 0)
......@@ -1535,10 +1537,10 @@ namespace AMDiS {
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
sendDofs.init(nLevels, feSpaces);