Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 78d5dc6b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

More changes to get multilevel FETI-DP runing.

parent 531be06c
......@@ -15,8 +15,13 @@
namespace AMDiS {
void ElementObjectDatabase::create()
void ElementObjectDatabase::create(map<int, int>& rankMap, MeshLevelData& ld)
{
FUNCNAME("ElementObjectDatabase::create()");
macroElementRankMap = &rankMap;
levelData = &ld;
// === Fills macro element data structures. ===
TraverseStack stack;
......@@ -417,104 +422,41 @@ namespace AMDiS {
}
void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap,
MeshLevelData& levelData)
void ElementObjectDatabase::updateRankData()
{
FUNCNAME("ElementObjectDatabase::createRankData()");
FUNCNAME("ElementObjectDatabase::updateRankData()");
int nLevel = levelData.getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vertexOwner.clear();
vertexInRank.clear();
for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank =(* macroElementRankMap)[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
vertexMaxLevel[it->first] = maxLevel;
}
edgeOwner.clear();
edgeInRank.clear();
for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
edgeMaxLevel[it->first] = maxLevel;
}
faceOwner.clear();
faceInRank.clear();
for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
for (vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
int elementInRank = (*macroElementRankMap)[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
ranksInLevel[0].insert(elementInRank);
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
}
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
faceMaxLevel[it->first] = maxLevel;
}
}
......@@ -630,6 +572,98 @@ namespace AMDiS {
}
int ElementObjectDatabase::getIterateOwner()
{
FUNCNAME("ElementObjectDatabase::getIterateOwner()");
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
{
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
default:
ERROR_EXIT("Should not happen!\n");
}
return owner;
}
int ElementObjectDatabase::getIterateMaxLevel()
{
FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()");
int nLevel = levelData->getLevelNumber();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vector<std::set<int> > ranksInLevel;
ranksInLevel.resize(nLevel);
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
break;
default:
ERROR_EXIT("Should not happen!\n");
}
for (std::set<int>::iterator it = ranksInLevel[0].begin();
it != ranksInLevel[0].end(); ++it)
for (int level = 1; level < nLevel; level++)
ranksInLevel[level].insert(levelData->getLevelId(level, *it));
int maxLevel = 0;
for (int level = 1; level < nLevel; level++)
if (ranksInLevel[level].size() > 1)
maxLevel = level;
return maxLevel;
}
void ElementObjectDatabase::serialize(ostream &out)
{
FUNCNAME("ElementObjectDatabase::serialize()");
......@@ -686,11 +720,6 @@ namespace AMDiS {
ERROR_EXIT("REWRITE SERIALIZATION!\n");
/*
SerUtil::serialize(out, vertexOwner);
SerUtil::serialize(out, edgeOwner);
SerUtil::serialize(out, faceOwner);
*/
nSize = vertexInRank.size();
SerUtil::serialize(out, nSize);
......@@ -825,12 +854,7 @@ namespace AMDiS {
faceLocalMap[data] = face;
}
ERROR_EXIT("REWRITE DESERIALIZATION!\n");
SerUtil::deserialize(in, vertexOwner);
SerUtil::deserialize(in, edgeOwner);
SerUtil::deserialize(in, faceOwner);
SerUtil::deserialize(in, nSize);
vertexInRank.clear();
......
......@@ -105,7 +105,9 @@ namespace AMDiS {
ElementObjectDatabase()
: feSpace(NULL),
mesh(NULL),
iterGeoPos(CENTER)
iterGeoPos(CENTER),
macroElementRankMap(NULL),
levelData(NULL)
{}
void setFeSpace(const FiniteElemSpace *fe)
......@@ -119,7 +121,12 @@ namespace AMDiS {
return mesh;
}
void create();
/*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/
void create(map<int, int>& macroElementRankMap,
MeshLevelData& levelData);
void createMacroElementInfo(vector<MacroElement*> &mel);
......@@ -127,13 +134,8 @@ namespace AMDiS {
* Create for a filled object database the membership information for all
* element objects. An object is owned by a rank, if the rank has the
* heighest rank number of all ranks where the object is part of.
*
* \param[in] macroElementRankMap Maps to each macro element of the mesh
* the rank that owns this macro element.
*/
void createRankData(map<int, int>& macroElementRankMap,
MeshLevelData& levelData);
void updateRankData();
/** \brief
* Iterates over all elements for one geometrical index, i.e., over all
......@@ -227,67 +229,11 @@ namespace AMDiS {
}
}
/// Returns the rank owner of the current iterator position.
int getIterateOwner()
{
switch (iterGeoPos) {
case VERTEX:
return vertexOwner[vertexIter->first];
break;
case EDGE:
return edgeOwner[edgeIter->first];
break;
case FACE:
return faceOwner[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
int getIterateOwner();
/// Returns the rank owner of the current iterator position.
int getIterateMaxLevel()
{
switch (iterGeoPos) {
case VERTEX:
return vertexMaxLevel[vertexIter->first];
break;
case EDGE:
return edgeMaxLevel[edgeIter->first];
break;
case FACE:
return faceMaxLevel[faceIter->first];
break;
default:
ERROR_EXIT("Should not happen!\n");
// Will never be reached, just to avoid compiler warnings.
return -1;
}
}
/// Returns the rank owner of a vertex DOF.
int getOwner(DegreeOfFreedom vertex)
{
return vertexOwner[vertex];
}
/// Returns the rank owner of an edge.
int getOwner(DofEdge edge)
{
return edgeOwner[edge];
}
/// Returns the rank owner of an face.
int getOwner(DofFace face, int level)
{
return faceOwner[face];
}
int getIterateMaxLevel();
/// Checks if a given vertex DOF is in a given rank.
int isInRank(DegreeOfFreedom vertex, int rank)
......@@ -567,23 +513,7 @@ namespace AMDiS {
/// Maps to an element object the corresponding face.
map<ElementObjectData, DofFace> faceLocalMap;
/// Defines for all vertex DOFs the rank that ownes this vertex DOF.
map<DegreeOfFreedom, int> vertexOwner;
/// Defines for all edges the rank that ownes this edge.
map<DofEdge, int> edgeOwner;
/// Defines for all faces the rank that ownes this face.
map<DofFace, int> faceOwner;
map<DegreeOfFreedom, int> vertexMaxLevel;
map<DofEdge, int> edgeMaxLevel;
map<DofFace, int> faceMaxLevel;
/// Defines to each vertex DOF a map that maps to each rank number the element
/// objects that have this vertex DOF in common.
map<DegreeOfFreedom, map<int, ElementObjectData> > vertexInRank;
......@@ -636,11 +566,15 @@ namespace AMDiS {
map<pair<ElementObjectData, ElementObjectData>, bool> faceReverseMode;
map<int, int> *macroElementRankMap;
/// Maps to each macro element index a pointer to the corresponding element.
map<int, Element*> macroElIndexMap;
/// Maps to each macro element index the type of this element.
map<int, int> macroElIndexTypeMap;
MeshLevelData* levelData;
};
}
......
......@@ -19,28 +19,6 @@
namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewAtomicOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicOther(int rank)
{
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewAtomicPer(int rank)
{
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
void InteriorBoundary::create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb)
{
......@@ -70,7 +48,6 @@ namespace AMDiS {
AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex);
......@@ -101,7 +78,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOwn(it2->first);
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
if (geoIndex == EDGE)
b.neighObj.reverseMode =
......@@ -125,7 +102,7 @@ namespace AMDiS {
bound.type = INTERIOR;
AtomicBoundary& b = getNewAtomicOther(owner);
AtomicBoundary& b = getNewOther(owner);
b = bound;
if (geoIndex == EDGE)
b.rankObj.reverseMode =
......@@ -169,7 +146,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
}
......@@ -203,7 +180,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -249,7 +226,7 @@ namespace AMDiS {
bound.type = it->second;
AtomicBoundary& b = getNewAtomicPer(otherElementRank);
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (mpiRank > otherElementRank)
......@@ -465,6 +442,30 @@ namespace AMDiS {
}
AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewOther(int rank)
{
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
}
AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
{
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
void InteriorBoundary::serializeExcludeList(std::ostream &out,
ExcludeList &list)
{
......
......@@ -44,26 +44,47 @@ namespace AMDiS {
void create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb);
AtomicBoundary& getNewAtomicOwn(int rank);
AtomicBoundary& getNewAtomicOther(int rank);
AtomicBoundary& getNewAtomicPer(int rank);
/// Writes this object to a file.
void serialize(ostream &out);
/// Reads the state of an interior boundary from a file.
void deserialize(istream &in, map<int, Element*> &elIndexMap);
RankToBoundMap& getOwn()
{
return own;
}
RankToBoundMap& getOther()
{
return other;
}
RankToBoundMap& getPeriodic()
{
return periodic;
}
bool hasPeriodic()
{
return static_cast<bool>(periodic.size());
}
private:
AtomicBoundary& getNewOwn(int rank);
AtomicBoundary& getNewOther(int rank);
AtomicBoundary& getNewPeriodic(int rank);
void serializeExcludeList(ostream &out, ExcludeList &list);
void deserializeExcludeList(istream &in, ExcludeList &list);
public:
private:
RankToBoundMap own, other, periodic;
friend class ParallelDebug;
public:
/// Iterator for the interior boundary object.
......
......@@ -907,17 +907,19 @@ namespace AMDiS {
// important. Therefore, we add all boundaries to one boundary container.
RankToBoundMap allBound;
for (InteriorBoundary::iterator it(intBoundary.own); !it.end(); ++it)
for (InteriorBoundary::iterator it(intBoundary.getOwn()); !it.end(); ++it)
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE))
allBound[it.getRank()].push_back(*it);
for (InteriorBoundary::iterator it(intBoundary.other); !it.end(); ++it)
for (InteriorBoundary::iterator it(intBoundary.getOther());
!it.end(); ++it)
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE))
allBound[it.getRank()].push_back(*it);
for (InteriorBoundary::iterator it(intBoundary.periodic); !it.end(); ++it) {
for (InteriorBoundary::iterator it(intBoundary.getPeriodic());
!it.end(); ++it) {
if (it.getRank() == mpiRank) {
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE)) {
......@@ -1508,8 +1510,8 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
elObjDb.create();
elObjDb.createRankData(partitionMap, levelData);
elObjDb.create(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
}
......@@ -1518,7 +1520,7 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
elObjDb.createRankData(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
#if (DEBUG != 0)
......@@ -1535,10 +1537,10 @@ namespace AMDiS {
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");