Commit 64e91ba5 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed periodic boundaries for FETI-DP

parent 8930cc64
...@@ -43,10 +43,7 @@ namespace AMDiS { ...@@ -43,10 +43,7 @@ namespace AMDiS {
} }
// Handle periodic boudaries // Handle periodic boudaries
if (removePeriodicBoundary == false) createPeriodicData();
createPeriodicData();
else
removePeriodicData();
// Create data about the reverse modes of neighbouring elements. // Create data about the reverse modes of neighbouring elements.
createReverseModeData(); createReverseModeData();
...@@ -183,13 +180,7 @@ namespace AMDiS { ...@@ -183,13 +180,7 @@ namespace AMDiS {
DegreeOfFreedom vertex = el->getDof(ith, 0); DegreeOfFreedom vertex = el->getDof(ith, 0);
int elIndex = el->getIndex(); int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith); ElementObjectData elObj(elIndex, ith);
if (elIndex == 53 && ith == 0)
MSG("A: 53/0 ON DOF %d\n", vertex);
if (elIndex == 229 && ith == 0)
MSG("A: 229/0 ON DOF %d\n", vertex);
vertexElements[vertex].push_back(elObj); vertexElements[vertex].push_back(elObj);
vertexLocalMap[elObj] = vertex; vertexLocalMap[elObj] = vertex;
} }
...@@ -412,10 +403,6 @@ namespace AMDiS { ...@@ -412,10 +403,6 @@ namespace AMDiS {
} }
void ElementObjectDatabase::removePeriodicData()
{
}
BoundaryType ElementObjectDatabase::getNewBoundaryType() BoundaryType ElementObjectDatabase::getNewBoundaryType()
{ {
FUNCNAME("ElementObjectDatabase::getNewBoundaryType()"); FUNCNAME("ElementObjectDatabase::getNewBoundaryType()");
...@@ -626,26 +613,51 @@ void ElementObjectDatabase::removePeriodicData() ...@@ -626,26 +613,51 @@ void ElementObjectDatabase::removePeriodicData()
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n"); TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
vector<ElementObjectData> *objData;
switch (iterGeoPos) { switch (iterGeoPos) {
case VERTEX: case VERTEX:
objData = &(vertexElements[vertexIter->first]); return getOwner(vertexElements[vertexIter->first], level);
break; break;
case EDGE: case EDGE:
objData = &(edgeElements[edgeIter->first]); return getOwner(edgeElements[edgeIter->first], level);
break; break;
case FACE: case FACE:
objData = &(faceElements[faceIter->first]); return getOwner(faceElements[faceIter->first], level);
break; break;
} }
ERROR_EXIT("There is something reallllly wrong!\n");
return -1;
}
int ElementObjectDatabase::getOwner(DegreeOfFreedom vertex, int level)
{
return getOwner(vertexElements[vertex], level);
}
int ElementObjectDatabase::getOwner(DofEdge edge, int level)
{
return getOwner(edgeElements[edge], level);
}
int ElementObjectDatabase::getOwner(DofFace face, int level)
{
return getOwner(faceElements[face], level);
}
int ElementObjectDatabase::getOwner(vector<ElementObjectData>& objData,
int level)
{
int owner = -1;
std::set<int> &levelRanks = levelData->getLevelRanks(level); std::set<int> &levelRanks = levelData->getLevelRanks(level);
bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
for (vector<ElementObjectData>::iterator it = objData->begin(); for (vector<ElementObjectData>::iterator it = objData.begin();
it != objData->end(); ++it) { it != objData.end(); ++it) {
int elRank = (*macroElementRankMap)[it->elIndex]; int elRank = (*macroElementRankMap)[it->elIndex];
if (allRanks || levelRanks.count(elRank)) if (allRanks || levelRanks.count(elRank))
owner = std::max(owner, elRank); owner = std::max(owner, elRank);
......
...@@ -51,8 +51,7 @@ namespace AMDiS { ...@@ -51,8 +51,7 @@ namespace AMDiS {
struct ElementObjectData { struct ElementObjectData {
ElementObjectData(int a = -1, int b = 0) ElementObjectData(int a = -1, int b = 0)
: elIndex(a), : elIndex(a),
ithObject(b), ithObject(b)
mappedOnPeriodicBoundary(false)
{} {}
/// Index of the element this object is part of. /// Index of the element this object is part of.
...@@ -61,16 +60,11 @@ namespace AMDiS { ...@@ -61,16 +60,11 @@ namespace AMDiS {
/// Index of the object within the element. /// Index of the object within the element.
int ithObject; int ithObject;
/// If true, the element does not exists in mesh but is due to a mapping
/// on a periodic boundary.
bool mappedOnPeriodicBoundary;
/// Write this element object to disk. /// Write this element object to disk.
void serialize(ostream &out) const void serialize(ostream &out) const
{ {
SerUtil::serialize(out, elIndex); SerUtil::serialize(out, elIndex);
SerUtil::serialize(out, ithObject); SerUtil::serialize(out, ithObject);
SerUtil::serialize(out, mappedOnPeriodicBoundary);
} }
/// Read this element object from disk. /// Read this element object from disk.
...@@ -78,7 +72,6 @@ namespace AMDiS { ...@@ -78,7 +72,6 @@ namespace AMDiS {
{ {
SerUtil::deserialize(in, elIndex); SerUtil::deserialize(in, elIndex);
SerUtil::deserialize(in, ithObject); SerUtil::deserialize(in, ithObject);
SerUtil::deserialize(in, mappedOnPeriodicBoundary);
} }
/// Compare this element object with another one. /// Compare this element object with another one.
...@@ -114,11 +107,8 @@ namespace AMDiS { ...@@ -114,11 +107,8 @@ namespace AMDiS {
mesh(NULL), mesh(NULL),
iterGeoPos(CENTER), iterGeoPos(CENTER),
macroElementRankMap(NULL), macroElementRankMap(NULL),
levelData(NULL), levelData(NULL)
removePeriodicBoundary(false) {}
{
Parameters::get("parallel->remove periodic boundary", removePeriodicBoundary);
}
void setFeSpace(const FiniteElemSpace *fe) void setFeSpace(const FiniteElemSpace *fe)
{ {
...@@ -242,6 +232,15 @@ namespace AMDiS { ...@@ -242,6 +232,15 @@ namespace AMDiS {
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateOwner(int level); int getIterateOwner(int level);
/// Returns the owner of a macro element vertex.
int getOwner(DegreeOfFreedom vertex, int level);
/// Returns the owner of a macro element edge.
int getOwner(DofEdge edge, int level);
/// Returns the owner of a macro element face.
int getOwner(DofFace face, int level);
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateMaxLevel(); int getIterateMaxLevel();
...@@ -452,8 +451,6 @@ namespace AMDiS { ...@@ -452,8 +451,6 @@ namespace AMDiS {
*/ */
void createPeriodicData(); void createPeriodicData();
void removePeriodicData();
/// Creates on all boundaries the reverse mode flag. /// Creates on all boundaries the reverse mode flag.
void createReverseModeData(); void createReverseModeData();
...@@ -474,6 +471,7 @@ namespace AMDiS { ...@@ -474,6 +471,7 @@ namespace AMDiS {
/// Some auxiliary function to read the element object database from disk. /// Some auxiliary function to read the element object database from disk.
void deserialize(istream &in, map<int, ElementObjectData>& data); void deserialize(istream &in, map<int, ElementObjectData>& data);
int getOwner(vector<ElementObjectData>& objData, int level);
private: private:
const FiniteElemSpace* feSpace; const FiniteElemSpace* feSpace;
...@@ -562,12 +560,6 @@ namespace AMDiS { ...@@ -562,12 +560,6 @@ namespace AMDiS {
map<int, int> macroElIndexTypeMap; map<int, int> macroElIndexTypeMap;
MeshLevelData* levelData; MeshLevelData* levelData;
/// If this variable is set to true, the mesh distributor removes all
/// periodic boundary conditions. The element neighbourhood relation is
/// not changed. Thus, when using some domain decomposition method, this is
/// a natural way to deal with periodic boundary conditions.
bool removePeriodicBoundary;
}; };
} }
......
...@@ -77,9 +77,6 @@ namespace AMDiS { ...@@ -77,9 +77,6 @@ namespace AMDiS {
int owner = elObjDb.getIterateOwner(level); int owner = elObjDb.getIterateOwner(level);
ElementObjectData& rankBoundEl = objData[globalMpiRank]; ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (rankBoundEl.mappedOnPeriodicBoundary)
continue;
AtomicBoundary bound; AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel(); bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex); bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
...@@ -131,13 +128,7 @@ namespace AMDiS { ...@@ -131,13 +128,7 @@ namespace AMDiS {
("Should not happen!\n"); ("Should not happen!\n");
ElementObjectData& ownerBoundEl = objData[owner]; ElementObjectData& ownerBoundEl = objData[owner];
if (rankBoundEl.elIndex == 53 &&
geoIndex == 1 &&
rankBoundEl.ithObject == 0) {
MSG("OWNER: %d %d\n", owner, ownerBoundEl);
}
bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex); bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
bound.neighObj.elIndex = ownerBoundEl.elIndex; bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = -1;
...@@ -161,6 +152,10 @@ namespace AMDiS { ...@@ -161,6 +152,10 @@ namespace AMDiS {
// === Create periodic boundary data structure. === // === Create periodic boundary data structure. ===
int removePeriodicBoundary = 0;
Parameters::get("parallel->remove periodic boundary", removePeriodicBoundary);
for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjDb.getPeriodicVertices().begin(); for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjDb.getPeriodicVertices().begin();
it != elObjDb.getPeriodicVertices().end(); ++it) { it != elObjDb.getPeriodicVertices().end(); ++it) {
if (elObjDb.isInRank(it->first.first, globalMpiRank) == false) if (elObjDb.isInRank(it->first.first, globalMpiRank) == false)
...@@ -188,19 +183,57 @@ namespace AMDiS { ...@@ -188,19 +183,57 @@ namespace AMDiS {
bound.neighObj.subObj = VERTEX; bound.neighObj.subObj = VERTEX;
bound.neighObj.ithObj = perDofEl1.ithObject; bound.neighObj.ithObj = perDofEl1.ithObject;
bound.type = it->second; if (removePeriodicBoundary) {
bound.type = INTERIOR;
if (MeshDistributor::sebastianMode) {
if (bound.rankObj.elIndex == 3 && map<int, ElementObjectData> objData =
bound.rankObj.ithObj == 1 || elObjDb.getElementsInRank(it->first.first);
bound.rankObj.elIndex == 78 && objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
bound.rankObj.ithObj == 2) { elObjDb.getElementsInRank(it->first.second).end());
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound; int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
} }
} else { } else {
AtomicBoundary& b = getNewPeriodic(otherElementRank); bound.type = it->second;
b = bound;
if (MeshDistributor::sebastianMode) {
if (bound.rankObj.elIndex == 3 &&
bound.rankObj.ithObj == 1 ||
bound.rankObj.elIndex == 78 &&
bound.rankObj.ithObj == 2) {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} else {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} }
} }
} }
...@@ -232,18 +265,62 @@ namespace AMDiS { ...@@ -232,18 +265,62 @@ namespace AMDiS {
bound.neighObj.elType = elObjDb.getElementType(perEdgeEl1.elIndex); bound.neighObj.elType = elObjDb.getElementType(perEdgeEl1.elIndex);
bound.neighObj.subObj = EDGE; bound.neighObj.subObj = EDGE;
bound.neighObj.ithObj = perEdgeEl1.ithObject; bound.neighObj.ithObj = perEdgeEl1.ithObject;
bound.type = it->second;
AtomicBoundary& b = getNewPeriodic(otherElementRank); if (removePeriodicBoundary) {
b = bound; bound.type = INTERIOR;
if (globalMpiRank > otherElementRank) map<int, ElementObjectData> objData =
b.neighObj.reverseMode = elObjDb.getElementsInRank(it->first.first);
elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
else elObjDb.getElementsInRank(it->first.second).end());
b.rankObj.reverseMode =
elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
b.rankObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, ownerBoundEl);
}
} else {
bound.type = it->second;
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (globalMpiRank > otherElementRank)
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1);
else
b.rankObj.reverseMode =
elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1);
}
} }
} }
...@@ -280,17 +357,61 @@ namespace AMDiS { ...@@ -280,17 +357,61 @@ namespace AMDiS {
bound.neighObj.subObj = FACE; bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = perFaceEl1.ithObject; bound.neighObj.ithObj = perFaceEl1.ithObject;
bound.type = it->second; if (removePeriodicBoundary) {
bound.type = INTERIOR;
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound; map<int, ElementObjectData> objData =
elObjDb.getElementsInRank(it->first.first);
if (globalMpiRank > otherElementRank) objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
b.neighObj.reverseMode = elObjDb.getElementsInRank(it->first.second).end());
elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1);
else int owner = std::max(elObjDb.getOwner(it->first.first, level),
b.rankObj.reverseMode = elObjDb.getOwner(it->first.second, level));
elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
b.rankObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, ownerBoundEl);
}
} else {
bound.type = it->second;
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
if (globalMpiRank > otherElementRank)
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1);
else
b.rankObj.reverseMode =
elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1);
}
} }
} }
...@@ -358,6 +479,8 @@ namespace AMDiS { ...@@ -358,6 +479,8 @@ namespace AMDiS {
} }
#if 0
// === Do the same for the periodic boundaries. === // === Do the same for the periodic boundaries. ===
if (periodic.size() > 0) { if (periodic.size() > 0) {
...@@ -412,6 +535,8 @@ namespace AMDiS { ...@@ -412,6 +535,8 @@ namespace AMDiS {
} }
} }
} // periodicBoundary.boundary.size() > 0 } // periodicBoundary.boundary.size() > 0
#endif
} }
......
...@@ -445,9 +445,6 @@ namespace AMDiS { ...@@ -445,9 +445,6 @@ namespace AMDiS {
it != vertices.end(); ++it) { it != vertices.end(); ++it) {
double e = 1e-8; double e = 1e-8;
if (meshLevel == 0) { if (meshLevel == 0) {
WorldVector<double> c;
feSpace->getMesh()->getDofIndexCoords(*it, feSpace, c);
MSG("PRIMAL COORD %f %f\n", c[0], c[1]);
primals.insert(**it); primals.insert(**it);
} else { } else {
WorldVector<double> c; WorldVector<double> c;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment