Commit 64e91ba5 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed periodic boundaries for FETI-DP

parent 8930cc64
...@@ -43,10 +43,7 @@ namespace AMDiS { ...@@ -43,10 +43,7 @@ namespace AMDiS {
} }
// Handle periodic boudaries // Handle periodic boudaries
if (removePeriodicBoundary == false)
createPeriodicData(); createPeriodicData();
else
removePeriodicData();
// Create data about the reverse modes of neighbouring elements. // Create data about the reverse modes of neighbouring elements.
createReverseModeData(); createReverseModeData();
...@@ -184,12 +181,6 @@ namespace AMDiS { ...@@ -184,12 +181,6 @@ namespace AMDiS {
int elIndex = el->getIndex(); int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith); ElementObjectData elObj(elIndex, ith);
if (elIndex == 53 && ith == 0)
MSG("A: 53/0 ON DOF %d\n", vertex);
if (elIndex == 229 && ith == 0)
MSG("A: 229/0 ON DOF %d\n", vertex);
vertexElements[vertex].push_back(elObj); vertexElements[vertex].push_back(elObj);
vertexLocalMap[elObj] = vertex; vertexLocalMap[elObj] = vertex;
} }
...@@ -412,10 +403,6 @@ namespace AMDiS { ...@@ -412,10 +403,6 @@ namespace AMDiS {
} }
void ElementObjectDatabase::removePeriodicData()
{
}
BoundaryType ElementObjectDatabase::getNewBoundaryType() BoundaryType ElementObjectDatabase::getNewBoundaryType()
{ {
FUNCNAME("ElementObjectDatabase::getNewBoundaryType()"); FUNCNAME("ElementObjectDatabase::getNewBoundaryType()");
...@@ -626,26 +613,51 @@ void ElementObjectDatabase::removePeriodicData() ...@@ -626,26 +613,51 @@ void ElementObjectDatabase::removePeriodicData()
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n"); TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
vector<ElementObjectData> *objData;
switch (iterGeoPos) { switch (iterGeoPos) {
case VERTEX: case VERTEX:
objData = &(vertexElements[vertexIter->first]); return getOwner(vertexElements[vertexIter->first], level);
break; break;
case EDGE: case EDGE:
objData = &(edgeElements[edgeIter->first]); return getOwner(edgeElements[edgeIter->first], level);
break; break;
case FACE: case FACE:
objData = &(faceElements[faceIter->first]); return getOwner(faceElements[faceIter->first], level);
break; break;
} }
ERROR_EXIT("There is something reallllly wrong!\n");
return -1;
}
int ElementObjectDatabase::getOwner(DegreeOfFreedom vertex, int level)
{
return getOwner(vertexElements[vertex], level);
}
int ElementObjectDatabase::getOwner(DofEdge edge, int level)
{
return getOwner(edgeElements[edge], level);
}
int ElementObjectDatabase::getOwner(DofFace face, int level)
{
return getOwner(faceElements[face], level);
}
int ElementObjectDatabase::getOwner(vector<ElementObjectData>& objData,
int level)
{
int owner = -1;
std::set<int> &levelRanks = levelData->getLevelRanks(level); std::set<int> &levelRanks = levelData->getLevelRanks(level);
bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
for (vector<ElementObjectData>::iterator it = objData->begin(); for (vector<ElementObjectData>::iterator it = objData.begin();
it != objData->end(); ++it) { it != objData.end(); ++it) {
int elRank = (*macroElementRankMap)[it->elIndex]; int elRank = (*macroElementRankMap)[it->elIndex];
if (allRanks || levelRanks.count(elRank)) if (allRanks || levelRanks.count(elRank))
owner = std::max(owner, elRank); owner = std::max(owner, elRank);
......
...@@ -51,8 +51,7 @@ namespace AMDiS { ...@@ -51,8 +51,7 @@ namespace AMDiS {
struct ElementObjectData { struct ElementObjectData {
ElementObjectData(int a = -1, int b = 0) ElementObjectData(int a = -1, int b = 0)
: elIndex(a), : elIndex(a),
ithObject(b), ithObject(b)
mappedOnPeriodicBoundary(false)
{} {}
/// Index of the element this object is part of. /// Index of the element this object is part of.
...@@ -61,16 +60,11 @@ namespace AMDiS { ...@@ -61,16 +60,11 @@ namespace AMDiS {
/// Index of the object within the element. /// Index of the object within the element.
int ithObject; int ithObject;
/// If true, the element does not exists in mesh but is due to a mapping
/// on a periodic boundary.
bool mappedOnPeriodicBoundary;
/// Write this element object to disk. /// Write this element object to disk.
void serialize(ostream &out) const void serialize(ostream &out) const
{ {
SerUtil::serialize(out, elIndex); SerUtil::serialize(out, elIndex);
SerUtil::serialize(out, ithObject); SerUtil::serialize(out, ithObject);
SerUtil::serialize(out, mappedOnPeriodicBoundary);
} }
/// Read this element object from disk. /// Read this element object from disk.
...@@ -78,7 +72,6 @@ namespace AMDiS { ...@@ -78,7 +72,6 @@ namespace AMDiS {
{ {
SerUtil::deserialize(in, elIndex); SerUtil::deserialize(in, elIndex);
SerUtil::deserialize(in, ithObject); SerUtil::deserialize(in, ithObject);
SerUtil::deserialize(in, mappedOnPeriodicBoundary);
} }
/// Compare this element object with another one. /// Compare this element object with another one.
...@@ -114,11 +107,8 @@ namespace AMDiS { ...@@ -114,11 +107,8 @@ namespace AMDiS {
mesh(NULL), mesh(NULL),
iterGeoPos(CENTER), iterGeoPos(CENTER),
macroElementRankMap(NULL), macroElementRankMap(NULL),
levelData(NULL), levelData(NULL)
removePeriodicBoundary(false) {}
{
Parameters::get("parallel->remove periodic boundary", removePeriodicBoundary);
}
void setFeSpace(const FiniteElemSpace *fe) void setFeSpace(const FiniteElemSpace *fe)
{ {
...@@ -242,6 +232,15 @@ namespace AMDiS { ...@@ -242,6 +232,15 @@ namespace AMDiS {
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateOwner(int level); int getIterateOwner(int level);
/// Returns the owner of a macro element vertex.
int getOwner(DegreeOfFreedom vertex, int level);
/// Returns the owner of a macro element edge.
int getOwner(DofEdge edge, int level);
/// Returns the owner of a macro element face.
int getOwner(DofFace face, int level);
/// Returns the rank owner of the current iterator position. /// Returns the rank owner of the current iterator position.
int getIterateMaxLevel(); int getIterateMaxLevel();
...@@ -452,8 +451,6 @@ namespace AMDiS { ...@@ -452,8 +451,6 @@ namespace AMDiS {
*/ */
void createPeriodicData(); void createPeriodicData();
void removePeriodicData();
/// Creates on all boundaries the reverse mode flag. /// Creates on all boundaries the reverse mode flag.
void createReverseModeData(); void createReverseModeData();
...@@ -474,6 +471,7 @@ namespace AMDiS { ...@@ -474,6 +471,7 @@ namespace AMDiS {
/// Some auxiliary function to read the element object database from disk. /// Some auxiliary function to read the element object database from disk.
void deserialize(istream &in, map<int, ElementObjectData>& data); void deserialize(istream &in, map<int, ElementObjectData>& data);
int getOwner(vector<ElementObjectData>& objData, int level);
private: private:
const FiniteElemSpace* feSpace; const FiniteElemSpace* feSpace;
...@@ -562,12 +560,6 @@ namespace AMDiS { ...@@ -562,12 +560,6 @@ namespace AMDiS {
map<int, int> macroElIndexTypeMap; map<int, int> macroElIndexTypeMap;
MeshLevelData* levelData; MeshLevelData* levelData;
/// If this variable is set to true, the mesh distributor removes all
/// periodic boundary conditions. The element neighbourhood relation is
/// not changed. Thus, when using some domain decomposition method, this is
/// a natural way to deal with periodic boundary conditions.
bool removePeriodicBoundary;
}; };
} }
......
...@@ -77,9 +77,6 @@ namespace AMDiS { ...@@ -77,9 +77,6 @@ namespace AMDiS {
int owner = elObjDb.getIterateOwner(level); int owner = elObjDb.getIterateOwner(level);
ElementObjectData& rankBoundEl = objData[globalMpiRank]; ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (rankBoundEl.mappedOnPeriodicBoundary)
continue;
AtomicBoundary bound; AtomicBoundary bound;
bound.maxLevel = elObjDb.getIterateMaxLevel(); bound.maxLevel = elObjDb.getIterateMaxLevel();
bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex); bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
...@@ -132,12 +129,6 @@ namespace AMDiS { ...@@ -132,12 +129,6 @@ namespace AMDiS {
ElementObjectData& ownerBoundEl = objData[owner]; ElementObjectData& ownerBoundEl = objData[owner];
if (rankBoundEl.elIndex == 53 &&
geoIndex == 1 &&
rankBoundEl.ithObject == 0) {
MSG("OWNER: %d %d\n", owner, ownerBoundEl);
}
bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex); bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
bound.neighObj.elIndex = ownerBoundEl.elIndex; bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = -1;
...@@ -161,6 +152,10 @@ namespace AMDiS { ...@@ -161,6 +152,10 @@ namespace AMDiS {
// === Create periodic boundary data structure. === // === Create periodic boundary data structure. ===
int removePeriodicBoundary = 0;
Parameters::get("parallel->remove periodic boundary", removePeriodicBoundary);
for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjDb.getPeriodicVertices().begin(); for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjDb.getPeriodicVertices().begin();
it != elObjDb.getPeriodicVertices().end(); ++it) { it != elObjDb.getPeriodicVertices().end(); ++it) {
if (elObjDb.isInRank(it->first.first, globalMpiRank) == false) if (elObjDb.isInRank(it->first.first, globalMpiRank) == false)
...@@ -188,6 +183,43 @@ namespace AMDiS { ...@@ -188,6 +183,43 @@ namespace AMDiS {
bound.neighObj.subObj = VERTEX; bound.neighObj.subObj = VERTEX;
bound.neighObj.ithObj = perDofEl1.ithObject; bound.neighObj.ithObj = perDofEl1.ithObject;
if (removePeriodicBoundary) {
bound.type = INTERIOR;
map<int, ElementObjectData> objData =
elObjDb.getElementsInRank(it->first.first);
objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
elObjDb.getElementsInRank(it->first.second).end());
int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
}
} else {
bound.type = it->second; bound.type = it->second;
if (MeshDistributor::sebastianMode) { if (MeshDistributor::sebastianMode) {
...@@ -204,6 +236,7 @@ namespace AMDiS { ...@@ -204,6 +236,7 @@ namespace AMDiS {
} }
} }
} }
}
for (PerBoundMap<DofEdge>::iterator it = elObjDb.getPeriodicEdges().begin(); for (PerBoundMap<DofEdge>::iterator it = elObjDb.getPeriodicEdges().begin();
...@@ -233,6 +266,49 @@ namespace AMDiS { ...@@ -233,6 +266,49 @@ namespace AMDiS {
bound.neighObj.subObj = EDGE; bound.neighObj.subObj = EDGE;
bound.neighObj.ithObj = perEdgeEl1.ithObject; bound.neighObj.ithObj = perEdgeEl1.ithObject;
if (removePeriodicBoundary) {
bound.type = INTERIOR;
map<int, ElementObjectData> objData =
elObjDb.getElementsInRank(it->first.first);
objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
elObjDb.getElementsInRank(it->first.second).end());
int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
b.rankObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, ownerBoundEl);
}
} else {
bound.type = it->second; bound.type = it->second;
AtomicBoundary& b = getNewPeriodic(otherElementRank); AtomicBoundary& b = getNewPeriodic(otherElementRank);
...@@ -246,6 +322,7 @@ namespace AMDiS { ...@@ -246,6 +322,7 @@ namespace AMDiS {
elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1);
} }
} }
}
for (PerBoundMap<DofFace>::iterator it = elObjDb.getPeriodicFaces().begin(); for (PerBoundMap<DofFace>::iterator it = elObjDb.getPeriodicFaces().begin();
...@@ -280,6 +357,49 @@ namespace AMDiS { ...@@ -280,6 +357,49 @@ namespace AMDiS {
bound.neighObj.subObj = FACE; bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = perFaceEl1.ithObject; bound.neighObj.ithObj = perFaceEl1.ithObject;
if (removePeriodicBoundary) {
bound.type = INTERIOR;
map<int, ElementObjectData> objData =
elObjDb.getElementsInRank(it->first.first);
objData.insert(elObjDb.getElementsInRank(it->first.second).begin(),
elObjDb.getElementsInRank(it->first.second).end());
int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
}
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
b.rankObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, ownerBoundEl);
}
} else {
bound.type = it->second; bound.type = it->second;
AtomicBoundary& b = getNewPeriodic(otherElementRank); AtomicBoundary& b = getNewPeriodic(otherElementRank);
...@@ -293,6 +413,7 @@ namespace AMDiS { ...@@ -293,6 +413,7 @@ namespace AMDiS {
elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1);
} }
} }
}
// === Once we have this information, we must care about the order of the === // === Once we have this information, we must care about the order of the ===
// === atomic bounds in the three boundary handling object. Eventually === // === atomic bounds in the three boundary handling object. Eventually ===
...@@ -358,6 +479,8 @@ namespace AMDiS { ...@@ -358,6 +479,8 @@ namespace AMDiS {
} }
#if 0
// === Do the same for the periodic boundaries. === // === Do the same for the periodic boundaries. ===
if (periodic.size() > 0) { if (periodic.size() > 0) {
...@@ -412,6 +535,8 @@ namespace AMDiS { ...@@ -412,6 +535,8 @@ namespace AMDiS {
} }
} }
} // periodicBoundary.boundary.size() > 0 } // periodicBoundary.boundary.size() > 0
#endif
} }
......
...@@ -445,9 +445,6 @@ namespace AMDiS { ...@@ -445,9 +445,6 @@ namespace AMDiS {
it != vertices.end(); ++it) { it != vertices.end(); ++it) {
double e = 1e-8; double e = 1e-8;
if (meshLevel == 0) { if (meshLevel == 0) {
WorldVector<double> c;
feSpace->getMesh()->getDofIndexCoords(*it, feSpace, c);
MSG("PRIMAL COORD %f %f\n", c[0], c[1]);
primals.insert(**it); primals.insert(**it);
} else { } else {
WorldVector<double> c; WorldVector<double> c;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment