Commit 7f81c570 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some issues for removing periodic boundary conditions.

parent 35eda406
...@@ -505,6 +505,7 @@ namespace AMDiS { ...@@ -505,6 +505,7 @@ namespace AMDiS {
{ {
FUNCNAME("ProblemStat::createSolver()"); FUNCNAME("ProblemStat::createSolver()");
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
// === create solver === // === create solver ===
string solverType("0"); string solverType("0");
string initFileStr = name + "->solver"; string initFileStr = name + "->solver";
...@@ -515,6 +516,7 @@ namespace AMDiS { ...@@ -515,6 +516,7 @@ namespace AMDiS {
solverCreator->setName(initFileStr); solverCreator->setName(initFileStr);
solver = solverCreator->create(); solver = solverCreator->create();
solver->initParameters(); solver->initParameters();
#endif
} }
......
...@@ -48,9 +48,7 @@ namespace AMDiS { ...@@ -48,9 +48,7 @@ namespace AMDiS {
DegreeOfFreedom **dof = macroInfo->dof; DegreeOfFreedom **dof = macroInfo->dof;
// === read periodic data ================================= // === read periodic data =================================
if (periodicFile != "") { if (periodicFile != "") {
WARNING("Periodic boundaries may lead to errors in small meshes if element neighbours are not set!\n");
FILE *file = fopen(periodicFile.c_str(), "r"); FILE *file = fopen(periodicFile.c_str(), "r");
TEST_EXIT(file)("can't open file %s\n", periodicFile.c_str()); TEST_EXIT(file)("can't open file %s\n", periodicFile.c_str());
......
...@@ -105,16 +105,15 @@ namespace AMDiS { ...@@ -105,16 +105,15 @@ namespace AMDiS {
periodicEdgeAssoc[edge0].insert(edge1); periodicEdgeAssoc[edge0].insert(edge1);
// Add both vertices of the edge to be periodic. // Add both vertices of the edge to be periodic.
periodicVertices[make_pair(edge0.first, edge1.first)] = boundaryType; DegreeOfFreedom mappedDof0 =
periodicVertices[make_pair(edge0.second, edge1.second)] = boundaryType; mesh->getPeriodicAssociations(boundaryType)[edge0.first];
DegreeOfFreedom mappedDof1 =
mesh->getPeriodicAssociations(boundaryType)[edge0.second];
periodicVertices[make_pair(edge0.first, mappedDof0)] = boundaryType;
periodicVertices[make_pair(edge0.second, mappedDof1)] = boundaryType;
periodicDofAssoc[edge0.first].insert(boundaryType); periodicDofAssoc[edge0.first].insert(boundaryType);
periodicDofAssoc[edge0.second].insert(boundaryType); periodicDofAssoc[edge0.second].insert(boundaryType);
TEST_EXIT_DBG(edge0.first ==
mesh->getPeriodicAssociations(boundaryType)[edge1.first] &&
edge0.second ==
mesh->getPeriodicAssociations(boundaryType)[edge1.second])
("Should not happen!\n");
} }
} }
break; break;
...@@ -134,12 +133,16 @@ namespace AMDiS { ...@@ -134,12 +133,16 @@ namespace AMDiS {
periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i); periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
/// Add all three vertices of the face to be periodic. /// Add all three vertices of the face to be periodic.
periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] = DegreeOfFreedom mappedDof0 =
boundaryType; mesh->getPeriodicAssociations(boundaryType)[face0.get<0>()];
periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] = DegreeOfFreedom mappedDof1 =
boundaryType; mesh->getPeriodicAssociations(boundaryType)[face0.get<1>()];
periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] = DegreeOfFreedom mappedDof2 =
boundaryType; mesh->getPeriodicAssociations(boundaryType)[face0.get<2>()];
periodicVertices[make_pair(face0.get<0>(), mappedDof0)] = boundaryType;
periodicVertices[make_pair(face0.get<1>(), mappedDof1)] = boundaryType;
periodicVertices[make_pair(face0.get<2>(), mappedDof2)] = boundaryType;
periodicDofAssoc[face0.get<0>()].insert(boundaryType); periodicDofAssoc[face0.get<0>()].insert(boundaryType);
periodicDofAssoc[face0.get<1>()].insert(boundaryType); periodicDofAssoc[face0.get<1>()].insert(boundaryType);
......
...@@ -309,24 +309,32 @@ namespace AMDiS { ...@@ -309,24 +309,32 @@ namespace AMDiS {
return faceElements[face]; return faceElements[face];
} }
/// Returns, for a given vertex, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// Returns a map that maps to each rank all macro elements in this rank that /// contains this vertex. If more than one element in one subdomain contains
/// have a given vertex DOF in common. /// the vertex, the element with the highest element index is given. If the
/// vertex is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DegreeOfFreedom vertex) map<int, ElementObjectData>& getElementsInRank(DegreeOfFreedom vertex)
{ {
return vertexInRank[vertex]; return vertexInRank[vertex];
} }
/// Returns a map that maps to each rank all macro elements in this rank that /// Returns, for a given edge, a map that maps from rank numbers to
/// have a given edge in common. /// element data objects, which identify on the rank the element which
/// contains this edge. If more than one element in one subdomain contains
/// the edge, the element with the highest element index is given. If the
/// edge is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofEdge edge) map<int, ElementObjectData>& getElementsInRank(DofEdge edge)
{ {
return edgeInRank[edge]; return edgeInRank[edge];
} }
/// Returns a map that maps to each rank all macro elements in this rank that /// Returns, for a given face, a map that maps from rank numbers to
/// have a given face in common. /// element data objects, which identify on the rank the element which
/// contains this face. If the face is not contained in a rank's subdomain,
/// it will not be considered in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofFace face) map<int, ElementObjectData>& getElementsInRank(DofFace face)
{ {
return faceInRank[face]; return faceInRank[face];
......
...@@ -110,7 +110,7 @@ namespace AMDiS { ...@@ -110,7 +110,7 @@ namespace AMDiS {
bound.neighObj.ithObj = it2->second.ithObject; bound.neighObj.ithObj = it2->second.ithObject;
bound.type = INTERIOR; bound.type = INTERIOR;
AtomicBoundary& b = getNewOwn(it2->first); AtomicBoundary& b = getNewOwn(it2->first);
b = bound; b = bound;
if (geoIndex == EDGE) if (geoIndex == EDGE)
...@@ -181,6 +181,7 @@ namespace AMDiS { ...@@ -181,6 +181,7 @@ namespace AMDiS {
bound.neighObj.subObj = VERTEX; bound.neighObj.subObj = VERTEX;
bound.neighObj.ithObj = perDofEl1.ithObject; bound.neighObj.ithObj = perDofEl1.ithObject;
if (removePeriodicBoundary) { if (removePeriodicBoundary) {
bound.type = INTERIOR; bound.type = INTERIOR;
...@@ -191,47 +192,56 @@ namespace AMDiS { ...@@ -191,47 +192,56 @@ namespace AMDiS {
int owner = std::max(elObjDb.getOwner(it->first.first, level), int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level)); elObjDb.getOwner(it->first.second, level));
// Consider the following configuration of four elements partitioned
// to four ranks:
//
// |--------|--------|(*)
// | /| /|
// | 3 / | 0 / |
// | / | / |
// | / | / |
// | / | / |
// | / 2 | / 1 |
// | / | / |
// |/-------|/-------|
//
// We are in now interested in vertex (*). For the first, rank 1 owns
// the interior boundary with rank 0 on this vertex. When we remove
// periodic boundaries, which are applied on the left and right outer
// boundary, rank 3 becomes the owner of this vertex. Rank 0 and rank 1
// will have an interior boundary with rank 3 on this vertex, but rank
// 0 and rank 1 have no more interior boundaries on vertex (*). Thus we
// have to search and remove for this scenario.
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
// And than we can add the boundary to either own or other part of the
// interior database.
if (owner == globalMpiRank) { if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin(); AtomicBoundary& b = getNewOwn(otherElementRank);
it2 != objData.end(); ++it2) { b = bound;
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
}
} else { } else {
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.ithObj = ownerBoundEl.ithObject;
AtomicBoundary& b = getNewOther(owner); AtomicBoundary& b = getNewOther(owner);
b = bound; b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
} }
} else { } else {
bound.type = it->second; bound.type = it->second;
if (MeshDistributor::sebastianMode) { AtomicBoundary& b = getNewPeriodic(otherElementRank);
if (bound.rankObj.elIndex == 3 && b = bound;
bound.rankObj.ithObj == 1 ||
bound.rankObj.elIndex == 78 &&
bound.rankObj.ithObj == 2) {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} else {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} }
} }
} }
...@@ -276,24 +286,15 @@ namespace AMDiS { ...@@ -276,24 +286,15 @@ namespace AMDiS {
elObjDb.getOwner(it->first.second, level)); elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank]; ElementObjectData& rankBoundEl = objData[globalMpiRank];
// See comments in the same part of code for VERTEX
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
if (owner == globalMpiRank) { if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin(); AtomicBoundary& b = getNewOwn(otherElementRank);
it2 != objData.end(); ++it2) { b = bound;
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
}
} else { } else {
AtomicBoundary& b = getNewOther(owner); AtomicBoundary& b = getNewOther(owner);
b = bound; b = bound;
...@@ -368,23 +369,8 @@ namespace AMDiS { ...@@ -368,23 +369,8 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[globalMpiRank]; ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) { if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin(); AtomicBoundary& b = getNewOwn(otherElementRank);
it2 != objData.end(); ++it2) { b = bound;
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
}
} else { } else {
AtomicBoundary& b = getNewOther(owner); AtomicBoundary& b = getNewOther(owner);
b = bound; b = bound;
...@@ -413,6 +399,7 @@ namespace AMDiS { ...@@ -413,6 +399,7 @@ namespace AMDiS {
} }
} }
// === Once we have this information, we must care about the order of the === // === Once we have this information, we must care about the order of the ===
// === atomic bounds in the three boundary handling object. Eventually === // === atomic bounds in the three boundary handling object. Eventually ===
// === all the boundaries have to be in the same order on both ranks that === // === all the boundaries have to be in the same order on both ranks that ===
...@@ -454,19 +441,25 @@ namespace AMDiS { ...@@ -454,19 +441,25 @@ namespace AMDiS {
// If the expected object is not at place, search for it. // If the expected object is not at place, search for it.
BoundaryObject &recvedBound = BoundaryObject &receivedBound =
stdMpi.getRecvData()[rank][j].rankObj; stdMpi.getRecvData()[rank][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) { if ((rankIt->second)[j].neighObj != receivedBound) {
unsigned int k = j + 1; unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++) for (; k < rankIt->second.size(); k++) {
if ((rankIt->second)[k].neighObj == recvedBound) if ((rankIt->second)[k].neighObj == receivedBound)
break; break;
}
// The element must always be found, because the list is just in // The element must always be found, because the list is just in
// another order. // another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); TEST_EXIT_DBG(k < rankIt->second.size())
("Cannot find element %d/%d/%d received from rank %d!\n",
receivedBound.elIndex,
receivedBound.subObj,
receivedBound.ithObj,
rank);
// Swap the current with the found element. // Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k]; AtomicBoundary tmpBound = (rankIt->second)[k];
...@@ -531,6 +524,11 @@ namespace AMDiS { ...@@ -531,6 +524,11 @@ namespace AMDiS {
} }
} }
} // periodicBoundary.boundary.size() > 0 } // periodicBoundary.boundary.size() > 0
// === Run verification procedure. ===
verifyBoundary();
} }
...@@ -644,8 +642,37 @@ namespace AMDiS { ...@@ -644,8 +642,37 @@ namespace AMDiS {
} }
void InteriorBoundary::verifyBoundary()
{
FUNCNAME("InteriorBoundary::verifyBoundary()");
// === Check if no other boundery rank object is also included in own ===
// === boundary part, which would make no sence. ===
std::set<BoundaryObject> allOwnBounds;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
allOwnBounds.insert(bIt->rankObj);
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
if (allOwnBounds.count(bIt->rankObj)) {
ERROR_EXIT("Boundary %d/%d/%d is included in both, own and other interior boundary part!\n",
bIt->rankObj.elIndex,
bIt->rankObj.subObj,
bIt->rankObj.ithObj);
}
}
AtomicBoundary& InteriorBoundary::getNewOwn(int rank) AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{ {
FUNCNAME("InteriorBoundary::getNewOwn()");
int size = own[rank].size(); int size = own[rank].size();
own[rank].resize(size + 1); own[rank].resize(size + 1);
return own[rank][size]; return own[rank][size];
...@@ -654,6 +681,8 @@ namespace AMDiS { ...@@ -654,6 +681,8 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewOther(int rank) AtomicBoundary& InteriorBoundary::getNewOther(int rank)
{ {
FUNCNAME("InteriorBoundary::getNewOther()");
int size = other[rank].size(); int size = other[rank].size();
other[rank].resize(size + 1); other[rank].resize(size + 1);
return other[rank][size]; return other[rank][size];
...@@ -662,12 +691,86 @@ namespace AMDiS { ...@@ -662,12 +691,86 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank) AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
{ {
FUNCNAME("InteriorBoundary::getNewPeriodic()");
int size = periodic[rank].size(); int size = periodic[rank].size();
periodic[rank].resize(size + 1); periodic[rank].resize(size + 1);
return periodic[rank][size]; return periodic[rank][size];
} }
bool InteriorBoundary::checkOther(AtomicBoundary& bound, int rank)
{
FUNCNAME("InteriorBoundary::checkOther()");
int globalMpiRank = MPI::COMM_WORLD.Get_rank();
TEST_EXIT(rank > globalMpiRank)
("Wrong MPI ranks: %d %d\n", rank, globalMpiRank);
bool found = false;
if (other.count(rank)) {
vector<AtomicBoundary>& otherBounds = other[rank];
for (int i = 0; i < static_cast<int>(otherBounds.size()); i++) {
if (otherBounds[i].rankObj == bound.rankObj) {
if (otherBounds[i].neighObj != bound.neighObj) {
ERROR_EXIT("If this really can happen, I have to thing about this!\n");
} else {
found = true;
break;
}
}
}
}
return found;
}
bool InteriorBoundary::removeOwn(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOwn()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
bool InteriorBoundary::removeOther(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOther()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
void InteriorBoundary::serializeExcludeList(ostream &out, void InteriorBoundary::serializeExcludeList(ostream &out,
ExcludeList &list) ExcludeList &list)
{ {
......
...@@ -86,12 +86,32 @@ namespace AMDiS { ...@@ -86,12 +86,32 @@ namespace AMDiS {
void deserialize(istream &in, Mesh *mesh); void deserialize(istream &in, Mesh *mesh);
private: private:
/// In this function, we put some verification procedures to check for
/// consistency if the created interior boundary data. This function is
/// enabled even for optimized mode to check all possible meshes. Thus,
/// everything herein should be fast. If the code is stable, we can think
/// to remove this function.
void verifyBoundary();
AtomicBoundary& getNewOwn(int rank); AtomicBoundary& getNewOwn(int rank);
AtomicBoundary& getNewOther(int rank); AtomicBoundary& getNewOther(int rank);
AtomicBoundary& getNewPeriodic(int rank); AtomicBoundary& getNewPeriodic(int rank);
/// Checks whether the given boundary is already a other boundary with
/// given rank number. Returns true, if the bound is already in the other
/// boundary database.
bool checkOther(AtomicBoundary& bound, int rank);