Skip to content
GitLab
Commit 7f81c570 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some issues for removing periodic boundary conditions.

parent 35eda406
Hide whitespace changes
Inline Side-by-side
AMDiS/src/ProblemStat.cc
View file @ 7f81c570
......@@ -505,6 +505,7 @@ namespace AMDiS {
{
FUNCNAME("ProblemStat::createSolver()");
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
// === create solver ===
string solverType("0");
string initFileStr = name + "->solver";
......@@ -515,6 +516,7 @@ namespace AMDiS {
solverCreator->setName(initFileStr);
solver = solverCreator->create();
solver->initParameters();
#endif
}
......
AMDiS/src/io/MacroReader.cc
View file @ 7f81c570
......@@ -48,9 +48,7 @@ namespace AMDiS {
DegreeOfFreedom **dof = macroInfo->dof;
// === read periodic data =================================
if (periodicFile != "") {
WARNING("Periodic boundaries may lead to errors in small meshes if element neighbours are not set!\n");
if (periodicFile != "") {
FILE *file = fopen(periodicFile.c_str(), "r");
TEST_EXIT(file)("can't open file %s\n", periodicFile.c_str());
......
AMDiS/src/parallel/ElementObjectDatabase.cc
View file @ 7f81c570
......@@ -105,16 +105,15 @@ namespace AMDiS {
periodicEdgeAssoc[edge0].insert(edge1);
// Add both vertices of the edge to be periodic.
periodicVertices[make_pair(edge0.first, edge1.first)] = boundaryType;
periodicVertices[make_pair(edge0.second, edge1.second)] = boundaryType;
DegreeOfFreedom mappedDof0 =
mesh->getPeriodicAssociations(boundaryType)[edge0.first];
DegreeOfFreedom mappedDof1 =
mesh->getPeriodicAssociations(boundaryType)[edge0.second];
periodicVertices[make_pair(edge0.first, mappedDof0)] = boundaryType;
periodicVertices[make_pair(edge0.second, mappedDof1)] = boundaryType;
periodicDofAssoc[edge0.first].insert(boundaryType);
periodicDofAssoc[edge0.second].insert(boundaryType);
TEST_EXIT_DBG(edge0.first ==
mesh->getPeriodicAssociations(boundaryType)[edge1.first] &&
edge0.second ==
mesh->getPeriodicAssociations(boundaryType)[edge1.second])
("Should not happen!\n");
periodicDofAssoc[edge0.second].insert(boundaryType);
}
}
break;
......@@ -134,12 +133,16 @@ namespace AMDiS {
periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
/// Add all three vertices of the face to be periodic.
periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] =
boundaryType;
periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] =
boundaryType;
periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] =
boundaryType;
DegreeOfFreedom mappedDof0 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<0>()];
DegreeOfFreedom mappedDof1 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<1>()];
DegreeOfFreedom mappedDof2 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<2>()];
periodicVertices[make_pair(face0.get<0>(), mappedDof0)] = boundaryType;
periodicVertices[make_pair(face0.get<1>(), mappedDof1)] = boundaryType;
periodicVertices[make_pair(face0.get<2>(), mappedDof2)] = boundaryType;
periodicDofAssoc[face0.get<0>()].insert(boundaryType);
periodicDofAssoc[face0.get<1>()].insert(boundaryType);
......
AMDiS/src/parallel/ElementObjectDatabase.h
View file @ 7f81c570
......@@ -309,24 +309,32 @@ namespace AMDiS {
return faceElements[face];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given vertex DOF in common.
/// Returns, for a given vertex, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this vertex. If more than one element in one subdomain contains
/// the vertex, the element with the highest element index is given. If the
/// vertex is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DegreeOfFreedom vertex)
{
return vertexInRank[vertex];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given edge in common.
/// Returns, for a given edge, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this edge. If more than one element in one subdomain contains
/// the edge, the element with the highest element index is given. If the
/// edge is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofEdge edge)
{
return edgeInRank[edge];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given face in common.
/// Returns, for a given face, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this face. If the face is not contained in a rank's subdomain,
/// it will not be considered in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofFace face)
{
return faceInRank[face];
......
AMDiS/src/parallel/InteriorBoundary.cc
View file @ 7f81c570
......@@ -110,7 +110,7 @@ namespace AMDiS {
bound.neighObj.ithObj = it2->second.ithObject;
bound.type = INTERIOR;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
if (geoIndex == EDGE)
......@@ -181,6 +181,7 @@ namespace AMDiS {
bound.neighObj.subObj = VERTEX;
bound.neighObj.ithObj = perDofEl1.ithObject;
if (removePeriodicBoundary) {
bound.type = INTERIOR;
......@@ -191,47 +192,56 @@ namespace AMDiS {
int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
// Consider the following configuration of four elements partitioned
// to four ranks:
//
// |--------|--------|(*)
// | /| /|
// | 3 / | 0 / |
// | / | / |
// | / | / |
// | / | / |
// | / 2 | / 1 |
// | / | / |
// |/-------|/-------|
//
// We are in now interested in vertex (*). For the first, rank 1 owns
// the interior boundary with rank 0 on this vertex. When we remove
// periodic boundaries, which are applied on the left and right outer
// boundary, rank 3 becomes the owner of this vertex. Rank 0 and rank 1
// will have an interior boundary with rank 3 on this vertex, but rank
// 0 and rank 1 have no more interior boundaries on vertex (*). Thus we
// have to search and remove for this scenario.
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
// And than we can add the boundary to either own or other part of the
// interior database.
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.ithObj = ownerBoundEl.ithObject;
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
}
} else {
bound.type = it->second;
if (MeshDistributor::sebastianMode) {
if (bound.rankObj.elIndex == 3 &&
bound.rankObj.ithObj == 1 ||
bound.rankObj.elIndex == 78 &&
bound.rankObj.ithObj == 2) {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} else {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
}
}
......@@ -276,24 +286,15 @@ namespace AMDiS {
elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank];
// See comments in the same part of code for VERTEX
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
......@@ -368,23 +369,8 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
......@@ -413,6 +399,7 @@ namespace AMDiS {
}
}
// === Once we have this information, we must care about the order of the ===
// === atomic bounds in the three boundary handling object. Eventually ===
// === all the boundaries have to be in the same order on both ranks that ===
......@@ -454,19 +441,25 @@ namespace AMDiS {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound =
BoundaryObject &receivedBound =
stdMpi.getRecvData()[rank][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
if ((rankIt->second)[j].neighObj != receivedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
for (; k < rankIt->second.size(); k++) {
if ((rankIt->second)[k].neighObj == receivedBound)
break;
}
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
TEST_EXIT_DBG(k < rankIt->second.size())
("Cannot find element %d/%d/%d received from rank %d!\n",
receivedBound.elIndex,
receivedBound.subObj,
receivedBound.ithObj,
rank);
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
......@@ -531,6 +524,11 @@ namespace AMDiS {
}
}
} // periodicBoundary.boundary.size() > 0
// === Run verification procedure. ===
verifyBoundary();
}
......@@ -644,8 +642,37 @@ namespace AMDiS {
}
void InteriorBoundary::verifyBoundary()
{
FUNCNAME("InteriorBoundary::verifyBoundary()");
// === Check if no other boundery rank object is also included in own ===
// === boundary part, which would make no sence. ===
std::set<BoundaryObject> allOwnBounds;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
allOwnBounds.insert(bIt->rankObj);
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
if (allOwnBounds.count(bIt->rankObj)) {
ERROR_EXIT("Boundary %d/%d/%d is included in both, own and other interior boundary part!\n",
bIt->rankObj.elIndex,
bIt->rankObj.subObj,
bIt->rankObj.ithObj);
}
}
AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{
FUNCNAME("InteriorBoundary::getNewOwn()");
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
......@@ -654,6 +681,8 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewOther(int rank)
{
FUNCNAME("InteriorBoundary::getNewOther()");
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
......@@ -662,12 +691,86 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
{
FUNCNAME("InteriorBoundary::getNewPeriodic()");
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
bool InteriorBoundary::checkOther(AtomicBoundary& bound, int rank)
{
FUNCNAME("InteriorBoundary::checkOther()");
int globalMpiRank = MPI::COMM_WORLD.Get_rank();
TEST_EXIT(rank > globalMpiRank)
("Wrong MPI ranks: %d %d\n", rank, globalMpiRank);
bool found = false;
if (other.count(rank)) {
vector<AtomicBoundary>& otherBounds = other[rank];
for (int i = 0; i < static_cast<int>(otherBounds.size()); i++) {
if (otherBounds[i].rankObj == bound.rankObj) {
if (otherBounds[i].neighObj != bound.neighObj) {
ERROR_EXIT("If this really can happen, I have to thing about this!\n");
} else {
found = true;
break;
}
}
}
}
return found;
}
bool InteriorBoundary::removeOwn(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOwn()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
bool InteriorBoundary::removeOther(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOther()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
void InteriorBoundary::serializeExcludeList(ostream &out,
ExcludeList &list)
{
......
AMDiS/src/parallel/InteriorBoundary.h
View file @ 7f81c570
......@@ -86,12 +86,32 @@ namespace AMDiS {
void deserialize(istream &in, Mesh *mesh);
private:
/// In this function, we put some verification procedures to check for
/// consistency if the created interior boundary data. This function is
/// enabled even for optimized mode to check all possible meshes. Thus,
/// everything herein should be fast. If the code is stable, we can think
/// to remove this function.
void verifyBoundary();
AtomicBoundary& getNewOwn(int rank);
AtomicBoundary& getNewOther(int rank);
AtomicBoundary& getNewPeriodic(int rank);
/// Checks whether the given boundary is already a other boundary with
/// given rank number. Returns true, if the bound is already in the other
/// boundary database.
bool checkOther(AtomicBoundary& bound, int rank);
/// Removes the given boundary object from all owned boundaries. Returns
/// true, if at least one object has been removed, otherwise false.
bool removeOwn(BoundaryObject& bound);
/// Removes the given boundary object from all other boundaries. Returns
/// true, if at least one object has been removed, otherwise false.
bool removeOther(BoundaryObject& bound);
void serialize(ostream &out, RankToBoundMap& boundary);
void deserialize(istream &in,
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 7f81c570
......@@ -68,8 +68,6 @@ namespace AMDiS {
return (*dof1 < *dof2);
}
bool MeshDistributor::sebastianMode = false;
MeshDistributor::MeshDistributor()
: problemStat(0),
initialized(false),
......@@ -1696,7 +1694,7 @@ namespace AMDiS {
dofMaps[i]->update();
#if (DEBUG != 0)
dofMaps[i]->printInfo();
// dofMaps[i]->printInfo();
#endif
}
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 7f81c570
......@@ -290,7 +290,7 @@ namespace AMDiS {
void setBoundaryDofRequirement(Flag flag)
{
createBoundaryDofFlag = flag;
createBoundaryDofFlag |= flag;
}
BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace,
......@@ -584,8 +584,6 @@ namespace AMDiS {
vector<ParallelDofMapping*> dofMaps;
public:
static bool sebastianMode;
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
/// communication structure vectors, \ref sendDofs and \ref recvDofs.
......
AMDiS/src/parallel/ParallelCoarseSpaceMatVec.h
View file @ 7f81c570
......@@ -47,6 +47,7 @@ namespace AMDiS {
*/
class ParallelCoarseSpaceMatVec {
public:
/// Constructor
ParallelCoarseSpaceMatVec();
/// Set parallel DOF mapping for the interior DOFs.
......@@ -54,6 +55,12 @@ namespace AMDiS {
{
interiorMap = interiorDofs;
}
/// Returns the parallel DOF mapping for the interior DOFs.
ParallelDofMapping* getDofMapping()
{
return interiorMap;
}
/** \brief
* Sets the coarse space for all or a specific component.
......
AMDiS/src/parallel/ParallelProblemStatBase.h
View file @ 7f81c570
......@@ -40,8 +40,8 @@ namespace AMDiS {
virtual ~ParallelProblemStatBase() {}
void buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector);
bool assembleMatrix = true,
bool assembleVector = true);
void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL,
......
AMDiS/src/parallel/PetscProblemStat.cc
View file @ 7f81c570
......@@ -33,15 +33,16 @@ namespace AMDiS {
FUNCNAME("PetscProblemStat::PetscProblemStat()");
string tmp("");
Parameters::get("parallel->solver", tmp);
string initFileStr = nameStr + "->solver";
Parameters::get(initFileStr, tmp);
if (tmp == "petsc-schur") {
petscSolver = new PetscSolverSchur();
} else if (tmp == "petsc-feti") {
petscSolver = new PetscSolverFeti();
petscSolver = new PetscSolverFeti(initFileStr);
} else if (tmp == "petsc-block") {
petscSolver = new PetscSolverGlobalBlockMatrix();
} else if (tmp == "petsc" || tmp == "") {
} else if (tmp == "petsc") {
petscSolver = new PetscSolverGlobalMatrix();
} else if (tmp == "bddcml") {
#ifdef HAVE_BDDC_ML
......
AMDiS/src/parallel/PetscSolverFeti.cc
View file @ 7f81c570
......@@ -28,8 +28,9 @@ namespace AMDiS {
using namespace std;
PetscSolverFeti::PetscSolverFeti()
PetscSolverFeti::PetscSolverFeti(string name)
: PetscSolver(),
initFileStr(name),
primalDofMap(COMPONENT_WISE),
dualDofMap(COMPONENT_WISE),
interfaceDofMap(COMPONENT_WISE),
......@@ -51,8 +52,8 @@ namespace AMDiS {