Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 7f81c570 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some issues for removing periodic boundary conditions.

parent 35eda406
......@@ -505,6 +505,7 @@ namespace AMDiS {
{
FUNCNAME("ProblemStat::createSolver()");
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
// === create solver ===
string solverType("0");
string initFileStr = name + "->solver";
......@@ -515,6 +516,7 @@ namespace AMDiS {
solverCreator->setName(initFileStr);
solver = solverCreator->create();
solver->initParameters();
#endif
}
......
......@@ -48,9 +48,7 @@ namespace AMDiS {
DegreeOfFreedom **dof = macroInfo->dof;
// === read periodic data =================================
if (periodicFile != "") {
WARNING("Periodic boundaries may lead to errors in small meshes if element neighbours are not set!\n");
if (periodicFile != "") {
FILE *file = fopen(periodicFile.c_str(), "r");
TEST_EXIT(file)("can't open file %s\n", periodicFile.c_str());
......
......@@ -105,16 +105,15 @@ namespace AMDiS {
periodicEdgeAssoc[edge0].insert(edge1);
// Add both vertices of the edge to be periodic.
periodicVertices[make_pair(edge0.first, edge1.first)] = boundaryType;
periodicVertices[make_pair(edge0.second, edge1.second)] = boundaryType;
DegreeOfFreedom mappedDof0 =
mesh->getPeriodicAssociations(boundaryType)[edge0.first];
DegreeOfFreedom mappedDof1 =
mesh->getPeriodicAssociations(boundaryType)[edge0.second];
periodicVertices[make_pair(edge0.first, mappedDof0)] = boundaryType;
periodicVertices[make_pair(edge0.second, mappedDof1)] = boundaryType;
periodicDofAssoc[edge0.first].insert(boundaryType);
periodicDofAssoc[edge0.second].insert(boundaryType);
TEST_EXIT_DBG(edge0.first ==
mesh->getPeriodicAssociations(boundaryType)[edge1.first] &&
edge0.second ==
mesh->getPeriodicAssociations(boundaryType)[edge1.second])
("Should not happen!\n");
periodicDofAssoc[edge0.second].insert(boundaryType);
}
}
break;
......@@ -134,12 +133,16 @@ namespace AMDiS {
periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
/// Add all three vertices of the face to be periodic.
periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] =
boundaryType;
periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] =
boundaryType;
periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] =
boundaryType;
DegreeOfFreedom mappedDof0 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<0>()];
DegreeOfFreedom mappedDof1 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<1>()];
DegreeOfFreedom mappedDof2 =
mesh->getPeriodicAssociations(boundaryType)[face0.get<2>()];
periodicVertices[make_pair(face0.get<0>(), mappedDof0)] = boundaryType;
periodicVertices[make_pair(face0.get<1>(), mappedDof1)] = boundaryType;
periodicVertices[make_pair(face0.get<2>(), mappedDof2)] = boundaryType;
periodicDofAssoc[face0.get<0>()].insert(boundaryType);
periodicDofAssoc[face0.get<1>()].insert(boundaryType);
......
......@@ -309,24 +309,32 @@ namespace AMDiS {
return faceElements[face];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given vertex DOF in common.
/// Returns, for a given vertex, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this vertex. If more than one element in one subdomain contains
/// the vertex, the element with the highest element index is given. If the
/// vertex is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DegreeOfFreedom vertex)
{
return vertexInRank[vertex];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given edge in common.
/// Returns, for a given edge, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this edge. If more than one element in one subdomain contains
/// the edge, the element with the highest element index is given. If the
/// edge is not contained in a rank's subdomain, it will not be considered
/// in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofEdge edge)
{
return edgeInRank[edge];
}
/// Returns a map that maps to each rank all macro elements in this rank that
/// have a given face in common.
/// Returns, for a given face, a map that maps from rank numbers to
/// element data objects, which identify on the rank the element which
/// contains this face. If the face is not contained in a rank's subdomain,
/// it will not be considered in this mapping.
map<int, ElementObjectData>& getElementsInRank(DofFace face)
{
return faceInRank[face];
......
......@@ -110,7 +110,7 @@ namespace AMDiS {
bound.neighObj.ithObj = it2->second.ithObject;
bound.type = INTERIOR;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
if (geoIndex == EDGE)
......@@ -181,6 +181,7 @@ namespace AMDiS {
bound.neighObj.subObj = VERTEX;
bound.neighObj.ithObj = perDofEl1.ithObject;
if (removePeriodicBoundary) {
bound.type = INTERIOR;
......@@ -191,47 +192,56 @@ namespace AMDiS {
int owner = std::max(elObjDb.getOwner(it->first.first, level),
elObjDb.getOwner(it->first.second, level));
// Consider the following configuration of four elements partitioned
// to four ranks:
//
// |--------|--------|(*)
// | /| /|
// | 3 / | 0 / |
// | / | / |
// | / | / |
// | / | / |
// | / 2 | / 1 |
// | / | / |
// |/-------|/-------|
//
// We are in now interested in vertex (*). For the first, rank 1 owns
// the interior boundary with rank 0 on this vertex. When we remove
// periodic boundaries, which are applied on the left and right outer
// boundary, rank 3 becomes the owner of this vertex. Rank 0 and rank 1
// will have an interior boundary with rank 3 on this vertex, but rank
// 0 and rank 1 have no more interior boundaries on vertex (*). Thus we
// have to search and remove for this scenario.
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
// And than we can add the boundary to either own or other part of the
// interior database.
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.ithObj = ownerBoundEl.ithObject;
AtomicBoundary& b = getNewOther(owner);
b = bound;
ElementObjectData& ownerBoundEl = objData[owner];
b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
b.neighObj.elIndex = ownerBoundEl.elIndex;
b.neighObj.elType = -1;
b.neighObj.ithObj = ownerBoundEl.ithObject;
}
} else {
bound.type = it->second;
if (MeshDistributor::sebastianMode) {
if (bound.rankObj.elIndex == 3 &&
bound.rankObj.ithObj == 1 ||
bound.rankObj.elIndex == 78 &&
bound.rankObj.ithObj == 2) {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
} else {
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
AtomicBoundary& b = getNewPeriodic(otherElementRank);
b = bound;
}
}
}
......@@ -276,24 +286,15 @@ namespace AMDiS {
elObjDb.getOwner(it->first.second, level));
ElementObjectData& rankBoundEl = objData[globalMpiRank];
// See comments in the same part of code for VERTEX
if (owner != globalMpiRank) {
removeOwn(bound.rankObj);
removeOther(bound.rankObj);
}
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
......@@ -368,23 +369,8 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[globalMpiRank];
if (owner == globalMpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == globalMpiRank)
continue;
if (!allRanks && levelRanks.count(it2->first) == 0)
continue;
AtomicBoundary& b = getNewOwn(it2->first);
b = bound;
b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
b.neighObj.elIndex = it2->second.elIndex;
b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
b.neighObj.ithObj = it2->second.ithObject;
b.neighObj.reverseMode =
elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
}
AtomicBoundary& b = getNewOwn(otherElementRank);
b = bound;
} else {
AtomicBoundary& b = getNewOther(owner);
b = bound;
......@@ -413,6 +399,7 @@ namespace AMDiS {
}
}
// === Once we have this information, we must care about the order of the ===
// === atomic bounds in the three boundary handling object. Eventually ===
// === all the boundaries have to be in the same order on both ranks that ===
......@@ -454,19 +441,25 @@ namespace AMDiS {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound =
BoundaryObject &receivedBound =
stdMpi.getRecvData()[rank][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
if ((rankIt->second)[j].neighObj != receivedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
for (; k < rankIt->second.size(); k++) {
if ((rankIt->second)[k].neighObj == receivedBound)
break;
}
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
TEST_EXIT_DBG(k < rankIt->second.size())
("Cannot find element %d/%d/%d received from rank %d!\n",
receivedBound.elIndex,
receivedBound.subObj,
receivedBound.ithObj,
rank);
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
......@@ -531,6 +524,11 @@ namespace AMDiS {
}
}
} // periodicBoundary.boundary.size() > 0
// === Run verification procedure. ===
verifyBoundary();
}
......@@ -644,8 +642,37 @@ namespace AMDiS {
}
void InteriorBoundary::verifyBoundary()
{
FUNCNAME("InteriorBoundary::verifyBoundary()");
// === Check if no other boundery rank object is also included in own ===
// === boundary part, which would make no sence. ===
std::set<BoundaryObject> allOwnBounds;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
allOwnBounds.insert(bIt->rankObj);
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it)
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt)
if (allOwnBounds.count(bIt->rankObj)) {
ERROR_EXIT("Boundary %d/%d/%d is included in both, own and other interior boundary part!\n",
bIt->rankObj.elIndex,
bIt->rankObj.subObj,
bIt->rankObj.ithObj);
}
}
AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
{
FUNCNAME("InteriorBoundary::getNewOwn()");
int size = own[rank].size();
own[rank].resize(size + 1);
return own[rank][size];
......@@ -654,6 +681,8 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewOther(int rank)
{
FUNCNAME("InteriorBoundary::getNewOther()");
int size = other[rank].size();
other[rank].resize(size + 1);
return other[rank][size];
......@@ -662,12 +691,86 @@ namespace AMDiS {
AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
{
FUNCNAME("InteriorBoundary::getNewPeriodic()");
int size = periodic[rank].size();
periodic[rank].resize(size + 1);
return periodic[rank][size];
}
bool InteriorBoundary::checkOther(AtomicBoundary& bound, int rank)
{
FUNCNAME("InteriorBoundary::checkOther()");
int globalMpiRank = MPI::COMM_WORLD.Get_rank();
TEST_EXIT(rank > globalMpiRank)
("Wrong MPI ranks: %d %d\n", rank, globalMpiRank);
bool found = false;
if (other.count(rank)) {
vector<AtomicBoundary>& otherBounds = other[rank];
for (int i = 0; i < static_cast<int>(otherBounds.size()); i++) {
if (otherBounds[i].rankObj == bound.rankObj) {
if (otherBounds[i].neighObj != bound.neighObj) {
ERROR_EXIT("If this really can happen, I have to thing about this!\n");
} else {
found = true;
break;
}
}
}
}
return found;
}
bool InteriorBoundary::removeOwn(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOwn()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
it != own.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
bool InteriorBoundary::removeOther(BoundaryObject& bound)
{
FUNCNAME("InteriorBoundary::removeOther()");
bool removed = false;
for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
it != other.end(); ++it) {
for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
if (bIt->rankObj == bound) {
it->second.erase(bIt);
removed = true;
break;
}
}
}
return removed;
}
void InteriorBoundary::serializeExcludeList(ostream &out,
ExcludeList &list)
{
......
......@@ -86,12 +86,32 @@ namespace AMDiS {
void deserialize(istream &in, Mesh *mesh);
private:
/// In this function, we put some verification procedures to check for
/// consistency if the created interior boundary data. This function is
/// enabled even for optimized mode to check all possible meshes. Thus,
/// everything herein should be fast. If the code is stable, we can think
/// to remove this function.
void verifyBoundary();
AtomicBoundary& getNewOwn(int rank);
AtomicBoundary& getNewOther(int rank);
AtomicBoundary& getNewPeriodic(int rank);
/// Checks whether the given boundary is already a other boundary with
/// given rank number. Returns true, if the bound is already in the other
/// boundary database.
bool checkOther(AtomicBoundary& bound, int rank);
/// Removes the given boundary object from all owned boundaries. Returns
/// true, if at least one object has been removed, otherwise false.
bool removeOwn(BoundaryObject& bound);
/// Removes the given boundary object from all other boundaries. Returns
/// true, if at least one object has been removed, otherwise false.
bool removeOther(BoundaryObject& bound);
void serialize(ostream &out, RankToBoundMap& boundary);
void deserialize(istream &in,
......
......@@ -68,8 +68,6 @@ namespace AMDiS {
return (*dof1 < *dof2);
}
bool MeshDistributor::sebastianMode = false;
MeshDistributor::MeshDistributor()
: problemStat(0),
initialized(false),
......@@ -1696,7 +1694,7 @@ namespace AMDiS {
dofMaps[i]->update();
#if (DEBUG != 0)
dofMaps[i]->printInfo();
// dofMaps[i]->printInfo();
#endif
}
......
......@@ -290,7 +290,7 @@ namespace AMDiS {
void setBoundaryDofRequirement(Flag flag)
{
createBoundaryDofFlag = flag;
createBoundaryDofFlag |= flag;
}
BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace,
......@@ -584,8 +584,6 @@ namespace AMDiS {
vector<ParallelDofMapping*> dofMaps;
public:
static bool sebastianMode;
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
/// communication structure vectors, \ref sendDofs and \ref recvDofs.
......
......@@ -47,6 +47,7 @@ namespace AMDiS {
*/
class ParallelCoarseSpaceMatVec {
public:
/// Constructor
ParallelCoarseSpaceMatVec();
/// Set parallel DOF mapping for the interior DOFs.
......@@ -54,6 +55,12 @@ namespace AMDiS {
{
interiorMap = interiorDofs;
}
/// Returns the parallel DOF mapping for the interior DOFs.
ParallelDofMapping* getDofMapping()
{
return interiorMap;
}
/** \brief
* Sets the coarse space for all or a specific component.
......
......@@ -40,8 +40,8 @@ namespace AMDiS {
virtual ~ParallelProblemStatBase() {}
void buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector);
bool assembleMatrix = true,
bool assembleVector = true);
void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL,
......
......@@ -33,15 +33,16 @@ namespace AMDiS {
FUNCNAME("PetscProblemStat::PetscProblemStat()");
string tmp("");
Parameters::get("parallel->solver", tmp);
string initFileStr = nameStr + "->solver";
Parameters::get(initFileStr, tmp);
if (tmp == "petsc-schur") {
petscSolver = new PetscSolverSchur();
} else if (tmp == "petsc-feti") {
petscSolver = new PetscSolverFeti();
petscSolver = new PetscSolverFeti(initFileStr);
} else if (tmp == "petsc-block") {
petscSolver = new PetscSolverGlobalBlockMatrix();
} else if (tmp == "petsc" || tmp == "") {
} else if (tmp == "petsc") {
petscSolver = new PetscSolverGlobalMatrix();
} else if (tmp == "bddcml") {
#ifdef HAVE_BDDC_ML
......
......@@ -28,8 +28,9 @@ namespace AMDiS {
using namespace std;
PetscSolverFeti::PetscSolverFeti()
PetscSolverFeti::PetscSolverFeti(string name)
: PetscSolver(),
initFileStr(name),
primalDofMap(COMPONENT_WISE),
dualDofMap(COMPONENT_WISE),
interfaceDofMap