Skip to content

GitLab

A
amdis
Commit 5eb8344b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

Fixed 3D refinement problem in parallel computations with mesh repartitioning.

parent 2eeaa8b6
Hide whitespace changes
Inline Side-by-side
Showing with 423 additions and 187 deletions
AMDiS/src/CoarseningManager3d.cc
View file @ 5eb8344b
......@@ -19,36 +19,40 @@
#include "RCNeighbourList.h"
#include "FixVec.h"
#include "DOFIndexed.h"
#include "Debug.h"
namespace AMDiS {
void CoarseningManager3d::coarsenFunction(ElInfo *el_info)
void CoarseningManager3d::coarsenFunction(ElInfo *elInfo)
{
FUNCNAME("CoarseningManager3d::coarsenFunction()");
Tetrahedron *el =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(el_info->getElement()));
dynamic_cast<Tetrahedron*>(const_cast<Element*>(elInfo->getElement()));
// If element must not be coarsend, return.
if (el->getMark() >= 0)
return; // el must not be coarsend, return :-(
return;
// Single leaves don't get coarsened.
if (el->isLeaf())
return; // single leaves don't get coarsened
return;
if (el->getChild(0)->getMark() >= 0 || el->getChild(1)->getMark() >= 0) {
// one of the children must not be coarsend; return :-(
// One of the children must not be coarsend, so return.
el->setMark(0);
return;
}
if (!(el->getChild(0)->isLeaf()) || !(el->getChild(1)->isLeaf())) {
// one of the children is not a leaf element; try again later on
// One of the children is not a leaf element. Try again later on.
doMore = true;
return;
}
DegreeOfFreedom *edge[2];
int n_neigh, bound = 0;
ElInfo *elinfo = el_info;
/****************************************************************************/
/* get a list for storing all elements at the coarsening edge and fill it */
......@@ -60,24 +64,95 @@ namespace AMDiS {
/* give the refinement edge the right orientation */
/****************************************************************************/
if (el->getDof(0,0) < el->getDof(1,0)) {
edge[0] = const_cast<int*>(el->getDof(0));
edge[1] = const_cast<int*>(el->getDof(1));
if (el->getDof(0, 0) < el->getDof(1, 0)) {
edge[0] = const_cast<DegreeOfFreedom*>(el->getDof(0));
edge[1] = const_cast<DegreeOfFreedom*>(el->getDof(1));
} else {
edge[1] = const_cast<int*>(el->getDof(0));
edge[0] = const_cast<int*>(el->getDof(1));
edge[1] = const_cast<DegreeOfFreedom*>(el->getDof(0));
edge[0] = const_cast<DegreeOfFreedom*>(el->getDof(1));
}
coarsenList.setElement(0, el, true);
n_neigh = 1;
coarsenList.setOppVertex(0, 0, 0);
coarsenList.setElType(0, el_info->getType());
coarsenList.setElType(0, elInfo->getType());
bound = false;
if (getCoarsenPatch(elinfo, edge, 0, coarsenList, &n_neigh)) {
getCoarsenPatch(elinfo, edge, 1, coarsenList, &n_neigh);
if (getCoarsenPatch(elInfo, edge, 0, coarsenList, &n_neigh)) {
getCoarsenPatch(elInfo, edge, 1, coarsenList, &n_neigh);
bound = true;
}
#if HAVE_PARALLEL_DOMAIN_AMDIS
Element *otherEl = NULL;
int otherEdge = -1;
FixRefinementPatch::getOtherEl(stack, &otherEl, otherEdge);
// === If the refinement edge must be fixed, add also the other part of this ===
// === edge to the refinement patch. ===
if (otherEl) {
// TODO: Remove these two lines and make something more meaningful!!
el->setMark(0);
return;
TraverseStack stack2;
ElInfo *elInfo2 =
stack2.traverseFirstOneMacro(mesh, otherEl->getIndex(), -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_NEIGH |
Mesh::FILL_BOUND);
bool foundEdge = false;
while (elInfo2) {
Element *el2 = elInfo2->getElement();
for (int i = 0; i < 6; i++) {
DofEdge edge2 = elInfo2->getElement()->getEdge(i);
if (edge2.first == *(edge[0]) && edge2.second == *(edge[1]) && !el2->isLeaf()) {
if (!el2->getChild(0)->isLeaf() || !el2->getChild(1)->isLeaf()) {
int edgeNo0 = el->getEdgeOfChild(0, i, elInfo2->getType());
int edgeNo1 = el->getEdgeOfChild(1, i, elInfo2->getType());
bool refineChildFirst =
!(i > 0 &&
(edgeNo0 >= 0 && !el2->getChild(0)->isLeaf()) ||
(edgeNo1 >= 0 && !el2->getChild(1)->isLeaf()));
if (refineChildFirst) {
// TODO: WAS SOLL ICH NUR MACHEN???
el->setMark(0);
return;
}
} else {
coarsenList.setElType(n_neigh, elInfo2->getType());
coarsenList.setElement(n_neigh, elInfo2->getElement(), true);
n_neigh++;
foundEdge = true;
TraverseStack *tmpStack = stack;
stack = &stack2;
if (getCoarsenPatch(elInfo2, edge, 0, coarsenList, &n_neigh)) {
getCoarsenPatch(elInfo2, edge, 1, coarsenList, &n_neigh);
bound = true;
}
stack = tmpStack;
break;
}
}
}
elInfo2 = stack2.traverseNext(elInfo2);
}
TEST_EXIT_DBG(foundEdge)("Should not happen!\n");
}
#endif
coarsenList.fillNeighbourRelations(n_neigh, bound);
/****************************************************************************/
......@@ -95,8 +170,8 @@ namespace AMDiS {
while (edge[0] != NULL) {
coarsenList.periodicSplit(edge, next_edge,
&n_neigh, &n_neigh_periodic,
periodicList);
&n_neigh, &n_neigh_periodic,
periodicList);
coarsenPatch(periodicList, n_neigh_periodic, bound);
......@@ -119,22 +194,19 @@ namespace AMDiS {
Tetrahedron *el =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(coarsenList.getElement(index)));
Tetrahedron *child[2];
Tetrahedron *neigh;
int dir, el_type, i, node, opp_v;
child[0] = dynamic_cast<Tetrahedron*>(const_cast<Element*>(el->getChild(0)));
child[1] = dynamic_cast<Tetrahedron*>(const_cast<Element*>(el->getChild(1)));
el_type = coarsenList.getType(index);
int el_type = coarsenList.getType(index);
/****************************************************************************/
/* Information about patch neighbours is still valid! But edge and face */
/* dof's in a common face of patch neighbours have to be removed */
/****************************************************************************/
for (dir = 0; dir < 2; dir++) {
neigh =
for (int dir = 0; dir < 2; dir++) {
Tetrahedron *neigh =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(coarsenList.getNeighbourElement(index, dir)));
opp_v = coarsenList.getOppVertex(index, dir);
if (!neigh || neigh->isLeaf()) {
/****************************************************************************/
......@@ -142,11 +214,11 @@ namespace AMDiS {
/****************************************************************************/
if (mesh->getNumberOfDofs(EDGE)) {
node = mesh->getNode(EDGE) + Tetrahedron::nChildEdge[el_type][0][dir];
int node = mesh->getNode(EDGE) + Tetrahedron::nChildEdge[el_type][0][dir];
mesh->freeDof(const_cast<int*>(child[0]->getDof(node)), EDGE);
}
if (mesh->getNumberOfDofs(FACE)) {
node = mesh->getNode(FACE) + Tetrahedron::nChildFace[el_type][0][dir];
int node = mesh->getNode(FACE) + Tetrahedron::nChildFace[el_type][0][dir];
mesh->freeDof(const_cast<int*>(child[0]->getDof(node)), FACE);
node = mesh->getNode(FACE) + Tetrahedron::nChildFace[el_type][1][dir];
mesh->freeDof(const_cast<int*>(child[1]->getDof(node)), FACE);
......@@ -160,14 +232,14 @@ namespace AMDiS {
/****************************************************************************/
if (mesh->getNumberOfDofs(FACE)) {
node = mesh->getNode(FACE);
int node = mesh->getNode(FACE);
mesh->freeDof(const_cast<int*>(child[0]->getDof(node)), FACE);
}
if (mesh->getNumberOfDofs(CENTER)) {
node = mesh->getNode(CENTER);
for (i = 0; i < 2; i++)
int node = mesh->getNode(CENTER);
for (int i = 0; i < 2; i++)
mesh->freeDof(const_cast<int*>(child[i]->getDof(node)), CENTER);
}
......@@ -191,7 +263,7 @@ namespace AMDiS {
/****************************************************************************/
/* get_coarse_patch: gets the patch for coarsening starting on element */
/* el_info->el in direction of neighbour [3-dir]; returns 1 if a boundary */
/* elInfo->el in direction of neighbour [3-dir]; returns 1 if a boundary */
/* reached and 0 if we come back to the starting element. */
/* */
/* if NEIGH_IN_EL we only can find the complete coarsening patch if the */
......@@ -205,16 +277,13 @@ namespace AMDiS {
/* the starting element before we return */
/****************************************************************************/
bool CoarseningManager3d::getCoarsenPatch(ElInfo *el_info,
bool CoarseningManager3d::getCoarsenPatch(ElInfo *elInfo,
DegreeOfFreedom *edge[2],
int dir,
RCNeighbourList &coarsenList,
int *n_neigh)
{
FUNCNAME("CoarseningManager3d::getCoarsenPatch");
ElInfo *neigh_info;
Tetrahedron *el, *neigh;
int i, j, k, opp_v, edge_no;
FUNCNAME("CoarseningManager3d::getCoarsenPatch()");
static unsigned char next_el[6][2] = {{3,2},
{1,3},
......@@ -223,15 +292,16 @@ namespace AMDiS {
{0,2},
{0,1}};
el = dynamic_cast<Tetrahedron*>(const_cast<Element*>(el_info->getElement()));
neigh =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(el_info->getNeighbour(3 - dir)));
Tetrahedron *el =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(elInfo->getElement()));
Tetrahedron *neigh =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(elInfo->getNeighbour(3 - dir)));
if (neigh == NULL)
return true;
opp_v = el_info->getOppVertex(3 - dir);
int opp_v = elInfo->getOppVertex(3 - dir);
ElInfo *neigh_info = stack->traverseNeighbour3d(elInfo, 3 - dir);
neigh_info = stack->traverseNeighbour3d(el_info, 3 - dir);
TEST_EXIT_DBG(neigh == neigh_info->getElement())
("neigh %d and neigh_info->el %d are not identical\n",
neigh->getIndex(), neigh_info->getElement()->getIndex());
......@@ -244,6 +314,7 @@ namespace AMDiS {
coarsenList.setElType(*n_neigh, neigh_info->getType());
int n_vertices = mesh->getGeo(VERTEX);
int i, j, k, edge_no;
while (neigh != el) {
for (j = 0; j < n_vertices; j++)
......@@ -261,10 +332,15 @@ namespace AMDiS {
if (mesh->associated(neigh->getDof(k, 0), edge[1][0]))
break;
TEST_EXIT_DBG(j < n_vertices && k < n_vertices)
("dof %d or dof %d not found on element %d with nodes (%d %d %d %d)\n",
edge[0][0], edge[1][0], neigh->getIndex(), neigh->getDof(0,0),
neigh->getDof(1,0), neigh->getDof(2,0), neigh->getDof(3,0));
TEST_EXIT_DBG(j < n_vertices)
("dof %d not found on element %d with nodes (%d %d %d %d)\n",
edge[0][0], neigh->getIndex(), neigh->getDof(0, 0),
neigh->getDof(1, 0), neigh->getDof(2, 0), neigh->getDof(3, 0));
TEST_EXIT_DBG(k < n_vertices)
("dof %d not found on element %d with nodes (%d %d %d %d)\n",
edge[1][0], neigh->getIndex(), neigh->getDof(0, 0),
neigh->getDof(1, 0), neigh->getDof(2, 0), neigh->getDof(3, 0));
}
edge_no = Tetrahedron::edgeOfDofs[j][k];
coarsenList.setCoarsePatch(*n_neigh, edge_no == 0);
......@@ -366,6 +442,12 @@ namespace AMDiS {
// === And now start to coarsen the patch: remove dof's of the coarsening edge. ===
WorldVector<double> c;
FiniteElemSpace *feSpace = FiniteElemSpace::provideFeSpace(mesh);
mesh->getDofIndexCoords(el->getChild(0)->getDof(3), feSpace, c);
// MSG("Free dof %p %f %f %f\n", el->getChild(0)->getDof(3), c[0], c[1], c[2]);
mesh->freeDof(const_cast<int*>(el->getChild(0)->getDof(3)), VERTEX);
mesh->incrementNumberOfVertices(-1);
......
AMDiS/src/Debug.cc
View file @ 5eb8344b
......@@ -74,6 +74,19 @@ namespace AMDiS {
}
void colorEdgeInMesh(FiniteElemSpace *feSpace,
Element *el,
int localEdgeNo,
std::string filename)
{
DOFVector<double> tmp(feSpace, "tmp");
tmp.set(0.0);
tmp[el->getEdge(localEdgeNo).first] = 1.0;
tmp[el->getEdge(localEdgeNo).second] = 1.0;
VtkWriter::writeFile(tmp, filename);
}
void writeElementIndexMesh(Mesh *mesh, std::string filename, int level)
{
FUNCNAME("debug::writeElementIndexMesh()");
......
AMDiS/src/Debug.h
View file @ 5eb8344b
......@@ -66,6 +66,11 @@ namespace AMDiS {
*/
void writeDofIndexMesh(FiniteElemSpace *feSpace);
void colorEdgeInMesh(FiniteElemSpace *feSpace,
Element *el,
int localEdgeNo,
std::string filename);
/** \brief
* Creates a vtu file where all elements in the mesh are colored by the global
* element indices.
......
AMDiS/src/ElInfo3d.cc
View file @ 5eb8344b
......@@ -483,7 +483,7 @@ namespace AMDiS {
normal[1] /= det;
normal[2] /= det;
} else {
MSG("not implemented for DIM_OF_WORLD = %d in 3d\n", dimOfWorld);
MSG("Not implemented for DIM_OF_WORLD = %d in 3D!\n", dimOfWorld);
}
return det;
......@@ -593,37 +593,11 @@ namespace AMDiS {
if ((nb = const_cast<Element*>((*neigh_old)[cv[i]]))) {
TEST_EXIT_DBG(nb->getChild(0))("nonconforming triangulation\n");
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("NEIGH %d OF EL is %d\n", i, nb->getIndex());
MSG("EL HAS DOFS: %d %d %d %d\n",
elOld->getDof(0, 0),
elOld->getDof(1, 0),
elOld->getDof(2, 0),
elOld->getDof(3, 0));
MSG("NEIGH-EL HAS DOFS: %d %d %d %d\n",
nb->getDof(0, 0),
nb->getDof(1, 0),
nb->getDof(2, 0),
nb->getDof(3, 0));
}
int k;
for (k = 0; k < 2; k++) { /* look at both childs of old neighbour */
nbk = const_cast<Element*>(nb->getChild(k));
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("TEST k = %d, nbk = %d, el = %d\n", k,
nbk->getDof(0,0), elOld->getDof(ichild,0));
MSG("TEST PTR nbk = %p, el = %p\n",
nbk->getDof(0), elOld->getDof(ichild));
}
if (nbk->getDof(0) == elOld->getDof(ichild)) {
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("DRIN!\n");
}
/* opp. vertex */
dof = const_cast<int*>(nb->getDof(elInfoOld->oppVertex[cv[i]]));
......@@ -641,17 +615,13 @@ namespace AMDiS {
}
(*neigh_local)[i] = nbk->getChild(0);
oppVertex[i] = 3;
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("BREAK 1\n");
}
break;
}
} else {
if (dof != nbk->getDof(2)) {
ov = -1;
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("BREAK 2\n");
}
break;
}
ov = 2;
......@@ -663,18 +633,12 @@ namespace AMDiS {
(*neigh_local)[i] = nbk;
oppVertex[i] = ov;
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("BREAK 3\n");
}
break;
}
} /* end for k */
if (elOld->getIndex() == 15376 && ichild == 0) {
MSG("k = %d ov = %d\n", k, ov);
}
// === Test if periodic. ===
......
AMDiS/src/FiniteElemSpace.cc
View file @ 5eb8344b
......@@ -89,7 +89,7 @@ namespace AMDiS {
}
FiniteElemSpace *FiniteElemSpace::provideFeSpace(DOFAdmin *admin,
FiniteElemSpace* FiniteElemSpace::provideFeSpace(DOFAdmin *admin,
const BasisFunction *basFcts,
Mesh *mesh,
std::string name_)
......@@ -104,6 +104,22 @@ namespace AMDiS {
}
#if DEBUG
FiniteElemSpace* FiniteElemSpace::provideFeSpace(Mesh *mesh)
{
FUNCNAME("FiniteElemSpace::provideFeSpace()");
for (unsigned int i = 0; i < feSpaces.size(); i++)
if (feSpaces[i]->mesh == mesh)
return feSpaces[i];
ERROR_EXIT("FE space not found!\n");
return NULL;
}
#endif
int FiniteElemSpace::calcMemoryUsage()
{
int result = sizeof(FiniteElemSpace);
......
AMDiS/src/FiniteElemSpace.h
View file @ 5eb8344b
......@@ -49,6 +49,14 @@ namespace AMDiS {
Mesh *mesh,
std::string name = "");
#if DEBUG
/// For debugging it may be useful to get some FE space for a given mesh at a
/// position in code where it is not possible to access the FE space directly. The
/// function assumes that there is only one FE space defined for the mesh.
static FiniteElemSpace *provideFeSpace(Mesh *mesh);
#endif
/// Destructor.
~FiniteElemSpace();
......
AMDiS/src/Mesh.cc
View file @ 5eb8344b
......@@ -1426,34 +1426,6 @@ namespace AMDiS {
}
void Mesh::computeMatrixFillin(const FiniteElemSpace *feSpace,
std::vector<int> &nnzInRow, int &overall, int &average)
{
std::map<DegreeOfFreedom, int> dofCounter;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, Mesh::CALL_LEAF_EL);
ElementDofIterator elDofIter(feSpace);
while (elInfo) {
elDofIter.reset(elInfo->getElement());
do {
DegreeOfFreedom dof = elDofIter.getDof();
if (dofCounter.count(dof) == 0) {
dofCounter[dof] = 1;
} else {
dofCounter[dof]++;
}
} while (elDofIter.next());
elInfo = stack.traverseNext(elInfo);
}
overall = 0;
for (std::map<DegreeOfFreedom, int>::iterator it = dofCounter.begin();
it != dofCounter.end(); ++it)
overall += it->second * 15;
}
int Mesh::calcMemoryUsage()
{
int result = sizeof(Mesh);
......
AMDiS/src/Mesh.h
View file @ 5eb8344b
......@@ -595,13 +595,6 @@ namespace AMDiS {
///
void clearMacroFileInfo();
/** \brief
* Traverse this mesh to compute the number of non zero elements the assembled
* matrix will have in each row.
*/
void computeMatrixFillin(const FiniteElemSpace *feSpace,
std::vector<int> &nnzInRow, int &overall, int &average);
///
int calcMemoryUsage();
......
AMDiS/src/RCNeighbourList.cc
View file @ 5eb8344b
......@@ -320,6 +320,8 @@ namespace AMDiS {
int *n_neigh_periodic,
RCNeighbourList &periodicList)
{
FUNCNAME("RCNeighbourList::periodicSplit()");
*n_neigh_periodic = 0;
int count = 0;
int n_neigh_old = *n_neigh;
......@@ -356,7 +358,7 @@ namespace AMDiS {
if (edge[0]) {
TEST_EXIT_DBG((dof0 == edge[0] && dof1 == edge[1]) ||
(dof1 == edge[0] && dof0 == edge[1]))
("invalid macro file?\n");
("Invalid macro file?\n");
}
}
......
AMDiS/src/RefinementManager3d.cc
View file @ 5eb8344b
......@@ -29,11 +29,15 @@
namespace AMDiS {
FixRefinementPatch::ConnectedEdges FixRefinementPatch::connectedEdges(0);
void RefinementManager3d::bisectTetrahedron(RCNeighbourList& refineList,
int index,
DegreeOfFreedom* dof[3],
DegreeOfFreedom *edge[2])
{
FUNCNAME("RefinementManager3d::bisectTetrahedron()");
Tetrahedron *el =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(refineList.getElement(index)));
Tetrahedron *child[2];
......@@ -169,7 +173,6 @@ namespace AMDiS {
}
if (!neigh || neigh->isLeaf()) {
/****************************************************************************/
/* get new dof's in the midedge of the face of el and for the two midpoints*/
/* of the sub-faces. If face is an interior face those pointers have to be */
......@@ -190,7 +193,7 @@ namespace AMDiS {
node1 = mesh->getNode(FACE) + Tetrahedron::nChildFace[el_type][1][dir];
(const_cast<Element*>(el->getSecondChild()))->setDof(node1, mesh->getDof(FACE));
}
} else { /* if (!neigh || !neigh->child[0]) */
} else {
/****************************************************************************/
/* interior face and neighbour has been refined, look for position at the */
/* refinement edge */
......@@ -244,24 +247,24 @@ namespace AMDiS {
}
void RefinementManager3d::newCoordsFct(ElInfo *el_info, RCNeighbourList &refineList)
void RefinementManager3d::newCoordsFct(ElInfo *elInfo, RCNeighbourList &refineList)
{
FUNCNAME("RefinementManager3d::newCoordsFct()");
Element *el = el_info->getElement();
Element *el = elInfo->getElement();
DegreeOfFreedom *edge[2];
ElInfo *elinfo = el_info;
ElInfo *elinfo = elInfo;
int dow = Global::getGeo(WORLD);
Projection *projector = el_info->getProjection(0);
Projection *projector = elInfo->getProjection(0);
if (!projector || projector->getType() != VOLUME_PROJECTION)
projector = el_info->getProjection(4);
projector = elInfo->getProjection(4);
if (el->getFirstChild() && projector && (!el->isNewCoordSet())) {
WorldVector<double> *new_coord = new WorldVector<double>;
for (int j = 0; j < dow; j++)
(*new_coord)[j] = (el_info->getCoord(0)[j] + el_info->getCoord(1)[j]) * 0.5;
(*new_coord)[j] = (elInfo->getCoord(0)[j] + elInfo->getCoord(1)[j]) * 0.5;
projector->project(*new_coord);
......@@ -271,11 +274,11 @@ namespace AMDiS {
/* get the refinement patch */
/****************************************************************************/
refineList.setElement(0, el);
refineList.setElType(0, el_info->getType());
refineList.setElType(0, elInfo->getType());
int n_neigh = 1;
for (int i = 0; i < 2; i++)
edge[i] = const_cast<int*>(el_info->getElement()->getDof(i));
edge[i] = const_cast<int*>(elInfo->getElement()->getDof(i));
if (getRefinePatch(&elinfo, edge, 0, refineList, &n_neigh)) {
// Domain's boundary was reached while looping around the refinement edge.
......@@ -322,6 +325,8 @@ namespace AMDiS {
RCNeighbourList &refineList,
int n_neigh, bool bound)
{
FUNCNAME("RefinementManager3d::refinePatch()");
Tetrahedron *el =
dynamic_cast<Tetrahedron*>(const_cast<Element*>(refineList.getElement(0)));