Commit 0f8097b4 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

New approach for deleting double DOFs in mesh repartitioning.

parent 9fe7e69d
...@@ -768,6 +768,30 @@ namespace AMDiS { ...@@ -768,6 +768,30 @@ namespace AMDiS {
} }
void testDofsByCoords(FiniteElemSpace *feSpace,
DofContainer &dofs0, DofContainer &dofs1)
{
FUNCNAME("debug::testDofsByCoords()");
TEST_EXIT(dofs0.size() == dofs1.size())
("The dof containers have different sizes %d %d!\n",
dofs0.size(), dofs1.size());
DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds");
feSpace->getMesh()->getDofIndexCoords(feSpace, coords);
for (unsigned int i = 0; i < dofs0.size(); i++) {
WorldVector<double> tmp = coords[*(dofs0[i])];
tmp -= coords[*(dofs1[i])];
TEST_EXIT(norm(tmp) < 1e-13)
("DOFs %d and %d (i = %d) have different coords!\n",
*(dofs0[i]), *(dofs1[i]), i);
}
}
} // namespace debug } // namespace debug
} // namespace AMDiS } // namespace AMDiS
...@@ -193,6 +193,9 @@ namespace AMDiS { ...@@ -193,6 +193,9 @@ namespace AMDiS {
const DegreeOfFreedom* dof3, const DegreeOfFreedom* dof3,
DofContainer &vec); DofContainer &vec);
void testDofsByCoords(FiniteElemSpace *feSpace,
DofContainer &dofs0, DofContainer &dofs1);
} }
} }
......
...@@ -97,6 +97,32 @@ namespace AMDiS { ...@@ -97,6 +97,32 @@ namespace AMDiS {
nSize = vertexLocalMap.size();
SerUtil::serialize(out, nSize);
for (std::map<ElementObjectData, DegreeOfFreedom>::iterator it = vertexLocalMap.begin();
it != vertexLocalMap.end(); ++it) {
it->first.serialize(out);
SerUtil::serialize(out, it->second);
}
nSize = edgeLocalMap.size();
SerUtil::serialize(out, nSize);
for (std::map<ElementObjectData, DofEdge>::iterator it = edgeLocalMap.begin();
it != edgeLocalMap.end(); ++it) {
it->first.serialize(out);
SerUtil::serialize(out, it->second);
}
nSize = faceLocalMap.size();
SerUtil::serialize(out, nSize);
for (std::map<ElementObjectData, DofFace>::iterator it = faceLocalMap.begin();
it != faceLocalMap.end(); ++it) {
it->first.serialize(out);
SerUtil::serialize(out, it->second);
}
SerUtil::serialize(out, vertexOwner); SerUtil::serialize(out, vertexOwner);
SerUtil::serialize(out, edgeOwner); SerUtil::serialize(out, edgeOwner);
SerUtil::serialize(out, faceOwner); SerUtil::serialize(out, faceOwner);
...@@ -165,6 +191,38 @@ namespace AMDiS { ...@@ -165,6 +191,38 @@ namespace AMDiS {
} }
SerUtil::deserialize(in, nSize);
vertexLocalMap.clear();
for (int i = 0; i < nSize; i++) {
ElementObjectData data;
DegreeOfFreedom dof;
data.deserialize(in);
SerUtil::deserialize(in, dof);
vertexLocalMap[data] = dof;
}
SerUtil::deserialize(in, nSize);
edgeLocalMap.clear();
for (int i = 0; i < nSize; i++) {
ElementObjectData data;
DofEdge edge;
data.deserialize(in);
SerUtil::deserialize(in, edge);
edgeLocalMap[data] = edge;
}
SerUtil::deserialize(in, nSize);
faceLocalMap.clear();
for (int i = 0; i < nSize; i++) {
ElementObjectData data;
DofFace face;
data.deserialize(in);
SerUtil::deserialize(in, face);
faceLocalMap[data] = face;
}
SerUtil::deserialize(in, vertexOwner); SerUtil::deserialize(in, vertexOwner);
SerUtil::deserialize(in, edgeOwner); SerUtil::deserialize(in, edgeOwner);
SerUtil::deserialize(in, faceOwner); SerUtil::deserialize(in, faceOwner);
......
...@@ -48,7 +48,7 @@ namespace AMDiS { ...@@ -48,7 +48,7 @@ namespace AMDiS {
BoundaryType boundaryType; BoundaryType boundaryType;
void serialize(std::ostream &out) void serialize(std::ostream &out) const
{ {
SerUtil::serialize(out, elIndex); SerUtil::serialize(out, elIndex);
SerUtil::serialize(out, ithObject); SerUtil::serialize(out, ithObject);
...@@ -63,6 +63,20 @@ namespace AMDiS { ...@@ -63,6 +63,20 @@ namespace AMDiS {
SerUtil::deserialize(in, boundaryType); SerUtil::deserialize(in, boundaryType);
} }
bool operator==(ElementObjectData& cmp) const
{
return (elIndex == cmp.elIndex &&
ithObject == cmp.ithObject &&
boundaryType == cmp.boundaryType);
}
bool operator<(const ElementObjectData& rhs) const
{
return (elIndex < rhs.elIndex ||
(elIndex == rhs.elIndex &&
ithObject < rhs.ithObject));
}
}; };
...@@ -73,26 +87,56 @@ namespace AMDiS { ...@@ -73,26 +87,56 @@ namespace AMDiS {
: iterGeoPos(CENTER) : iterGeoPos(CENTER)
{} {}
void addVertex(DegreeOfFreedom vertex,
int elIndex, int ith, BoundaryType bound = INTERIOR) void addVertex(Element *el, int ith, BoundaryType bound = INTERIOR)
{ {
vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound)); DegreeOfFreedom vertex = el->getDof(ith, 0);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
vertexElements[vertex].push_back(elObj);
vertexLocalMap[elObj] = vertex;
} }
void addEdge(DofEdge edge,
int elIndex, int ith, BoundaryType bound = INTERIOR) void addEdge(Element *el, int ith, BoundaryType bound = INTERIOR)
{ {
edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound)); DofEdge edge = el->getEdge(ith);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
edgeElements[edge].push_back(elObj);
edgeLocalMap[elObj] = edge;
}
void addFace(Element *el, int ith, BoundaryType bound = INTERIOR)
{
DofFace face = el->getFace(ith);
int elIndex = el->getIndex();
ElementObjectData elObj(elIndex, ith, bound);
faceElements[face].push_back(elObj);
faceLocalMap[elObj] = face;
} }
void addFace(DofFace face,
int elIndex, int ith, BoundaryType bound = INTERIOR) void addElement(Element *el, BoundaryType bound = INTERIOR)
{ {
faceElements[face].push_back(ElementObjectData(elIndex, ith, bound)); for (int i = 0; i < el->getGeo(VERTEX); i++)
addVertex(el, i);
for (int i = 0; i < el->getGeo(EDGE); i++)
addEdge(el, i);
for (int i = 0; i < el->getGeo(FACE); i++)
addFace(el, i);
} }
void createRankData(std::map<int, int>& macroElementRankMap); void createRankData(std::map<int, int>& macroElementRankMap);
bool iterate(GeoIndex pos) bool iterate(GeoIndex pos)
{ {
if (iterGeoPos == CENTER) { if (iterGeoPos == CENTER) {
...@@ -258,6 +302,21 @@ namespace AMDiS { ...@@ -258,6 +302,21 @@ namespace AMDiS {
return faceInRank[face]; return faceInRank[face];
} }
DegreeOfFreedom getVertexLocalMap(ElementObjectData &data)
{
return vertexLocalMap[data];
}
DofEdge getEdgeLocalMap(ElementObjectData &data)
{
return edgeLocalMap[data];
}
DofFace getFaceLocalMap(ElementObjectData &data)
{
return faceLocalMap[data];
}
void serialize(std::ostream &out); void serialize(std::ostream &out);
void deserialize(std::istream &in); void deserialize(std::istream &in);
...@@ -276,14 +335,22 @@ namespace AMDiS { ...@@ -276,14 +335,22 @@ namespace AMDiS {
std::map<DofEdge, std::vector<ElementObjectData> > edgeElements; std::map<DofEdge, std::vector<ElementObjectData> > edgeElements;
std::map<DofFace, std::vector<ElementObjectData> > faceElements; std::map<DofFace, std::vector<ElementObjectData> > faceElements;
std::map<ElementObjectData, DegreeOfFreedom> vertexLocalMap;
std::map<ElementObjectData, DofEdge> edgeLocalMap;
std::map<ElementObjectData, DofFace> faceLocalMap;
std::map<DegreeOfFreedom, int> vertexOwner; std::map<DegreeOfFreedom, int> vertexOwner;
std::map<DofEdge, int> edgeOwner; std::map<DofEdge, int> edgeOwner;
std::map<DofFace, int> faceOwner; std::map<DofFace, int> faceOwner;
std::map<DegreeOfFreedom, std::map<int, ElementObjectData> > vertexInRank; std::map<DegreeOfFreedom, std::map<int, ElementObjectData> > vertexInRank;
std::map<DofEdge, std::map<int, ElementObjectData> > edgeInRank; std::map<DofEdge, std::map<int, ElementObjectData> > edgeInRank;
std::map<DofFace, std::map<int, ElementObjectData> > faceInRank; std::map<DofFace, std::map<int, ElementObjectData> > faceInRank;
std::map<DegreeOfFreedom, std::map<int, ElementObjectData> >::iterator vertexIter; std::map<DegreeOfFreedom, std::map<int, ElementObjectData> >::iterator vertexIter;
std::map<DofEdge, std::map<int, ElementObjectData> >::iterator edgeIter; std::map<DofEdge, std::map<int, ElementObjectData> >::iterator edgeIter;
std::map<DofFace, std::map<int, ElementObjectData> >::iterator faceIter; std::map<DofFace, std::map<int, ElementObjectData> >::iterator faceIter;
......
...@@ -44,13 +44,13 @@ namespace AMDiS { ...@@ -44,13 +44,13 @@ namespace AMDiS {
excludedSubstructures(0) excludedSubstructures(0)
{} {}
BoundaryObject(Element *e, int eType, GeoIndex sObj, int iObj) BoundaryObject(Element *e, int eType, GeoIndex sObj, int iObj, bool rMode = false)
: el(e), : el(e),
elIndex(e->getIndex()), elIndex(e->getIndex()),
elType(eType), elType(eType),
subObj(sObj), subObj(sObj),
ithObj(iObj), ithObj(iObj),
reverseMode(false), reverseMode(rMode),
excludedSubstructures(0) excludedSubstructures(0)
{} {}
......
//
// Software License for AMDiS // Software License for AMDiS
// //
// Copyright (c) 2010 Dresden University of Technology // Copyright (c) 2010 Dresden University of Technology
...@@ -68,6 +68,7 @@ namespace AMDiS { ...@@ -68,6 +68,7 @@ namespace AMDiS {
deserialized(false), deserialized(false),
writeSerializationFile(false), writeSerializationFile(false),
repartitioningAllowed(false), repartitioningAllowed(false),
repartitionIthChange(20),
nTimestepsAfterLastRepartitioning(0), nTimestepsAfterLastRepartitioning(0),
repartCounter(0), repartCounter(0),
debugOutputDir(""), debugOutputDir(""),
...@@ -83,7 +84,9 @@ namespace AMDiS { ...@@ -83,7 +84,9 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->repartitioning", "%d", &tmp); GET_PARAMETER(0, name + "->repartitioning", "%d", &tmp);
repartitioningAllowed = (tmp > 0); repartitioningAllowed = (tmp > 0);
GET_PARAMETER(0, name + "->debug output dir", &debugOutputDir); GET_PARAMETER(0, name + "->debug output dir", &debugOutputDir);
GET_PARAMETER(0, name + "->repartition ith change", "%d", &repartitionIthChange);
} }
...@@ -581,7 +584,7 @@ namespace AMDiS { ...@@ -581,7 +584,7 @@ namespace AMDiS {
nTimestepsAfterLastRepartitioning++; nTimestepsAfterLastRepartitioning++;
if (repartitioningAllowed) { if (repartitioningAllowed) {
if (nTimestepsAfterLastRepartitioning >= 20) { if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh(); repartitionMesh();
nTimestepsAfterLastRepartitioning = 0; nTimestepsAfterLastRepartitioning = 0;
} }
...@@ -1001,7 +1004,9 @@ namespace AMDiS { ...@@ -1001,7 +1004,9 @@ namespace AMDiS {
return; return;
#if (DEBUG != 0) #if (DEBUG != 0)
if (repartCounter == 0) { ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) {
std::stringstream oss; std::stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu"; oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp"); DOFVector<double> tmpa(feSpace, "tmp");
...@@ -1191,8 +1196,8 @@ namespace AMDiS { ...@@ -1191,8 +1196,8 @@ namespace AMDiS {
// === Remove double DOFs. === // === Remove double DOFs. ===
MeshManipulation meshManipulation(mesh); MeshManipulation meshManipulation(feSpace);
meshManipulation.deleteDoubleDofs(newMacroEl); meshManipulation.deleteDoubleDofs(newMacroEl, elObjects);
mesh->dofCompress(); mesh->dofCompress();
...@@ -1207,21 +1212,24 @@ namespace AMDiS { ...@@ -1207,21 +1212,24 @@ namespace AMDiS {
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs()); MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
ParallelDebug::testAllElements(*this); ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif #endif
partitioner->fillCoarsePartitionVec(&partitionVec); partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo(); updateInteriorBoundaryInfo();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
#if (DEBUG != 0) #if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n"); MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this); ParallelDebug::testAllElements(*this);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh"); debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh");
...@@ -1273,14 +1281,7 @@ namespace AMDiS { ...@@ -1273,14 +1281,7 @@ namespace AMDiS {
// === Add all sub object of the element to the variable elObjects. === // === Add all sub object of the element to the variable elObjects. ===
for (int i = 0; i < el->getGeo(VERTEX); i++) elObjects.addElement(el);
elObjects.addVertex(el->getDof(i, 0), el->getIndex(), i);
for (int i = 0; i < el->getGeo(EDGE); i++)
elObjects.addEdge(el->getEdge(i), el->getIndex(), i);
for (int i = 0; i < el->getGeo(FACE); i++)
elObjects.addFace(el->getFace(i), el->getIndex(), i);
// === Get periodic boundary information. === // === Get periodic boundary information. ===
...@@ -1801,7 +1802,8 @@ namespace AMDiS { ...@@ -1801,7 +1802,8 @@ namespace AMDiS {
// === Update dof admins due to new number of dofs. === // === Update dof admins due to new number of dofs. ===
lastMeshChangeIndex = mesh->getChangeIndex(); lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
std::stringstream oss; std::stringstream oss;
oss << debugOutputDir << "elementIndex-" << mpiRank << ".vtu"; oss << debugOutputDir << "elementIndex-" << mpiRank << ".vtu";
......
...@@ -568,6 +568,8 @@ namespace AMDiS { ...@@ -568,6 +568,8 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations. /// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed; bool repartitioningAllowed;
int repartitionIthChange;
int nTimestepsAfterLastRepartitioning; int nTimestepsAfterLastRepartitioning;
int repartCounter; int repartCounter;
......
...@@ -12,118 +12,118 @@ ...@@ -12,118 +12,118 @@
#include "parallel/MeshManipulation.h" #include "parallel/MeshManipulation.h"
#include "Mesh.h" #include "Mesh.h"
#include "BasisFunction.h"
#include "Traverse.h" #include "Traverse.h"
#include "Debug.h" #include "Debug.h"
namespace AMDiS { namespace AMDiS {
void MeshManipulation::deleteDoubleDofs(std::set<MacroElement*>& newMacroEl) void MeshManipulation::deleteDoubleDofs(std::set<MacroElement*>& newMacroEl,
ElementObjects &objects)
{ {
std::map<int, MacroElement*> leafInMacroEl; FUNCNAME("MeshManipulation::deleteDoubleDofs()");
TEST_EXIT(mesh->getDim() == 2)("Not yet supported for dim != 2!\n");
std::map<int, MacroElement*> macroIndexMap;
for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
it != newMacroEl.end(); ++it)
macroIndexMap[(*it)->getIndex()] = *it;
std::set<int> macrosProcessed;
std::map<const DegreeOfFreedom*, const DegreeOfFreedom*> mapDelDofs;
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL); ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) { while (elInfo) {
leafInMacroEl[elInfo->getElement()->getIndex()] = elInfo->getMacroElement(); if (newMacroEl.count(elInfo->getMacroElement()) == 0) {
int index = elInfo->getMacroElement()->getIndex();
macrosProcessed.insert(index);
macroIndexMap[index] = elInfo->getMacroElement();
}
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
deleteDoubleDofs(leafInMacroEl, newMacroEl, 0);
deleteDoubleDofs(leafInMacroEl, newMacroEl, 1);
}
for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
it != newMacroEl.end(); ++it) {
void MeshManipulation::deleteDoubleDofs(std::map<int, MacroElement*>& leafInMacroEl, for (int i = 0; i < mesh->getGeo(VERTEX); i++) {
std::set<MacroElement*>& newMacroEl, ElementObjectData elObj((*it)->getIndex(), i);
int mode) DegreeOfFreedom vertex = objects.getVertexLocalMap(elObj);
{ std::vector<ElementObjectData> &vertexEl = objects.getElements(vertex);
FUNCNAME("MeshManipulation::deleteDoubleDofs()");
std::map<const DegreeOfFreedom*, const DegreeOfFreedom*> mapDelDofs; for (std::vector<ElementObjectData>::iterator elIt = vertexEl.begin();
std::map<const DegreeOfFreedom*, int> mapDofsMacro; elIt != vertexEl.end(); ++elIt) {
if (elIt->elIndex == (*it)->getIndex())
continue;
TraverseStack stack; if (macrosProcessed.count(elIt->elIndex) == 1) {
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH); TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex) == 1)
while (elInfo) { ("Should not happen!\n");
Element *el = elInfo->getElement();
Element *el0 = (*it)->getElement();
for (int i = 0; i < mesh->getGeo(NEIGH); i++) { Element *el1 = macroIndexMap[elIt->elIndex]->getElement();