Commit ca3a8888 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed repartitioning bug for very small meshes.

parent a933472f
...@@ -123,7 +123,7 @@ namespace AMDiS { ...@@ -123,7 +123,7 @@ namespace AMDiS {
/// Returns \ref dof[i][j] which is the j-th DOF of the i-th node of Element. /// Returns \ref dof[i][j] which is the j-th DOF of the i-th node of Element.
inline DegreeOfFreedom getDof(int i, int j) const inline DegreeOfFreedom getDof(int i, int j) const
{ {
TEST_EXIT_DBG(dofValid)("Should not happen!\n"); TEST_EXIT_DBG(dofValid)("DOFs are not valid in element %d!\n", index);
return dof[i][j]; return dof[i][j];
} }
...@@ -131,7 +131,7 @@ namespace AMDiS { ...@@ -131,7 +131,7 @@ namespace AMDiS {
/// Returns \ref dof[i] which is a pointer to the DOFs of the i-th node. /// Returns \ref dof[i] which is a pointer to the DOFs of the i-th node.
inline const DegreeOfFreedom* getDof(int i) const inline const DegreeOfFreedom* getDof(int i) const
{ {
TEST_EXIT_DBG(dofValid)("Should not happen!\n"); TEST_EXIT_DBG(dofValid)("DOFs are not valid in element %d!\n", index);
return dof[i]; return dof[i];
} }
...@@ -139,7 +139,7 @@ namespace AMDiS { ...@@ -139,7 +139,7 @@ namespace AMDiS {
/// Returns a pointer to the DOFs of this Element /// Returns a pointer to the DOFs of this Element
inline const DegreeOfFreedom** getDof() const inline const DegreeOfFreedom** getDof() const
{ {
TEST_EXIT_DBG(dofValid)("Should not happen!\n"); TEST_EXIT_DBG(dofValid)("DOFs are not valid in element %d!\n", index);
return const_cast<const DegreeOfFreedom**>(dof); return const_cast<const DegreeOfFreedom**>(dof);
} }
......
...@@ -1006,9 +1006,10 @@ namespace AMDiS { ...@@ -1006,9 +1006,10 @@ namespace AMDiS {
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh); ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) { if (repartCounter == 0) {
std::stringstream oss; std::stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu"; oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
MSG("Write partitioning to %s\n", oss.str().c_str());
DOFVector<double> tmpa(feSpace, "tmp"); DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank); tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str()); VtkWriter::writeFile(&tmpa, oss.str());
......
...@@ -11,7 +11,8 @@ ...@@ -11,7 +11,8 @@
#include <queue> #include <queue>
#include "ParMetisPartitioner.h" #include "parallel/ParMetisPartitioner.h"
#include "parallel/MpiHelper.h"
#include "Mesh.h" #include "Mesh.h"
#include "Traverse.h" #include "Traverse.h"
#include "ElInfo.h" #include "ElInfo.h"
...@@ -32,7 +33,6 @@ namespace AMDiS { ...@@ -32,7 +33,6 @@ namespace AMDiS {
FUNCNAME("ParMetisMesh::ParMetisMesh()"); FUNCNAME("ParMetisMesh::ParMetisMesh()");
int mpiSize = mpiComm->Get_size(); int mpiSize = mpiComm->Get_size();
int nodeCounter = 0;
int elementCounter = 0; int elementCounter = 0;
int dow = Global::getGeo(WORLD); int dow = Global::getGeo(WORLD);
...@@ -51,7 +51,7 @@ namespace AMDiS { ...@@ -51,7 +51,7 @@ namespace AMDiS {
// allocate memory // allocate memory
eptr = new int[nElements + 1]; eptr = new int[nElements + 1];
eind = new int[nElements * (dim + 1)]; eind = new int[nElements * (mesh->getGeo(VERTEX))];
elmdist = new int[mpiSize + 1]; elmdist = new int[mpiSize + 1];
elem_p2a = new int[nElements]; elem_p2a = new int[nElements];
...@@ -74,10 +74,11 @@ namespace AMDiS { ...@@ -74,10 +74,11 @@ namespace AMDiS {
elmdist[i] += elmdist[i - 1]; elmdist[i] += elmdist[i - 1];
// traverse mesh and fill distributed ParMETIS data // traverse mesh and fill distributed ParMETIS data
DimVec<double> bary(dim, DEFAULT_VALUE, 1.0 / (dim + 1)); DimVec<double> bary(dim, DEFAULT_VALUE, 1.0 / mesh->getGeo(VERTEX));
WorldVector<double> world; WorldVector<double> world;
elementCounter = 0; elementCounter = 0;
int nodeCounter = 0;
elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL | Mesh::FILL_COORDS); elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL | Mesh::FILL_COORDS);
while (elInfo) { while (elInfo) {
...@@ -91,19 +92,16 @@ namespace AMDiS { ...@@ -91,19 +92,16 @@ namespace AMDiS {
setAMDiSIndex(elementCounter, index); setAMDiSIndex(elementCounter, index);
// write eptr entry // write eptr entry
nodeCounter += dim + 1; nodeCounter += mesh->getGeo(VERTEX);
*ptr_eptr = nodeCounter; *ptr_eptr = nodeCounter;
ptr_eptr++; ptr_eptr++;
// write eind entries (element nodes) // write eind entries (element nodes)
for (int i = 0; i < dim + 1; i++) { for (int i = 0; i < dim + 1; i++) {
if (mapLocalGlobal) { if (mapLocalGlobal)
TEST_EXIT_DBG(mapLocalGlobal->count(element->getDof(i, 0)))
("Should not happen!\n");
*ptr_eind = (*mapLocalGlobal)[element->getDof(i, 0)]; *ptr_eind = (*mapLocalGlobal)[element->getDof(i, 0)];
} else { else
*ptr_eind = element->getDof(i, 0); *ptr_eind = element->getDof(i, 0);
}
ptr_eind++; ptr_eind++;
} }
...@@ -112,7 +110,7 @@ namespace AMDiS { ...@@ -112,7 +110,7 @@ namespace AMDiS {
if (ptr_xyz) { if (ptr_xyz) {
elInfo->coordToWorld(bary, world); elInfo->coordToWorld(bary, world);
for (int i = 0; i < dim; i++) { for (int i = 0; i < dim; i++) {
*ptr_xyz = static_cast<float>(world[i]); *ptr_xyz = static_cast<float>(world[i]);
ptr_xyz++; ptr_xyz++;
} }
} }
...@@ -179,6 +177,50 @@ namespace AMDiS { ...@@ -179,6 +177,50 @@ namespace AMDiS {
} }
void ParMetisGraph::print()
{
FUNCNAME("ParMetisGraph::print()");
std::stringstream oss;
for (int i = 0; i <= MPI::COMM_WORLD.Get_size(); i++)
oss << parMetisMesh->getElementDist()[i] << " ";
MSG("Element dist = %s\n", oss.str().c_str());
int mpiRank = MPI::COMM_WORLD.Get_rank();
int nElements = parMetisMesh->getElementDist()[mpiRank + 1] -
parMetisMesh->getElementDist()[mpiRank];
MSG("nElements = %d in index range %d - %d\n",
nElements,
parMetisMesh->getElementDist()[mpiRank],
parMetisMesh->getElementDist()[mpiRank + 1]);
oss.str("");
oss.clear();
for (int i = 0; i <= nElements; i++)
oss << xadj[i] << ", ";
MSG("xadj = {%s}\n", oss.str().c_str());
oss.str("");
oss.clear();
for (int i = 0; i <= xadj[nElements] - 1; i++)
oss << adjncy[i] << ", ";
MSG("adjncy = {%s}\n", oss.str().c_str());
}
ParMetisPartitioner::~ParMetisPartitioner()
{
if (parMetisMesh)
delete parMetisMesh;
}
void ParMetisPartitioner::createPartitionData() void ParMetisPartitioner::createPartitionData()
{ {
FUNCNAME("ParMetrisPartitioner::createPartitionData()"); FUNCNAME("ParMetrisPartitioner::createPartitionData()");
...@@ -218,7 +260,9 @@ namespace AMDiS { ...@@ -218,7 +260,9 @@ namespace AMDiS {
int mpiSize = mpiComm->Get_size(); int mpiSize = mpiComm->Get_size();
// === create parmetis mesh ===
// === Create parmetis mesh ===
if (parMetisMesh) if (parMetisMesh)
delete parMetisMesh; delete parMetisMesh;
...@@ -228,7 +272,9 @@ namespace AMDiS { ...@@ -228,7 +272,9 @@ namespace AMDiS {
int nElements = parMetisMesh->getNumElements(); int nElements = parMetisMesh->getNumElements();
// === create weight array ===
// === Create weight array ===
std::vector<int> wgts(nElements); std::vector<int> wgts(nElements);
std::vector<float> floatWgts(nElements); std::vector<float> floatWgts(nElements);
unsigned int floatWgtsPos = 0; unsigned int floatWgtsPos = 0;
...@@ -257,10 +303,14 @@ namespace AMDiS { ...@@ -257,10 +303,14 @@ namespace AMDiS {
mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX); mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX);
maxWgt = tmp; maxWgt = tmp;
// === create dual graph ===
// === Create dual graph ===
ParMetisGraph parMetisGraph(parMetisMesh, mpiComm); ParMetisGraph parMetisGraph(parMetisMesh, mpiComm);
// === partitioning of dual graph ===
// === Partitioning of dual graph ===
int wgtflag = 2; // weights at vertices only! int wgtflag = 2; // weights at vertices only!
int numflag = 0; // c numbering style! int numflag = 0; // c numbering style!
int ncon = 1; // one weight at each vertex! int ncon = 1; // one weight at each vertex!
...@@ -275,11 +325,31 @@ namespace AMDiS { ...@@ -275,11 +325,31 @@ namespace AMDiS {
// set tpwgts // set tpwgts
for (int i = 0; i < mpiSize; i++) for (int i = 0; i < mpiSize; i++)
tpwgts[i] = 1.0 / nparts; tpwgts[i] = 1.0 / nparts;
float scale = 10000.0 / maxWgt; float scale = 10000.0 / maxWgt;
// scale wgts
for (int i = 0; i < nElements; i++)
// === Scale element weights. ===
double weightSum = 0.0;
for (int i = 0; i < nElements; i++) {
wgts[i] = static_cast<int>(floatWgts[i] * scale); wgts[i] = static_cast<int>(floatWgts[i] * scale);
weightSum += wgts[i];
}
mpi::globalAdd(weightSum);
weightSum /= mpiSize;
for (int i = 0; i < nElements; i++) {
if (wgts[i] > weightSum) {
MSG("ICH HABE EINEN %d > %f\n", wgts[i], weightSum);
wgts[i] = static_cast<int>(weightSum);
}
}
// === Start ParMETIS. ===
MPI_Comm tmpComm = MPI_Comm(*mpiComm); MPI_Comm tmpComm = MPI_Comm(*mpiComm);
...@@ -348,7 +418,9 @@ namespace AMDiS { ...@@ -348,7 +418,9 @@ namespace AMDiS {
ERROR_EXIT("unknown partitioning mode\n"); ERROR_EXIT("unknown partitioning mode\n");
} }
// === distribute new partition data ===
// === Distribute new partition data. ===
distributePartitioning(&(part[0])); distributePartitioning(&(part[0]));
} }
...@@ -357,7 +429,7 @@ namespace AMDiS { ...@@ -357,7 +429,7 @@ namespace AMDiS {
{ {
FUNCNAME("ParMetisPartitioner::fillCoarsePartitionVec()"); FUNCNAME("ParMetisPartitioner::fillCoarsePartitionVec()");
TEST_EXIT_DBG(partitionVec)("no partition vector\n"); TEST_EXIT_DBG(partitionVec)("No partition vector!\n");
partitionVec->clear(); partitionVec->clear();
......
...@@ -158,6 +158,8 @@ namespace AMDiS { ...@@ -158,6 +158,8 @@ namespace AMDiS {
return adjncy; return adjncy;
} }
void print();
protected: protected:
ParMetisMesh *parMetisMesh; ParMetisMesh *parMetisMesh;
...@@ -177,6 +179,8 @@ namespace AMDiS { ...@@ -177,6 +179,8 @@ namespace AMDiS {
mapLocalGlobal(NULL) mapLocalGlobal(NULL)
{} {}
~ParMetisPartitioner();
void partition(std::map<int, double> &elemWeights, void partition(std::map<int, double> &elemWeights,
PartitionMode mode = INITIAL, PartitionMode mode = INITIAL,
float itr = 1000000.0); float itr = 1000000.0);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment