diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 5a477d3d1e4344f7e3fcc86f947f9581939340a8..d7e59ccc008cdd610fd2e6d92109cabc0f8966f9 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile(false),
repartitioningAllowed(false),
repartitionIthChange(20),
- nTimestepsAfterLastRepartitioning(0),
- repartCounter(0),
+ nMeshChangesAfterLastRepartitioning(0),
+ repartitioningCounter(0),
debugOutputDir(""),
lastMeshChangeIndex(0)
{
@@ -168,9 +168,7 @@ namespace AMDiS {
if (writePartMesh > 0) {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
- writePartitioningMesh(debugOutputDir + "part.vtu");
- } else {
- MSG("Skip write part mesh!\n");
+ ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
}
}
#endif
@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. ===
- int sendValue = static_cast<int>(!meshChanged);
+ int sendValue = static_cast<int>(meshChanged);
recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
- nTimestepsAfterLastRepartitioning++;
+ nMeshChangesAfterLastRepartitioning++;
- if (repartitioningAllowed) {
- if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) {
- repartitionMesh();
- nTimestepsAfterLastRepartitioning = 0;
- }
+ if (repartitioningAllowed &&
+ nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
+ repartitionMesh();
+ nMeshChangesAfterLastRepartitioning = 0;
}
}
@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. ===
- bool meshFitTogether = true;
+ bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
@@ -679,7 +676,7 @@ namespace AMDiS {
int i = 0;
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
- boundIt != it->second.end(); ++boundIt) {
+ boundIt != it->second.end(); ++boundIt, i++) {
MeshStructure elCode;
elCode.init(boundIt->rankObj);
@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace);
- bool b =
+ meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->rankObj.reverseMode);
-
- if (b)
- meshFitTogether = false;
}
-
- i++;
}
}
- return meshFitTogether;
+ return meshChanged;
}
@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n");
- int nDofs = mesh->getDofAdmin(0).getUsedDofs();
+
+ // === First we check if the rank with the maximum number of DOFs has at ===
+ // === least 20% more DOFs than the rank with the minimum number of DOFs. ===
+ // === In this case, the mesh will be repartition. ===
+
int repartitioning = 0;
vector<int> nDofsInRank(mpiSize);
+ int nDofs = mesh->getDofAdmin(0).getUsedDofs();
mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
if (mpiRank == 0) {
@@ -828,7 +825,7 @@ namespace AMDiS {
MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n",
nOverallDofs, minDofs, maxDofs);
- if (static_cast<double>(maxDofs) / static_cast<double>(minDofs))
+ if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2)
repartitioning = 1;
mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
@@ -841,25 +838,19 @@ namespace AMDiS {
return;
double timePoint = MPI::Wtime();
+ repartitioningCounter++;
#if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh);
- if (repartCounter == 0) {
- stringstream oss;
- oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
- MSG("Write partitioning to %s\n", oss.str().c_str());
- DOFVector<double> tmpa(feSpace, "tmp");
- tmpa.set(mpiRank);
- VtkWriter::writeFile(&tmpa, oss.str());
-
- repartCounter++;
- }
+ if (repartitioningCounter == 0)
+ ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
+ repartitioningCounter, feSpace);
#endif
+ // === Create new element weights. ===
elemWeights.clear();
-
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
+
+ // === Run ParMETiS to calculate a new mesh partitioning. ===
+
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
-
if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n");
return;
}
-
oldPartitionVec = partitionVec;
@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh->dofCompress();
-#if (DEBUG != 0)
- stringstream oss;
- oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
- DOFVector<double> tmpa(feSpace, "tmp");
- tmpa.set(mpiRank);
- VtkWriter::writeFile(&tmpa, oss.str());
- repartCounter++;
-
- ParallelDebug::testAllElements(*this);
- ParallelDebug::testDoubleDofs(mesh);
-#endif
-
partitioner->fillCoarsePartitionVec(&partitionVec);
-
updateInteriorBoundaryInfo();
-
-#if (DEBUG != 0)
- ParallelDebug::printBoundaryInfo(*this);
-#endif
-
updateLocalGlobalNumbering();
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
+ ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
+ repartitioningCounter, feSpace);
ParallelDebug::testAllElements(*this);
+ ParallelDebug::testDoubleDofs(mesh);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::printBoundaryInfo(*this);
@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects.createRankData(partitionVec);
createBoundaryData();
+
+#if (DEBUG != 0)
+ ParallelDebug::printBoundaryInfo(*this);
+#endif
}
@@ -1872,8 +1853,8 @@ namespace AMDiS {
for (int i = 0; i < nSize; i++)
allMacroElements[i]->serialize(out);
- SerUtil::serialize(out, nTimestepsAfterLastRepartitioning);
- SerUtil::serialize(out, repartCounter);
+ SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
+ SerUtil::serialize(out, repartitioningCounter);
}
@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements[i]->getElement()->setMesh(mesh);
}
- SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning);
- SerUtil::deserialize(in, repartCounter);
+ SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
+ SerUtil::deserialize(in, repartitioningCounter);
deserialized = true;
}
@@ -2014,23 +1995,4 @@ namespace AMDiS {
}
}
-
- void MeshDistributor::writePartitioningMesh(string filename)
- {
- FUNCNAME("MeshDistributor::writePartitioningMesh()");
-
- map<int, double> vec;
- TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1,
- Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
-
- while (elInfo) {
- int index = elInfo->getElement()->getIndex();
- vec[index] = partitionVec[index];
- elInfo = stack.traverseNext(elInfo);
- }
-
- ElementFileWriter::writeFile(vec, mesh, filename);
- }
-
}
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index 6b8c3559611d26c9f00772da2b9fca0f5d5a7616..c9787946ceecfa88fe765a571a75013a569e324f 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached.
- * If the mesh has been changed by this function, it returns true. Otherwise, it
- * returns false, i.e., the given interior boundaries fit together on both sides.
*
* \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked.
+ *
+ * \return If the mesh has been changed by this function, it returns true.
+ * Otherwise, it returns false, i.e., the given interior boundaries
+ * fit together on both sides.
*/
bool checkAndAdaptBoundary(RankToBoundMap &allBound);
@@ -318,13 +320,6 @@ namespace AMDiS {
*/
void repartitionMesh();
- /** \brief
- * This functions create a Paraview file with the macro mesh where the elements
- * are colored by the partition they are part of. This function can be used for
- * debugging.
- */
- void writePartitioningMesh(string filename);
-
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs();
@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
+ /// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange;
- int nTimestepsAfterLastRepartitioning;
+ /// Counts the number of mesh changes after the last mesh repartitioning was done.
+ int nMeshChangesAfterLastRepartitioning;
- int repartCounter;
+ /// Countes the number of mesh repartitions that were done. Till now, this
+ /// variable is used only for debug outputs.
+ int repartitioningCounter;
/// Directory name where all debug output files should be written to.
string debugOutputDir;
diff --git a/AMDiS/src/parallel/MeshManipulation.cc b/AMDiS/src/parallel/MeshManipulation.cc
index 166e2f0a76a828c793e7e6808180282a7a7638d8..04c7b7e0cda067edd8e34ae418fe92cc45178886 100644
--- a/AMDiS/src/parallel/MeshManipulation.cc
+++ b/AMDiS/src/parallel/MeshManipulation.cc
@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
- meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
- return meshChanged;
+ return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
}
diff --git a/AMDiS/src/parallel/MeshManipulation.h b/AMDiS/src/parallel/MeshManipulation.h
index cac397290809f5322a293c6a1351136461b3f988..c85813ffd7efba0db64ffd20c0def6f0cdbb1b11 100644
--- a/AMDiS/src/parallel/MeshManipulation.h
+++ b/AMDiS/src/parallel/MeshManipulation.h
@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
+ *
+ * \return Returns true, if the mesh was changed, i.e. refined, to fit the
+ * element to the structure code.
*/
bool startFitElementToMeshCode(MeshStructure &code,
Element *el,
@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
+ *
+ * \return Returns true, if the mesh was changed, i.e. refined, to fit the
+ * element to the structure code.
*/
bool fitElementToMeshCode(MeshStructure &code,
TraverseStack &stack,
diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc
index 0d27d777e14f4e0d400c2b6fa1fdd08dc78b2c9d..1e1dced44fc895fb3459dcbedb0a775b85b566b8 100644
--- a/AMDiS/src/parallel/ParallelDebug.cc
+++ b/AMDiS/src/parallel/ParallelDebug.cc
@@ -18,6 +18,7 @@
#include "StdMpi.h"
#include "Debug.h"
#include "MpiHelper.h"
+#include "io/VtkWriter.h"
namespace AMDiS {
@@ -806,4 +807,39 @@ namespace AMDiS {
file.close();
}
+
+ void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
+ {
+ FUNCNAME("ParallelDebug::writeParitioning()");
+
+ map<int, double> vec;
+ TraverseStack stack;
+ ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1,
+ Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
+
+ while (elInfo) {
+ int index = elInfo->getElement()->getIndex();
+ vec[index] = pdb.partitionVec[index];
+ elInfo = stack.traverseNext(elInfo);
+ }
+
+ ElementFileWriter::writeFile(vec, pdb.mesh, filename);
+ }
+
+ void ParallelDebug::writePartitioningFile(std::string filename,
+ int counter,
+ FiniteElemSpace *feSpace)
+ {
+ FUNCNAME("ParallelDebug::writePartitioningFile()");
+
+ stringstream oss;
+ oss << filename;
+ if (counter >= 0)
+ oss << "-" << counter;
+ oss << ".vtu";
+
+ DOFVector<double> tmpa(feSpace, "tmp");
+ tmpa.set(MPI::COMM_WORLD.Get_rank());
+ VtkWriter::writeFile(&tmpa, oss.str());
+ }
}
diff --git a/AMDiS/src/parallel/ParallelDebug.h b/AMDiS/src/parallel/ParallelDebug.h
index 1dc7e315607c95aa8a9f22c31d2353ca19b53997..126cdd091c544f9eca8fcdbf56008e366a39f2f1 100644
--- a/AMDiS/src/parallel/ParallelDebug.h
+++ b/AMDiS/src/parallel/ParallelDebug.h
@@ -142,6 +142,25 @@ namespace AMDiS {
static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix);
+
+ /** \brief
+ * This functions create a Paraview file with the macro mesh where the elements
+ * are colored by the partition they are part of.
+ */
+ static void writePartitioning(MeshDistributor &pdb, string filename);
+
+ /** \brief
+ * The mesh is written to a value and all values are assigned by the rank number
+ * where the vertex is contained in.
+ *
+ * \param[in] filename Name of the output file without extension (.vtu).
+ * \param[in] counter Counter index. If not negative, this number is added
+ * to the filename.
+ * \param[in] feSpace
+ */
+ static void writePartitioningFile(std::string filename,
+ int counter,
+ FiniteElemSpace *feSpace);
};
} // namespace AMDiS
diff --git a/AMDiS/src/parallel/ParallelProblemStatBase.cc b/AMDiS/src/parallel/ParallelProblemStatBase.cc
index d7d95dc233b014a879e52146984f793671a1444b..d639f40376851e6d0375182ff0abe4083de48226 100644
--- a/AMDiS/src/parallel/ParallelProblemStatBase.cc
+++ b/AMDiS/src/parallel/ParallelProblemStatBase.cc
@@ -30,12 +30,12 @@ namespace AMDiS {
vm /= 1024.0;
rss /= 1024.0;
- MSG("My memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
+ MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
- MSG("Overall memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
+ MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
}