Commit f9a324f2 authored by Thomas Witkowski's avatar Thomas Witkowski

Some code refactoring of the last changes. Added some more comments to parallel code.

parent 8d3f6ec7
...@@ -70,8 +70,8 @@ namespace AMDiS { ...@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile(false), writeSerializationFile(false),
repartitioningAllowed(false), repartitioningAllowed(false),
repartitionIthChange(20), repartitionIthChange(20),
nTimestepsAfterLastRepartitioning(0), nMeshChangesAfterLastRepartitioning(0),
repartCounter(0), repartitioningCounter(0),
debugOutputDir(""), debugOutputDir(""),
lastMeshChangeIndex(0) lastMeshChangeIndex(0)
{ {
...@@ -168,9 +168,7 @@ namespace AMDiS { ...@@ -168,9 +168,7 @@ namespace AMDiS {
if (writePartMesh > 0) { if (writePartMesh > 0) {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu"); debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
writePartitioningMesh(debugOutputDir + "part.vtu"); ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
} else {
MSG("Skip write part mesh!\n");
} }
} }
#endif #endif
...@@ -607,7 +605,7 @@ namespace AMDiS { ...@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. === // === Check on all ranks if at least one rank's mesh has changed. ===
int sendValue = static_cast<int>(!meshChanged); int sendValue = static_cast<int>(meshChanged);
recvAllValues = 0; recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM); mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
...@@ -635,13 +633,12 @@ namespace AMDiS { ...@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. === // === The mesh has changed, so check if it is required to repartition the mesh. ===
nTimestepsAfterLastRepartitioning++; nMeshChangesAfterLastRepartitioning++;
if (repartitioningAllowed) { if (repartitioningAllowed &&
if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) { nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh(); repartitionMesh();
nTimestepsAfterLastRepartitioning = 0; nMeshChangesAfterLastRepartitioning = 0;
}
} }
} }
...@@ -671,7 +668,7 @@ namespace AMDiS { ...@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. === // === Compare received mesh structure codes. ===
bool meshFitTogether = true; bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) { for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
...@@ -679,7 +676,7 @@ namespace AMDiS { ...@@ -679,7 +676,7 @@ namespace AMDiS {
int i = 0; int i = 0;
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt, i++) {
MeshStructure elCode; MeshStructure elCode;
elCode.init(boundIt->rankObj); elCode.init(boundIt->rankObj);
...@@ -688,23 +685,18 @@ namespace AMDiS { ...@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n"); TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace); MeshManipulation meshManipulation(feSpace);
bool b = meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i], meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el, boundIt->rankObj.el,
boundIt->rankObj.subObj, boundIt->rankObj.subObj,
boundIt->rankObj.ithObj, boundIt->rankObj.ithObj,
boundIt->rankObj.elType, boundIt->rankObj.elType,
boundIt->rankObj.reverseMode); boundIt->rankObj.reverseMode);
if (b)
meshFitTogether = false;
} }
i++;
} }
} }
return meshFitTogether; return meshChanged;
} }
...@@ -809,10 +801,15 @@ namespace AMDiS { ...@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1) TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n"); ("Only meshes with one DOFAdmin are supported!\n");
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
// === First we check if the rank with the maximum number of DOFs has at ===
// === least 20% more DOFs than the rank with the minimum number of DOFs. ===
// === In this case, the mesh will be repartition. ===
int repartitioning = 0; int repartitioning = 0;
vector<int> nDofsInRank(mpiSize); vector<int> nDofsInRank(mpiSize);
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0); mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
if (mpiRank == 0) { if (mpiRank == 0) {
...@@ -828,7 +825,7 @@ namespace AMDiS { ...@@ -828,7 +825,7 @@ namespace AMDiS {
MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n", MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n",
nOverallDofs, minDofs, maxDofs); nOverallDofs, minDofs, maxDofs);
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs)) if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2)
repartitioning = 1; repartitioning = 1;
mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0); mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
...@@ -841,25 +838,19 @@ namespace AMDiS { ...@@ -841,25 +838,19 @@ namespace AMDiS {
return; return;
double timePoint = MPI::Wtime(); double timePoint = MPI::Wtime();
repartitioningCounter++;
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh); ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) { if (repartitioningCounter == 0)
stringstream oss; ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu"; repartitioningCounter, feSpace);
MSG("Write partitioning to %s\n", oss.str().c_str());
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
}
#endif #endif
// === Create new element weights. ===
elemWeights.clear(); elemWeights.clear();
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL); ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) { while (elInfo) {
...@@ -867,15 +858,16 @@ namespace AMDiS { ...@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
// === Run ParMETiS to calculate a new mesh partitioning. ===
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs); partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed = bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0); partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
if (!partitioningSucceed) { if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n"); MSG("ParMETIS created empty partition!\n");
return; return;
} }
oldPartitionVec = partitionVec; oldPartitionVec = partitionVec;
...@@ -1045,32 +1037,17 @@ namespace AMDiS { ...@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh->dofCompress(); mesh->dofCompress();
#if (DEBUG != 0)
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif
partitioner->fillCoarsePartitionVec(&partitionVec); partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo(); updateInteriorBoundaryInfo();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
#if (DEBUG != 0) #if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n"); MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
ParallelDebug::testAllElements(*this); ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::printBoundaryInfo(*this); ParallelDebug::printBoundaryInfo(*this);
...@@ -1096,6 +1073,10 @@ namespace AMDiS { ...@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects.createRankData(partitionVec); elObjects.createRankData(partitionVec);
createBoundaryData(); createBoundaryData();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
} }
...@@ -1872,8 +1853,8 @@ namespace AMDiS { ...@@ -1872,8 +1853,8 @@ namespace AMDiS {
for (int i = 0; i < nSize; i++) for (int i = 0; i < nSize; i++)
allMacroElements[i]->serialize(out); allMacroElements[i]->serialize(out);
SerUtil::serialize(out, nTimestepsAfterLastRepartitioning); SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
SerUtil::serialize(out, repartCounter); SerUtil::serialize(out, repartitioningCounter);
} }
...@@ -1939,8 +1920,8 @@ namespace AMDiS { ...@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements[i]->getElement()->setMesh(mesh); allMacroElements[i]->getElement()->setMesh(mesh);
} }
SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning); SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
SerUtil::deserialize(in, repartCounter); SerUtil::deserialize(in, repartitioningCounter);
deserialized = true; deserialized = true;
} }
...@@ -2014,23 +1995,4 @@ namespace AMDiS { ...@@ -2014,23 +1995,4 @@ namespace AMDiS {
} }
} }
void MeshDistributor::writePartitioningMesh(string filename)
{
FUNCNAME("MeshDistributor::writePartitioningMesh()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, mesh, filename);
}
} }
...@@ -303,11 +303,13 @@ namespace AMDiS { ...@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements, * refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function * it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached. * terminates. Hence, it must be called until a stable mesh refinement is reached.
* If the mesh has been changed by this function, it returns true. Otherwise, it
* returns false, i.e., the given interior boundaries fit together on both sides.
* *
* \param[in] allBound Defines a map from rank to interior boundaries which * \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked. * should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/ */
bool checkAndAdaptBoundary(RankToBoundMap &allBound); bool checkAndAdaptBoundary(RankToBoundMap &allBound);
...@@ -318,13 +320,6 @@ namespace AMDiS { ...@@ -318,13 +320,6 @@ namespace AMDiS {
*/ */
void repartitionMesh(); void repartitionMesh();
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of. This function can be used for
* debugging.
*/
void writePartitioningMesh(string filename);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems. /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs(); void setRankDofs();
...@@ -550,11 +545,15 @@ namespace AMDiS { ...@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations. /// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed; bool repartitioningAllowed;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange; int repartitionIthChange;
int nTimestepsAfterLastRepartitioning; /// Counts the number of mesh changes after the last mesh repartitioning was done.
int nMeshChangesAfterLastRepartitioning;
int repartCounter; /// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int repartitioningCounter;
/// Directory name where all debug output files should be written to. /// Directory name where all debug output files should be written to.
string debugOutputDir; string debugOutputDir;
......
...@@ -247,8 +247,7 @@ namespace AMDiS { ...@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n"); TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode); return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return meshChanged;
} }
......
...@@ -54,6 +54,9 @@ namespace AMDiS { ...@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given * \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where * in reverse mode, i.e., left and right children where
* changed when the code was created. * changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/ */
bool startFitElementToMeshCode(MeshStructure &code, bool startFitElementToMeshCode(MeshStructure &code,
Element *el, Element *el,
...@@ -77,6 +80,9 @@ namespace AMDiS { ...@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given * \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where * in reverse mode, i.e., left and right children where
* changed when the code was created. * changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/ */
bool fitElementToMeshCode(MeshStructure &code, bool fitElementToMeshCode(MeshStructure &code,
TraverseStack &stack, TraverseStack &stack,
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "StdMpi.h" #include "StdMpi.h"
#include "Debug.h" #include "Debug.h"
#include "MpiHelper.h" #include "MpiHelper.h"
#include "io/VtkWriter.h"
namespace AMDiS { namespace AMDiS {
...@@ -806,4 +807,39 @@ namespace AMDiS { ...@@ -806,4 +807,39 @@ namespace AMDiS {
file.close(); file.close();
} }
void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
{
FUNCNAME("ParallelDebug::writeParitioning()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = pdb.partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, pdb.mesh, filename);
}
void ParallelDebug::writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace)
{
FUNCNAME("ParallelDebug::writePartitioningFile()");
stringstream oss;
oss << filename;
if (counter >= 0)
oss << "-" << counter;
oss << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(MPI::COMM_WORLD.Get_rank());
VtkWriter::writeFile(&tmpa, oss.str());
}
} }
...@@ -142,6 +142,25 @@ namespace AMDiS { ...@@ -142,6 +142,25 @@ namespace AMDiS {
static void writeDebugFile(MeshDistributor &pdb, static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix); std::string prefix, std::string postfix);
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of.
*/
static void writePartitioning(MeshDistributor &pdb, string filename);
/** \brief
* The mesh is written to a value and all values are assigned by the rank number
* where the vertex is contained in.
*
* \param[in] filename Name of the output file without extension (.vtu).
* \param[in] counter Counter index. If not negative, this number is added
* to the filename.
* \param[in] feSpace
*/
static void writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace);
}; };
} // namespace AMDiS } // namespace AMDiS
......
...@@ -30,12 +30,12 @@ namespace AMDiS { ...@@ -30,12 +30,12 @@ namespace AMDiS {
vm /= 1024.0; vm /= 1024.0;
rss /= 1024.0; rss /= 1024.0;
MSG("My memory usage is VM = %f MB RSS = %f MB\n", vm, rss); MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm); mpi::globalAdd(vm);
mpi::globalAdd(rss); mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %f MB RSS = %f MB\n", vm, rss); MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment