Commit f9a324f2 authored by Thomas Witkowski's avatar Thomas Witkowski

Some code refactoring of the last changes. Added some more comments to parallel code.

parent 8d3f6ec7
......@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile(false),
repartitioningAllowed(false),
repartitionIthChange(20),
nTimestepsAfterLastRepartitioning(0),
repartCounter(0),
nMeshChangesAfterLastRepartitioning(0),
repartitioningCounter(0),
debugOutputDir(""),
lastMeshChangeIndex(0)
{
......@@ -168,9 +168,7 @@ namespace AMDiS {
if (writePartMesh > 0) {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
writePartitioningMesh(debugOutputDir + "part.vtu");
} else {
MSG("Skip write part mesh!\n");
ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
}
}
#endif
......@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. ===
int sendValue = static_cast<int>(!meshChanged);
int sendValue = static_cast<int>(meshChanged);
recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
......@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
nTimestepsAfterLastRepartitioning++;
nMeshChangesAfterLastRepartitioning++;
if (repartitioningAllowed) {
if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh();
nTimestepsAfterLastRepartitioning = 0;
}
if (repartitioningAllowed &&
nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh();
nMeshChangesAfterLastRepartitioning = 0;
}
}
......@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. ===
bool meshFitTogether = true;
bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
......@@ -679,7 +676,7 @@ namespace AMDiS {
int i = 0;
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) {
boundIt != it->second.end(); ++boundIt, i++) {
MeshStructure elCode;
elCode.init(boundIt->rankObj);
......@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace);
bool b =
meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->rankObj.reverseMode);
if (b)
meshFitTogether = false;
}
i++;
}
}
return meshFitTogether;
return meshChanged;
}
......@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n");
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
// === First we check if the rank with the maximum number of DOFs has at ===
// === least 20% more DOFs than the rank with the minimum number of DOFs. ===
// === In this case, the mesh will be repartition. ===
int repartitioning = 0;
vector<int> nDofsInRank(mpiSize);
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
if (mpiRank == 0) {
......@@ -828,7 +825,7 @@ namespace AMDiS {
MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n",
nOverallDofs, minDofs, maxDofs);
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs))
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2)
repartitioning = 1;
mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
......@@ -841,25 +838,19 @@ namespace AMDiS {
return;
double timePoint = MPI::Wtime();
repartitioningCounter++;
#if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) {
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
MSG("Write partitioning to %s\n", oss.str().c_str());
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
}
if (repartitioningCounter == 0)
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
#endif
// === Create new element weights. ===
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
......@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
// === Run ParMETiS to calculate a new mesh partitioning. ===
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n");
return;
}
oldPartitionVec = partitionVec;
......@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh->dofCompress();
#if (DEBUG != 0)
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
updateLocalGlobalNumbering();
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::printBoundaryInfo(*this);
......@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects.createRankData(partitionVec);
createBoundaryData();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
}
......@@ -1872,8 +1853,8 @@ namespace AMDiS {
for (int i = 0; i < nSize; i++)
allMacroElements[i]->serialize(out);
SerUtil::serialize(out, nTimestepsAfterLastRepartitioning);
SerUtil::serialize(out, repartCounter);
SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
SerUtil::serialize(out, repartitioningCounter);
}
......@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements[i]->getElement()->setMesh(mesh);
}
SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning);
SerUtil::deserialize(in, repartCounter);
SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
SerUtil::deserialize(in, repartitioningCounter);
deserialized = true;
}
......@@ -2014,23 +1995,4 @@ namespace AMDiS {
}
}
void MeshDistributor::writePartitioningMesh(string filename)
{
FUNCNAME("MeshDistributor::writePartitioningMesh()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, mesh, filename);
}
}
......@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached.
* If the mesh has been changed by this function, it returns true. Otherwise, it
* returns false, i.e., the given interior boundaries fit together on both sides.
*
* \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/
bool checkAndAdaptBoundary(RankToBoundMap &allBound);
......@@ -318,13 +320,6 @@ namespace AMDiS {
*/
void repartitionMesh();
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of. This function can be used for
* debugging.
*/
void writePartitioningMesh(string filename);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs();
......@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange;
int nTimestepsAfterLastRepartitioning;
/// Counts the number of mesh changes after the last mesh repartitioning was done.
int nMeshChangesAfterLastRepartitioning;
int repartCounter;
/// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int repartitioningCounter;
/// Directory name where all debug output files should be written to.
string debugOutputDir;
......
......@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return meshChanged;
return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
}
......
......@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool startFitElementToMeshCode(MeshStructure &code,
Element *el,
......@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool fitElementToMeshCode(MeshStructure &code,
TraverseStack &stack,
......
......@@ -18,6 +18,7 @@
#include "StdMpi.h"
#include "Debug.h"
#include "MpiHelper.h"
#include "io/VtkWriter.h"
namespace AMDiS {
......@@ -806,4 +807,39 @@ namespace AMDiS {
file.close();
}
void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
{
FUNCNAME("ParallelDebug::writeParitioning()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = pdb.partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, pdb.mesh, filename);
}
void ParallelDebug::writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace)
{
FUNCNAME("ParallelDebug::writePartitioningFile()");
stringstream oss;
oss << filename;
if (counter >= 0)
oss << "-" << counter;
oss << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(MPI::COMM_WORLD.Get_rank());
VtkWriter::writeFile(&tmpa, oss.str());
}
}
......@@ -142,6 +142,25 @@ namespace AMDiS {
static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix);
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of.
*/
static void writePartitioning(MeshDistributor &pdb, string filename);
/** \brief
* The mesh is written to a value and all values are assigned by the rank number
* where the vertex is contained in.
*
* \param[in] filename Name of the output file without extension (.vtu).
* \param[in] counter Counter index. If not negative, this number is added
* to the filename.
* \param[in] feSpace
*/
static void writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace);
};
} // namespace AMDiS
......
......@@ -30,12 +30,12 @@ namespace AMDiS {
vm /= 1024.0;
rss /= 1024.0;
MSG("My memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment