Skip to content
GitLab
Commit f9a324f2 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Some code refactoring of the last changes. Added some more comments to parallel code.

parent 8d3f6ec7
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/MeshDistributor.cc
View file @ f9a324f2
......@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile(false),
repartitioningAllowed(false),
repartitionIthChange(20),
nTimestepsAfterLastRepartitioning(0),
repartCounter(0),
nMeshChangesAfterLastRepartitioning(0),
repartitioningCounter(0),
debugOutputDir(""),
lastMeshChangeIndex(0)
{
......@@ -168,9 +168,7 @@ namespace AMDiS {
if (writePartMesh > 0) {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
writePartitioningMesh(debugOutputDir + "part.vtu");
} else {
MSG("Skip write part mesh!\n");
ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
}
}
#endif
......@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. ===
int sendValue = static_cast<int>(!meshChanged);
int sendValue = static_cast<int>(meshChanged);
recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
......@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
nTimestepsAfterLastRepartitioning++;
nMeshChangesAfterLastRepartitioning++;
if (repartitioningAllowed) {
if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh();
nTimestepsAfterLastRepartitioning = 0;
}
if (repartitioningAllowed &&
nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh();
nMeshChangesAfterLastRepartitioning = 0;
}
}
......@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. ===
bool meshFitTogether = true;
bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
......@@ -679,7 +676,7 @@ namespace AMDiS {
int i = 0;
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) {
boundIt != it->second.end(); ++boundIt, i++) {
MeshStructure elCode;
elCode.init(boundIt->rankObj);
......@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace);
bool b =
meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->rankObj.reverseMode);
if (b)
meshFitTogether = false;
}
i++;
}
}
return meshFitTogether;
return meshChanged;
}
......@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n");
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
// === First we check if the rank with the maximum number of DOFs has at ===
// === least 20% more DOFs than the rank with the minimum number of DOFs. ===
// === In this case, the mesh will be repartition. ===
int repartitioning = 0;
vector<int> nDofsInRank(mpiSize);
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
if (mpiRank == 0) {
......@@ -828,7 +825,7 @@ namespace AMDiS {
MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n",
nOverallDofs, minDofs, maxDofs);
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs))
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2)
repartitioning = 1;
mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
......@@ -841,25 +838,19 @@ namespace AMDiS {
return;
double timePoint = MPI::Wtime();
repartitioningCounter++;
#if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) {
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
MSG("Write partitioning to %s\n", oss.str().c_str());
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
}
if (repartitioningCounter == 0)
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
#endif
// === Create new element weights. ===
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
......@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
// === Run ParMETiS to calculate a new mesh partitioning. ===
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n");
return;
}
oldPartitionVec = partitionVec;
......@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh->dofCompress();
#if (DEBUG != 0)
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
updateLocalGlobalNumbering();
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::printBoundaryInfo(*this);
......@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects.createRankData(partitionVec);
createBoundaryData();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
}
......@@ -1872,8 +1853,8 @@ namespace AMDiS {
for (int i = 0; i < nSize; i++)
allMacroElements[i]->serialize(out);
SerUtil::serialize(out, nTimestepsAfterLastRepartitioning);
SerUtil::serialize(out, repartCounter);
SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
SerUtil::serialize(out, repartitioningCounter);
}
......@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements[i]->getElement()->setMesh(mesh);
}
SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning);
SerUtil::deserialize(in, repartCounter);
SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
SerUtil::deserialize(in, repartitioningCounter);
deserialized = true;
}
......@@ -2014,23 +1995,4 @@ namespace AMDiS {
}
}
void MeshDistributor::writePartitioningMesh(string filename)
{
FUNCNAME("MeshDistributor::writePartitioningMesh()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, mesh, filename);
}
}
AMDiS/src/parallel/MeshDistributor.h
View file @ f9a324f2
......@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached.
* If the mesh has been changed by this function, it returns true. Otherwise, it
* returns false, i.e., the given interior boundaries fit together on both sides.
*
* \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/
bool checkAndAdaptBoundary(RankToBoundMap &allBound);
......@@ -318,13 +320,6 @@ namespace AMDiS {
*/
void repartitionMesh();
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of. This function can be used for
* debugging.
*/
void writePartitioningMesh(string filename);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs();
......@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange;
int nTimestepsAfterLastRepartitioning;
/// Counts the number of mesh changes after the last mesh repartitioning was done.
int nMeshChangesAfterLastRepartitioning;
int repartCounter;
/// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int repartitioningCounter;
/// Directory name where all debug output files should be written to.
string debugOutputDir;
......
AMDiS/src/parallel/MeshManipulation.cc
View file @ f9a324f2
......@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return meshChanged;
return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
}
......
AMDiS/src/parallel/MeshManipulation.h
View file @ f9a324f2
......@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool startFitElementToMeshCode(MeshStructure &code,
Element *el,
......@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool fitElementToMeshCode(MeshStructure &code,
TraverseStack &stack,
......
AMDiS/src/parallel/ParallelDebug.cc
View file @ f9a324f2
......@@ -18,6 +18,7 @@
#include "StdMpi.h"
#include "Debug.h"
#include "MpiHelper.h"
#include "io/VtkWriter.h"
namespace AMDiS {
......@@ -806,4 +807,39 @@ namespace AMDiS {
file.close();
}
void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
{
FUNCNAME("ParallelDebug::writeParitioning()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = pdb.partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, pdb.mesh, filename);
}
void ParallelDebug::writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace)
{
FUNCNAME("ParallelDebug::writePartitioningFile()");
stringstream oss;
oss << filename;
if (counter >= 0)
oss << "-" << counter;
oss << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(MPI::COMM_WORLD.Get_rank());
VtkWriter::writeFile(&tmpa, oss.str());
}
}
AMDiS/src/parallel/ParallelDebug.h
View file @ f9a324f2
......@@ -142,6 +142,25 @@ namespace AMDiS {
static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix);
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of.
*/
static void writePartitioning(MeshDistributor &pdb, string filename);
/** \brief
* The mesh is written to a value and all values are assigned by the rank number
* where the vertex is contained in.
*
* \param[in] filename Name of the output file without extension (.vtu).
* \param[in] counter Counter index. If not negative, this number is added
* to the filename.
* \param[in] feSpace
*/
static void writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace);
};
} // namespace AMDiS
......
AMDiS/src/parallel/ParallelProblemStatBase.cc
View file @ f9a324f2
......@@ -30,12 +30,12 @@ namespace AMDiS {
vm /= 1024.0;
rss /= 1024.0;
MSG("My memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %f MB RSS = %f MB\n", vm, rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment