Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit f9a324f2 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Some code refactoring of the last changes. Added some more comments to parallel code.

parent 8d3f6ec7
...@@ -70,8 +70,8 @@ namespace AMDiS { ...@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile(false), writeSerializationFile(false),
repartitioningAllowed(false), repartitioningAllowed(false),
repartitionIthChange(20), repartitionIthChange(20),
nTimestepsAfterLastRepartitioning(0), nMeshChangesAfterLastRepartitioning(0),
repartCounter(0), repartitioningCounter(0),
debugOutputDir(""), debugOutputDir(""),
lastMeshChangeIndex(0) lastMeshChangeIndex(0)
{ {
...@@ -168,9 +168,7 @@ namespace AMDiS { ...@@ -168,9 +168,7 @@ namespace AMDiS {
if (writePartMesh > 0) { if (writePartMesh > 0) {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu"); debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
writePartitioningMesh(debugOutputDir + "part.vtu"); ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
} else {
MSG("Skip write part mesh!\n");
} }
} }
#endif #endif
...@@ -607,7 +605,7 @@ namespace AMDiS { ...@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. === // === Check on all ranks if at least one rank's mesh has changed. ===
int sendValue = static_cast<int>(!meshChanged); int sendValue = static_cast<int>(meshChanged);
recvAllValues = 0; recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM); mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
...@@ -635,13 +633,12 @@ namespace AMDiS { ...@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. === // === The mesh has changed, so check if it is required to repartition the mesh. ===
nTimestepsAfterLastRepartitioning++; nMeshChangesAfterLastRepartitioning++;
if (repartitioningAllowed) { if (repartitioningAllowed &&
if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) { nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh(); repartitionMesh();
nTimestepsAfterLastRepartitioning = 0; nMeshChangesAfterLastRepartitioning = 0;
}
} }
} }
...@@ -671,7 +668,7 @@ namespace AMDiS { ...@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. === // === Compare received mesh structure codes. ===
bool meshFitTogether = true; bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) { for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
...@@ -679,7 +676,7 @@ namespace AMDiS { ...@@ -679,7 +676,7 @@ namespace AMDiS {
int i = 0; int i = 0;
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt, i++) {
MeshStructure elCode; MeshStructure elCode;
elCode.init(boundIt->rankObj); elCode.init(boundIt->rankObj);
...@@ -688,23 +685,18 @@ namespace AMDiS { ...@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n"); TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace); MeshManipulation meshManipulation(feSpace);
bool b = meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i], meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el, boundIt->rankObj.el,
boundIt->rankObj.subObj, boundIt->rankObj.subObj,
boundIt->rankObj.ithObj, boundIt->rankObj.ithObj,
boundIt->rankObj.elType, boundIt->rankObj.elType,
boundIt->rankObj.reverseMode); boundIt->rankObj.reverseMode);
if (b)
meshFitTogether = false;
} }
i++;
} }
} }
return meshFitTogether; return meshChanged;
} }
...@@ -809,10 +801,15 @@ namespace AMDiS { ...@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1) TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n"); ("Only meshes with one DOFAdmin are supported!\n");
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
// === First we check if the rank with the maximum number of DOFs has at ===
// === least 20% more DOFs than the rank with the minimum number of DOFs. ===
// === In this case, the mesh will be repartition. ===
int repartitioning = 0; int repartitioning = 0;
vector<int> nDofsInRank(mpiSize); vector<int> nDofsInRank(mpiSize);
int nDofs = mesh->getDofAdmin(0).getUsedDofs();
mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0); mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
if (mpiRank == 0) { if (mpiRank == 0) {
...@@ -828,7 +825,7 @@ namespace AMDiS { ...@@ -828,7 +825,7 @@ namespace AMDiS {
MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n", MSG("Overall DOFs: %d Min DOFs: %d Max DOFs: %d\n",
nOverallDofs, minDofs, maxDofs); nOverallDofs, minDofs, maxDofs);
if (static_cast<double>(maxDofs) / static_cast<double>(minDofs)) if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2)
repartitioning = 1; repartitioning = 1;
mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0); mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
...@@ -841,25 +838,19 @@ namespace AMDiS { ...@@ -841,25 +838,19 @@ namespace AMDiS {
return; return;
double timePoint = MPI::Wtime(); double timePoint = MPI::Wtime();
repartitioningCounter++;
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh); ParallelDebug::testDoubleDofs(mesh);
if (repartCounter == 0) { if (repartitioningCounter == 0)
stringstream oss; ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu"; repartitioningCounter, feSpace);
MSG("Write partitioning to %s\n", oss.str().c_str());
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
}
#endif #endif
// === Create new element weights. ===
elemWeights.clear(); elemWeights.clear();
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL); ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) { while (elInfo) {
...@@ -867,15 +858,16 @@ namespace AMDiS { ...@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
// === Run ParMETiS to calculate a new mesh partitioning. ===
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs); partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed = bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0); partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
if (!partitioningSucceed) { if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n"); MSG("ParMETIS created empty partition!\n");
return; return;
} }
oldPartitionVec = partitionVec; oldPartitionVec = partitionVec;
...@@ -1045,32 +1037,17 @@ namespace AMDiS { ...@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh->dofCompress(); mesh->dofCompress();
#if (DEBUG != 0)
stringstream oss;
oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif
partitioner->fillCoarsePartitionVec(&partitionVec); partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo(); updateInteriorBoundaryInfo();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
#if (DEBUG != 0) #if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n"); MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning",
repartitioningCounter, feSpace);
ParallelDebug::testAllElements(*this); ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::printBoundaryInfo(*this); ParallelDebug::printBoundaryInfo(*this);
...@@ -1096,6 +1073,10 @@ namespace AMDiS { ...@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects.createRankData(partitionVec); elObjects.createRankData(partitionVec);
createBoundaryData(); createBoundaryData();
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
} }
...@@ -1872,8 +1853,8 @@ namespace AMDiS { ...@@ -1872,8 +1853,8 @@ namespace AMDiS {
for (int i = 0; i < nSize; i++) for (int i = 0; i < nSize; i++)
allMacroElements[i]->serialize(out); allMacroElements[i]->serialize(out);
SerUtil::serialize(out, nTimestepsAfterLastRepartitioning); SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
SerUtil::serialize(out, repartCounter); SerUtil::serialize(out, repartitioningCounter);
} }
...@@ -1939,8 +1920,8 @@ namespace AMDiS { ...@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements[i]->getElement()->setMesh(mesh); allMacroElements[i]->getElement()->setMesh(mesh);
} }
SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning); SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
SerUtil::deserialize(in, repartCounter); SerUtil::deserialize(in, repartitioningCounter);
deserialized = true; deserialized = true;
} }
...@@ -2014,23 +1995,4 @@ namespace AMDiS { ...@@ -2014,23 +1995,4 @@ namespace AMDiS {
} }
} }
void MeshDistributor::writePartitioningMesh(string filename)
{
FUNCNAME("MeshDistributor::writePartitioningMesh()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, mesh, filename);
}
} }
...@@ -303,11 +303,13 @@ namespace AMDiS { ...@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements, * refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function * it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached. * terminates. Hence, it must be called until a stable mesh refinement is reached.
* If the mesh has been changed by this function, it returns true. Otherwise, it
* returns false, i.e., the given interior boundaries fit together on both sides.
* *
* \param[in] allBound Defines a map from rank to interior boundaries which * \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked. * should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/ */
bool checkAndAdaptBoundary(RankToBoundMap &allBound); bool checkAndAdaptBoundary(RankToBoundMap &allBound);
...@@ -318,13 +320,6 @@ namespace AMDiS { ...@@ -318,13 +320,6 @@ namespace AMDiS {
*/ */
void repartitionMesh(); void repartitionMesh();
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of. This function can be used for
* debugging.
*/
void writePartitioningMesh(string filename);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems. /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void setRankDofs(); void setRankDofs();
...@@ -550,11 +545,15 @@ namespace AMDiS { ...@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations. /// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed; bool repartitioningAllowed;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int repartitionIthChange; int repartitionIthChange;
int nTimestepsAfterLastRepartitioning; /// Counts the number of mesh changes after the last mesh repartitioning was done.
int nMeshChangesAfterLastRepartitioning;
int repartCounter; /// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int repartitioningCounter;
/// Directory name where all debug output files should be written to. /// Directory name where all debug output files should be written to.
string debugOutputDir; string debugOutputDir;
......
...@@ -247,8 +247,7 @@ namespace AMDiS { ...@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n"); TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode); return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return meshChanged;
} }
......
...@@ -54,6 +54,9 @@ namespace AMDiS { ...@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given * \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where * in reverse mode, i.e., left and right children where
* changed when the code was created. * changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/ */
bool startFitElementToMeshCode(MeshStructure &code, bool startFitElementToMeshCode(MeshStructure &code,
Element *el, Element *el,
...@@ -77,6 +80,9 @@ namespace AMDiS { ...@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given * \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where * in reverse mode, i.e., left and right children where
* changed when the code was created. * changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/ */
bool fitElementToMeshCode(MeshStructure &code, bool fitElementToMeshCode(MeshStructure &code,
TraverseStack &stack, TraverseStack &stack,
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "StdMpi.h" #include "StdMpi.h"
#include "Debug.h" #include "Debug.h"
#include "MpiHelper.h" #include "MpiHelper.h"
#include "io/VtkWriter.h"
namespace AMDiS { namespace AMDiS {
...@@ -806,4 +807,39 @@ namespace AMDiS { ...@@ -806,4 +807,39 @@ namespace AMDiS {
file.close(); file.close();
} }
void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
{
FUNCNAME("ParallelDebug::writeParitioning()");
map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
int index = elInfo->getElement()->getIndex();
vec[index] = pdb.partitionVec[index];
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, pdb.mesh, filename);
}
void ParallelDebug::writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace)
{
FUNCNAME("ParallelDebug::writePartitioningFile()");
stringstream oss;
oss << filename;
if (counter >= 0)
oss << "-" << counter;
oss << ".vtu";
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(MPI::COMM_WORLD.Get_rank());
VtkWriter::writeFile(&tmpa, oss.str());
}
} }
...@@ -142,6 +142,25 @@ namespace AMDiS { ...@@ -142,6 +142,25 @@ namespace AMDiS {
static void writeDebugFile(MeshDistributor &pdb, static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix); std::string prefix, std::string postfix);
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of.
*/
static void writePartitioning(MeshDistributor &pdb, string filename);
/** \brief
* The mesh is written to a value and all values are assigned by the rank number
* where the vertex is contained in.
*
* \param[in] filename Name of the output file without extension (.vtu).
* \param[in] counter Counter index. If not negative, this number is added
* to the filename.
* \param[in] feSpace
*/
static void writePartitioningFile(std::string filename,
int counter,
FiniteElemSpace *feSpace);
}; };
} // namespace AMDiS } // namespace AMDiS
......
...@@ -30,12 +30,12 @@ namespace AMDiS { ...@@ -30,12 +30,12 @@ namespace AMDiS {
vm /= 1024.0; vm /= 1024.0;
rss /= 1024.0; rss /= 1024.0;
MSG("My memory usage is VM = %f MB RSS = %f MB\n", vm, rss); MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm); mpi::globalAdd(vm);
mpi::globalAdd(rss); mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %f MB RSS = %f MB\n", vm, rss); MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment