Some code refactoring of the last changes. Added some more comments to parallel code.

AMDiS/src/parallel/MeshDistributor.cc

@@ -70,8 +70,8 @@ namespace AMDiS {
       writeSerializationFile(false),
       repartitioningAllowed(false),
       repartitionIthChange(20),
-      nTimestepsAfterLastRepartitioning(0),
-      repartCounter(0),
+      nMeshChangesAfterLastRepartitioning(0),
+      repartitioningCounter(0),
       debugOutputDir(""),
       lastMeshChangeIndex(0)
   {
@@ -168,9 +168,7 @@ namespace AMDiS {
 
       if (writePartMesh > 0) {
 	debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex.vtu");
-	writePartitioningMesh(debugOutputDir + "part.vtu");
-      } else {
-	MSG("Skip write part mesh!\n");
+	ParallelDebug::writePartitioning(*this, debugOutputDir + "part.vtu");
       }
     }
 #endif
@@ -607,7 +605,7 @@ namespace AMDiS {
 
       // === Check on all ranks if at least one rank's mesh has changed. ===
 
-      int sendValue = static_cast<int>(!meshChanged);
+      int sendValue = static_cast<int>(meshChanged);
       recvAllValues = 0;
       mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
 
@@ -635,13 +633,12 @@ namespace AMDiS {
 
     // === The mesh has changed, so check if it is required to repartition the mesh. ===
 
-    nTimestepsAfterLastRepartitioning++;
+    nMeshChangesAfterLastRepartitioning++;
 
-    if (repartitioningAllowed) {
-      if (nTimestepsAfterLastRepartitioning >= repartitionIthChange) {
-	repartitionMesh();
-	nTimestepsAfterLastRepartitioning = 0;
-      }
+    if (repartitioningAllowed && 
+	nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
+      repartitionMesh();
+      nMeshChangesAfterLastRepartitioning = 0;
     }
   }
 
@@ -671,7 +668,7 @@ namespace AMDiS {
  
     // === Compare received mesh structure codes. ===
     
-    bool meshFitTogether = true;
+    bool meshChanged = false;
 
     for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) {
      
@@ -679,7 +676,7 @@ namespace AMDiS {
       int i = 0;
       
       for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
-	   boundIt != it->second.end(); ++boundIt) {
+	   boundIt != it->second.end(); ++boundIt, i++) {
 
 	MeshStructure elCode;	
 	elCode.init(boundIt->rankObj);
@@ -688,23 +685,18 @@ namespace AMDiS {
 	  TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
 
 	  MeshManipulation meshManipulation(feSpace);
-	  bool b = 
+	  meshChanged |=
 	    meshManipulation.startFitElementToMeshCode(recvCodes[i], 
 						       boundIt->rankObj.el,
 						       boundIt->rankObj.subObj,
 						       boundIt->rankObj.ithObj, 
 						       boundIt->rankObj.elType,
 						       boundIt->rankObj.reverseMode);
-
-	  if (b)
-	    meshFitTogether = false;
  	}
-
-	i++;
       }
     }
 
-    return meshFitTogether;
+    return meshChanged;
   }
 
 
@@ -809,10 +801,15 @@ namespace AMDiS {
     TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
       ("Only meshes with one DOFAdmin are supported!\n");
 
-    int nDofs = mesh->getDofAdmin(0).getUsedDofs();
+
+    // === First we check if the rank with the maximum number of DOFs has at  ===
+    // === least 20% more DOFs than the rank with the minimum number of DOFs. ===
+    // === In this case, the mesh will be repartition.                        ===
+
     int repartitioning = 0;
 
     vector<int> nDofsInRank(mpiSize);
+    int nDofs = mesh->getDofAdmin(0).getUsedDofs();
     mpiComm.Gather(&nDofs, 1, MPI_INT, &(nDofsInRank[0]), 1, MPI_INT, 0);
 
     if (mpiRank == 0) {
@@ -828,7 +825,7 @@ namespace AMDiS {
       MSG("Overall DOFs: %d    Min DOFs: %d    Max DOFs: %d\n", 
 	  nOverallDofs, minDofs, maxDofs);
 
-      if (static_cast<double>(maxDofs) / static_cast<double>(minDofs)) 
+      if (static_cast<double>(maxDofs) / static_cast<double>(minDofs) > 1.2) 
 	repartitioning = 1;
 
       mpiComm.Bcast(&repartitioning, 1, MPI_INT, 0);
@@ -841,25 +838,19 @@ namespace AMDiS {
       return;
 
     double timePoint = MPI::Wtime();
+    repartitioningCounter++;
 
 #if (DEBUG != 0)
     ParallelDebug::testDoubleDofs(mesh);
 
-    if (repartCounter == 0) {        
-      stringstream oss;
-      oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";
-      MSG("Write partitioning to %s\n", oss.str().c_str());
-      DOFVector<double> tmpa(feSpace, "tmp");
-      tmpa.set(mpiRank);
-      VtkWriter::writeFile(&tmpa, oss.str());
-      
-      repartCounter++;
-    }
+    if (repartitioningCounter == 0)
+      ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning", 
+					   repartitioningCounter, feSpace);
 #endif
 
+    // === Create new element weights. ===
 
     elemWeights.clear();
-
     TraverseStack stack;
     ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
     while (elInfo) {
@@ -867,15 +858,16 @@ namespace AMDiS {
       elInfo = stack.traverseNext(elInfo);
     }
 
+
+    // === Run ParMETiS to calculate a new mesh partitioning.  ===
+
     partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
     bool partitioningSucceed =
       partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
-
     if (!partitioningSucceed) {
       MSG("ParMETIS created empty partition!\n");
       return;
     }
-
     oldPartitionVec = partitionVec;
 
 
@@ -1045,32 +1037,17 @@ namespace AMDiS {
 
     mesh->dofCompress();
 
-#if (DEBUG != 0)
-    stringstream oss;
-    oss << debugOutputDir << "partitioning-" << repartCounter << ".vtu";    
-    DOFVector<double> tmpa(feSpace, "tmp");
-    tmpa.set(mpiRank);
-    VtkWriter::writeFile(&tmpa, oss.str());
-    repartCounter++;
-
-    ParallelDebug::testAllElements(*this);
-    ParallelDebug::testDoubleDofs(mesh);
-#endif
-
     partitioner->fillCoarsePartitionVec(&partitionVec);
-
     updateInteriorBoundaryInfo();
-
-#if (DEBUG != 0)
-    ParallelDebug::printBoundaryInfo(*this);
-#endif
-
     updateLocalGlobalNumbering();
 
 #if (DEBUG != 0)
     MSG("AMDiS runs in debug mode, so make some test ...\n");
 
+    ParallelDebug::writePartitioningFile(debugOutputDir + "partitioning", 
+					 repartitioningCounter, feSpace);
     ParallelDebug::testAllElements(*this);
+    ParallelDebug::testDoubleDofs(mesh);
     ParallelDebug::testInteriorBoundary(*this);
     ParallelDebug::printBoundaryInfo(*this);
 
@@ -1096,6 +1073,10 @@ namespace AMDiS {
 
     elObjects.createRankData(partitionVec);
     createBoundaryData();
+
+#if (DEBUG != 0)
+    ParallelDebug::printBoundaryInfo(*this);
+#endif
   }
 
 
@@ -1872,8 +1853,8 @@ namespace AMDiS {
     for (int i = 0; i < nSize; i++)
       allMacroElements[i]->serialize(out);
 
-    SerUtil::serialize(out, nTimestepsAfterLastRepartitioning);
-    SerUtil::serialize(out, repartCounter);
+    SerUtil::serialize(out, nMeshChangesAfterLastRepartitioning);
+    SerUtil::serialize(out, repartitioningCounter);
   }
 
 
@@ -1939,8 +1920,8 @@ namespace AMDiS {
       allMacroElements[i]->getElement()->setMesh(mesh);
     }
 
-    SerUtil::deserialize(in, nTimestepsAfterLastRepartitioning);
-    SerUtil::deserialize(in, repartCounter);
+    SerUtil::deserialize(in, nMeshChangesAfterLastRepartitioning);
+    SerUtil::deserialize(in, repartitioningCounter);
 
     deserialized = true;
   }
@@ -2014,23 +1995,4 @@ namespace AMDiS {
     }
   }
 
- 
-  void MeshDistributor::writePartitioningMesh(string filename)
-  {
-    FUNCNAME("MeshDistributor::writePartitioningMesh()");
-
-    map<int, double> vec;    
-    TraverseStack stack;
-    ElInfo *elInfo = stack.traverseFirst(mesh, -1, 
-					 Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
-    
-    while (elInfo) {		  
-      int index = elInfo->getElement()->getIndex();
-      vec[index] = partitionVec[index];
-      elInfo = stack.traverseNext(elInfo);
-    }
-
-    ElementFileWriter::writeFile(vec, mesh, filename);
-  }
-
 }

AMDiS/src/parallel/MeshDistributor.h

@@ -303,11 +303,13 @@ namespace AMDiS {
      * refinement of a certain element may forces the refinement of other elements,
      * it is not guaranteed that all rank's meshes fit together after this function
      * terminates. Hence, it must be called until a stable mesh refinement is reached.
-     * If the mesh has  been changed by this function, it returns true. Otherwise, it
-     * returns false, i.e., the given interior boundaries fit together on both sides.
      *
      * \param[in] allBound   Defines a map from rank to interior boundaries which 
      *                       should be checked.
+     *
+     * \return    If the mesh has  been changed by this function, it returns true. 
+     *            Otherwise, it returns false, i.e., the given interior boundaries 
+     *            fit together on both sides.
      */
     bool checkAndAdaptBoundary(RankToBoundMap &allBound);
   
@@ -318,13 +320,6 @@ namespace AMDiS {
      */
     void repartitionMesh();
 
-    /** \brief
-     * This functions create a Paraview file with the macro mesh where the elements
-     * are colored by the partition they are part of. This function can be used for
-     * debugging.
-     */
-    void writePartitioningMesh(string filename);
-
     /// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
     void setRankDofs();
 
@@ -550,11 +545,15 @@ namespace AMDiS {
     /// If true, it is possible to repartition the mesh during computations.
     bool repartitioningAllowed;
 
+    /// Stores the number of mesh changes that must lie in between to repartitionings.
     int repartitionIthChange;
 
-    int nTimestepsAfterLastRepartitioning;
+    /// Counts the number of mesh changes after the last mesh repartitioning was done.
+    int nMeshChangesAfterLastRepartitioning;
 
-    int repartCounter;
+    /// Countes the number of mesh repartitions that were done. Till now, this 
+    /// variable is used only for debug outputs.
+    int repartitioningCounter;
 
     /// Directory name where all debug output files should be written to.
     string debugOutputDir;

AMDiS/src/parallel/MeshManipulation.cc

@@ -247,8 +247,7 @@ namespace AMDiS {
 
       TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
 
-      meshChanged = fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
-      return meshChanged;
+      return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
     }
 
 

AMDiS/src/parallel/MeshManipulation.h

@@ -54,6 +54,9 @@ namespace AMDiS {
      * \param[in] reverseMode  Defines, whether the mesh structure code is given
      *                         in reverse mode, i.e., left and right children where
      *                         changed when the code was created.
+     *
+     * \return   Returns true, if the mesh was changed, i.e. refined, to fit the
+     *           element to the structure code.
      */
     bool startFitElementToMeshCode(MeshStructure &code, 
 				   Element *el, 
@@ -77,6 +80,9 @@ namespace AMDiS {
      * \param[in] reverseMode  Defines, whether the mesh structure code is given
      *                         in reverse mode, i.e., left and right children where
      *                         changed when the code was created.
+     *
+     * \return   Returns true, if the mesh was changed, i.e. refined, to fit the
+     *           element to the structure code.
      */
     bool fitElementToMeshCode(MeshStructure &code, 
 			      TraverseStack &stack,

AMDiS/src/parallel/ParallelDebug.cc

@@ -18,6 +18,7 @@
 #include "StdMpi.h"
 #include "Debug.h"
 #include "MpiHelper.h"
+#include "io/VtkWriter.h"
 
 namespace AMDiS {
 
@@ -806,4 +807,39 @@ namespace AMDiS {
     file.close();
   }
 
+
+  void ParallelDebug::writePartitioning(MeshDistributor &pdb, string filename)
+  {
+    FUNCNAME("ParallelDebug::writeParitioning()");
+
+    map<int, double> vec;    
+    TraverseStack stack;
+    ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, 
+					 Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
+    
+    while (elInfo) {		  
+      int index = elInfo->getElement()->getIndex();
+      vec[index] = pdb.partitionVec[index];
+      elInfo = stack.traverseNext(elInfo);
+    }
+
+    ElementFileWriter::writeFile(vec, pdb.mesh, filename);
+  }
+
+  void ParallelDebug::writePartitioningFile(std::string filename, 
+					    int counter,
+					    FiniteElemSpace *feSpace)
+  {
+    FUNCNAME("ParallelDebug::writePartitioningFile()");
+
+    stringstream oss;
+    oss << filename;
+    if (counter >= 0)
+      oss << "-" << counter;
+    oss << ".vtu";
+
+    DOFVector<double> tmpa(feSpace, "tmp");
+    tmpa.set(MPI::COMM_WORLD.Get_rank());
+    VtkWriter::writeFile(&tmpa, oss.str());
+  }
 }

AMDiS/src/parallel/ParallelDebug.h

@@ -142,6 +142,25 @@ namespace AMDiS {
 
     static void writeDebugFile(MeshDistributor &pdb,
 			       std::string prefix, std::string postfix);
+
+    /** \brief
+     * This functions create a Paraview file with the macro mesh where the elements
+     * are colored by the partition they are part of.
+     */
+    static void writePartitioning(MeshDistributor &pdb, string filename);
+
+    /** \brief
+     * The mesh is written to a value and all values are assigned by the rank number
+     * where the vertex is contained in.
+     *
+     * \param[in]  filename    Name of the output file without extension (.vtu).
+     * \param[in]  counter     Counter index. If not negative, this number is added
+     *                         to the filename.
+     * \param[in]  feSpace
+     */
+    static void writePartitioningFile(std::string filename,
+				      int counter,
+				      FiniteElemSpace *feSpace);
   };
   
 } // namespace AMDiS

AMDiS/src/parallel/ParallelProblemStatBase.cc

@@ -30,12 +30,12 @@ namespace AMDiS {
     vm /= 1024.0;
     rss /= 1024.0;
     
-    MSG("My memory usage is VM = %f MB    RSS = %f MB\n", vm, rss);
+    MSG("My memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
     
     mpi::globalAdd(vm);
     mpi::globalAdd(rss);
 
-    MSG("Overall memory usage is VM = %f MB    RSS = %f MB\n", vm, rss);
+    MSG("Overall memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
   }