Some performance issues.

AMDiS/src/Global.cc

@@ -200,6 +200,13 @@ namespace AMDiS {
 				const char *file, 
 				int line)
   {
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if (DEBUG == 0)
+    if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
+      return;
+#endif
+#endif
+
     static int old_line = -1;
 
     if (!out) 
@@ -229,6 +236,12 @@ namespace AMDiS {
 
   void Msg::print_warn(const char *format, ...)
   {
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if (DEBUG == 0)
+    if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
+      return;
+#endif
+#endif
     va_list arg;
     char buff[255];
 

AMDiS/src/Marker.h

@@ -64,11 +64,9 @@ namespace AMDiS {
     /// destructor
     virtual ~Marker() {}
 
-    /** \brief
-     * Marks element with newMark. If \ref maximumMarking is set, the element
-     * is marked only if the new mark is bigger than the old one. The return
-     * value specifies whether the element has been marked, or not.
-     */
+    /// Marks element with newMark. If \ref maximumMarking is set, the element
+    /// is marked only if the new mark is bigger than the old one. The return
+    /// value specifies whether the element has been marked, or not.
     inline void setMark(Element *el, char newMark) 
     {
       char oldMark = el->getMark();
@@ -120,7 +118,7 @@ namespace AMDiS {
       return elMarkCoarsen; 
     }
 
-    // Returns \ref name of the Marker
+    /// Returns \ref name of the Marker
     inline std::string getName() const 
     { 
       return name; 
@@ -133,10 +131,8 @@ namespace AMDiS {
     /// Name of the scalar marker.
     std::string name;
 
-    /** \brief
-     * Equal to -1 for scalar problems. Component number if marker is
-     * part of a vector valued marker.
-     */
+    /// Equal to -1 for scalar problems. Component number if marker is
+    /// part of a vector valued marker.
     int row;
 
     /// estimation sum
@@ -145,22 +141,16 @@ namespace AMDiS {
     /// estmation maximum
     double estMax;
 
-    /** \brief
-     * If true, elements are marked only if the new value is bigger than
-     * the current marking.
-     */
+    /// If true, elements are marked only if the new value is bigger than
+    /// the current marking.
     bool maximumMarking;
 
-    /** \brief
-     * Lower limit for error estimation, from which an element is marked for
-     * refinement
-     */
+    /// Lower limit for error estimation, from which an element is marked for
+    /// refinement
     double markRLimit;
 
-    /** \brief
-     * Upper limit for error estimation, from which an element is marked for 
-     * coarsening
-     */
+    /// Upper limit for error estimation, from which an element is marked for 
+    /// coarsening
     double markCLimit;
 
     /// power in estimator norm

AMDiS/src/parallel/MeshDistributor.cc

@@ -87,7 +87,8 @@ namespace AMDiS {
       createBoundaryDofFlag(0),
       boundaryDofInfo(1),
       meshAdaptivity(true),
-      hasPeriodicBoundary(false)
+      hasPeriodicBoundary(false),
+      printTimings(false)
   {
     FUNCNAME("MeshDistributor::ParalleDomainBase()");
 
@@ -99,7 +100,7 @@ namespace AMDiS {
     Parameters::get(name + "->debug output dir", debugOutputDir);
     Parameters::get(name + "->repartition ith change", repartitionIthChange);
     Parameters::get(name + "->mesh adaptivity", meshAdaptivity);
-
+    
     string partStr = "parmetis";
     Parameters::get(name + "->partitioner", partStr);
 
@@ -124,6 +125,8 @@ namespace AMDiS {
     Parameters::get(name + "->box partitioning", tmp);
     partitioner->setBoxPartitioning(static_cast<bool>(tmp));
 
+    Parameters::get(name + "->print timings", printTimings);
+
     TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
   }
 
@@ -1584,6 +1587,10 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::createBoundaryDofs()");
 
+    if (printTimings)
+      MPI::COMM_WORLD.Barrier();
+    double first = MPI::Wtime();
+
     // === Create DOF communicator. ===
 
     dofComm.init(0, levelData, feSpaces);
@@ -1596,52 +1603,57 @@ namespace AMDiS {
 
     // === If requested, create more information on communication DOFs. ===
     
-    if (!createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED))
-      return;
-
-    int nLevels = levelData.getLevelNumber();
-    boundaryDofInfo.resize(nLevels);
-
-    for (unsigned int i = 0; i < feSpaces.size(); i++) {
-      const FiniteElemSpace *feSpace = feSpaces[i];
+    if (createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) {
 
-      for (int level = 0; level < nLevels; level++) {
-
-	// === Clear data. ===
-	for (int geo = FACE; geo >= VERTEX; geo--)
-	  boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
+      int nLevels = levelData.getLevelNumber();
+      boundaryDofInfo.resize(nLevels);
+      
+      for (unsigned int i = 0; i < feSpaces.size(); i++) {
+	const FiniteElemSpace *feSpace = feSpaces[i];
 	
-	// === Create send DOFs. ===
-	for (int geo = FACE; geo >= VERTEX; geo--) {
-	  for (InteriorBoundary::iterator it(intBoundary.getOwn(), level); 
-	       !it.end(); ++it) {
-	    if (it->rankObj.subObj == geo) {
-	      DofContainer dofs;
-	      it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
-	      
-	      if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
-		boundaryDofInfo[level][feSpace].
-		  geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+	for (int level = 0; level < nLevels; level++) {
+	  
+	  // === Clear data. ===
+	  for (int geo = FACE; geo >= VERTEX; geo--)
+	    boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
+	  
+	  // === Create send DOFs. ===
+	  for (int geo = FACE; geo >= VERTEX; geo--) {
+	    for (InteriorBoundary::iterator it(intBoundary.getOwn(), level); 
+		 !it.end(); ++it) {
+	      if (it->rankObj.subObj == geo) {
+		DofContainer dofs;
+		it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
+		
+		if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
+		  boundaryDofInfo[level][feSpace].
+		    geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+	      }
 	    }
 	  }
-	}
-	
-	// === Create recv DOFs. ===
-	for (int geo = FACE; geo >= VERTEX; geo--) {
-	  for (InteriorBoundary::iterator it(intBoundary.getOther(), level); 
-	       !it.end(); ++it) {
-	    if (it->rankObj.subObj == geo) {
-	      DofContainer dofs;
-	      it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
-	      
-	      if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
-		boundaryDofInfo[level][feSpace].
-		  geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+	  
+	  // === Create recv DOFs. ===
+	  for (int geo = FACE; geo >= VERTEX; geo--) {
+	    for (InteriorBoundary::iterator it(intBoundary.getOther(), level); 
+		 !it.end(); ++it) {
+	      if (it->rankObj.subObj == geo) {
+		DofContainer dofs;
+		it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
+		
+		if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
+		  boundaryDofInfo[level][feSpace].
+		    geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
+	      }
 	    }
 	  }
 	}
       }
     }
+
+    if (printTimings) {
+      MPI::COMM_WORLD.Barrier();
+      MSG("MESHDIST 01 (createBoundaryDofs) needed %.5f seconds\n", MPI::Wtime() - first);
+    }
   }
 
 
@@ -1679,22 +1691,8 @@ namespace AMDiS {
 
 
     // === Update all registered DOF mapping objects. ===
-    
-    TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
 
-    for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
-      vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
-      
-      dofMaps[i]->clear();
-
-      for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
-	updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
-
-      dofMaps[i]->update();
-#if (DEBUG != 0)
-      //      dofMaps[i]->printInfo();
-#endif
-    }
+    updateParallelDofMappings();
 
 
     // === Create periodic DOF maps, if there are periodic boundaries.  ===
@@ -1769,6 +1767,37 @@ namespace AMDiS {
   }
 
 
+  void MeshDistributor::updateParallelDofMappings()
+  {
+    FUNCNAME("MeshDistributor::updateParallelDofMappings()");
+
+    if (printTimings)
+      MPI::COMM_WORLD.Barrier();
+    double first = MPI::Wtime();
+
+    TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
+
+    for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
+      vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
+      
+      dofMaps[i]->clear();
+
+      for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
+	updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
+
+      dofMaps[i]->update();
+#if (DEBUG != 0)
+      //      dofMaps[i]->printInfo();
+#endif
+    }
+
+    if (printTimings) {
+      MPI::COMM_WORLD.Barrier();
+      MSG("MESHDIST 02 (update DOF mappings) needed %.5f seconds\n", MPI::Wtime() - first);
+    }
+  }
+
+
   void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dofMap,
 						   const FiniteElemSpace *feSpace)
   {
@@ -1776,6 +1805,7 @@ namespace AMDiS {
 
     DofComm &dcom = dofMap.getDofComm();
 
+
     // === Get all DOFs in ranks partition. ===
 
     std::set<const DegreeOfFreedom*> rankDofSet;
@@ -1792,14 +1822,14 @@ namespace AMDiS {
 	 !it.end(); it.nextRank())
       for (; !it.endDofIter(); it.nextDof())
 	nonRankDofs.insert(it.getDof());
-    
-    for (unsigned int i = 0; i < rankDofs.size(); i++)
-      if (nonRankDofs.count(rankDofs[i]) == 0)	  
-	dofMap[feSpace].insertRankDof(*(rankDofs[i]));
-    
-    for (DofContainerSet::iterator it = nonRankDofs.begin();
-	 it != nonRankDofs.end(); ++it)
-      dofMap[feSpace].insertNonRankDof(**it);
+
+    for (DofContainer::iterator it = rankDofs.begin(); 
+	 it != rankDofs.end(); ++it) {
+      if (nonRankDofs.count(*it))
+	dofMap[feSpace].insertNonRankDof(**it);
+      else
+	dofMap[feSpace].insertRankDof(**it);
+    }
   }
 
 

AMDiS/src/parallel/MeshDistributor.h

@@ -322,6 +322,9 @@ namespace AMDiS {
       return levelData;
     }
 
+    /// Updates all registered parallel DOF mappings, see \ref dofMaps.
+    void updateParallelDofMappings();
+
     void updateLocalGlobalNumbering();
 
     /// Updates the local and global DOF numbering after the mesh has been 
@@ -583,6 +586,9 @@ namespace AMDiS {
     /// solver objects and must be updated automatically after mesh change.
     vector<ParallelDofMapping*> dofMaps;
 
+    /// If true, detailed timings for benchmarking are printed.
+    bool printTimings;
+
   public:
     /// The boundary DOFs are sorted by subobject entities, i.e., first all
     /// face DOFs, edge DOFs and to the last vertex DOFs will be set to

AMDiS/src/parallel/ParallelDofMapping.h

@@ -161,7 +161,8 @@ namespace AMDiS {
     
     /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
     /// the rank.
-    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
+    inline void insertRankDof(DegreeOfFreedom dof0, 
+			      DegreeOfFreedom dof1 = -1)
     {
       FUNCNAME("ComponentDofMap::insertRankDof()");
       
@@ -176,7 +177,8 @@ namespace AMDiS {
     
     /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
     /// but is owned by a different rank, thus it is part of an interior boundary.
-    void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
+    inline void insertNonRankDof(DegreeOfFreedom dof0, 
+				 DegreeOfFreedom dof1 = -1)
     {
       FUNCNAME("ComponentDofMap::insertNonRankDof()");
       
@@ -281,7 +283,7 @@ namespace AMDiS {
     DofMap dofMap;
 
     /// Set of all DOFs that are in mapping but are not owned by the rank.
-    boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
+    boost::container::flat_set<DegreeOfFreedom> nonRankDofs;    
 
     /// If true, a global index mapping will be computed for all DOFs.
     bool globalMapping;