More changes to get multilevel FETI-DP runing.

AMDiS/src/parallel/ElementObjectDatabase.cc

@@ -15,8 +15,13 @@
 
 namespace AMDiS {
 
-  void ElementObjectDatabase::create()
+  void ElementObjectDatabase::create(map<int, int>& rankMap, MeshLevelData& ld)
   {
+    FUNCNAME("ElementObjectDatabase::create()");
+
+    macroElementRankMap = &rankMap;
+    levelData = &ld;
+
     // === Fills macro element data structures. ===
     
     TraverseStack stack;
@@ -417,104 +422,41 @@ namespace AMDiS {
   }
 
 
-  void ElementObjectDatabase::createRankData(map<int, int>& macroElementRankMap,
-					     MeshLevelData& levelData)
+  void ElementObjectDatabase::updateRankData()
   {
-    FUNCNAME("ElementObjectDatabase::createRankData()");
+    FUNCNAME("ElementObjectDatabase::updateRankData()");
     
-    int nLevel = levelData.getLevelNumber();
-    TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
-
-    vertexOwner.clear();
     vertexInRank.clear();
     for (map<DegreeOfFreedom, vector<ElementObjectData> >::iterator it = vertexElements.begin();
 	 it != vertexElements.end(); ++it) {
-
-      vector<std::set<int> > ranksInLevel;
-      ranksInLevel.resize(nLevel);
-
       for (vector<ElementObjectData>::iterator it2 = it->second.begin(); 
 	   it2 != it->second.end(); ++it2) {
-	int elementInRank = macroElementRankMap[it2->elIndex];
-	
+	int elementInRank =(* macroElementRankMap)[it2->elIndex];	
 	if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
 	  vertexInRank[it->first][elementInRank] = *it2;
-
-	vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
-
-	ranksInLevel[0].insert(elementInRank);
-	for (int level = 1; level < nLevel; level++)
-	  ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
       }
-
-      int maxLevel = 0;
-      for (int level = 1; level < nLevel; level++)
-	if (ranksInLevel[level].size() > 1)
-	  maxLevel = level;
-
-      vertexMaxLevel[it->first] = maxLevel;
     }
     
-
-    edgeOwner.clear();
     edgeInRank.clear();
     for (map<DofEdge, vector<ElementObjectData> >::iterator it = edgeElements.begin();
 	 it != edgeElements.end(); ++it) {
-
-      vector<std::set<int> > ranksInLevel;
-      ranksInLevel.resize(nLevel);
-
       for (vector<ElementObjectData>::iterator it2 = it->second.begin(); 
 	   it2 != it->second.end(); ++it2) {
-	int elementInRank = macroElementRankMap[it2->elIndex];
-	
+	int elementInRank = (*macroElementRankMap)[it2->elIndex];
 	if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
 	  edgeInRank[it->first][elementInRank] = *it2;
-
-	edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
-
-	ranksInLevel[0].insert(elementInRank);
-	for (int level = 1; level < nLevel; level++)
-	  ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
       }
-
-      int maxLevel = 0;
-      for (int level = 1; level < nLevel; level++)
-	if (ranksInLevel[level].size() > 1)
-	  maxLevel = level;
-
-      edgeMaxLevel[it->first] = maxLevel;
     }
     
-
-    faceOwner.clear();
     faceInRank.clear();
     for (map<DofFace, vector<ElementObjectData> >::iterator it = faceElements.begin();
 	 it != faceElements.end(); ++it) {
-
-      vector<std::set<int> > ranksInLevel;
-      ranksInLevel.resize(nLevel);
-
       for (vector<ElementObjectData>::iterator it2 = it->second.begin(); 
 	   it2 != it->second.end(); ++it2) {
-	int elementInRank = macroElementRankMap[it2->elIndex];
-	
+	int elementInRank = (*macroElementRankMap)[it2->elIndex];
 	if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
 	  faceInRank[it->first][elementInRank] = *it2;
-
-	faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
-
-	ranksInLevel[0].insert(elementInRank);
-	for (int level = 1; level < nLevel; level++)
-	  ranksInLevel[level].insert(levelData.getLevelId(level, elementInRank));
       }
-
-      int maxLevel = 0;
-      for (int level = 1; level < nLevel; level++)
-	if (ranksInLevel[level].size() > 1)
-	  maxLevel = level;
-
-      faceMaxLevel[it->first] = maxLevel;
     }
   }
 
@@ -630,6 +572,98 @@ namespace AMDiS {
   }
 
 
+  int ElementObjectDatabase::getIterateOwner()
+  {
+    FUNCNAME("ElementObjectDatabase::getIterateOwner()");
+
+    TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
+
+    int owner = -1;
+
+    switch (iterGeoPos) {
+    case VERTEX:
+      {
+	vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
+	for (vector<ElementObjectData>::iterator it = vertexData.begin();
+	     it != vertexData.end(); ++it)
+	  owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
+      }
+      break;
+    case EDGE:
+      {
+	vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
+	for (vector<ElementObjectData>::iterator it = edgeData.begin();
+	     it != edgeData.end(); ++it)
+	  owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
+      }
+      break;
+    case FACE:
+      {
+      	vector<ElementObjectData>& faceData = faceElements[faceIter->first];
+	for (vector<ElementObjectData>::iterator it = faceData.begin();
+	     it != faceData.end(); ++it)
+	  owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
+      }
+      break;
+    default:
+	ERROR_EXIT("Should not happen!\n");
+    }    
+
+    return owner;
+  }
+
+
+  int ElementObjectDatabase::getIterateMaxLevel()
+  {
+    FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()");
+
+    int nLevel = levelData->getLevelNumber();
+    TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
+
+    vector<std::set<int> > ranksInLevel;
+    ranksInLevel.resize(nLevel);
+
+    switch (iterGeoPos) {
+    case VERTEX:
+      {
+	vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
+	for (vector<ElementObjectData>::iterator it = vertexData.begin();
+	     it != vertexData.end(); ++it)
+	  ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
+      }
+      break;
+    case EDGE:
+      {
+	vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
+	for (vector<ElementObjectData>::iterator it = edgeData.begin();
+	     it != edgeData.end(); ++it)
+	  ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
+      }
+      break;
+    case FACE:
+      vector<ElementObjectData>& faceData = faceElements[faceIter->first];
+      for (vector<ElementObjectData>::iterator it = faceData.begin();
+	   it != faceData.end(); ++it)
+	ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
+      break;
+    default:
+      ERROR_EXIT("Should not happen!\n");
+    }
+
+    for (std::set<int>::iterator it = ranksInLevel[0].begin(); 
+	 it != ranksInLevel[0].end(); ++it) 
+      for (int level = 1; level < nLevel; level++)
+	ranksInLevel[level].insert(levelData->getLevelId(level, *it));
+
+    int maxLevel = 0;
+    for (int level = 1; level < nLevel; level++)
+      if (ranksInLevel[level].size() > 1)
+	maxLevel = level;
+    
+    return maxLevel;
+  }
+
+
   void ElementObjectDatabase::serialize(ostream &out)
   {
     FUNCNAME("ElementObjectDatabase::serialize()");
@@ -686,11 +720,6 @@ namespace AMDiS {
 
 
     ERROR_EXIT("REWRITE SERIALIZATION!\n");
-    /*
-    SerUtil::serialize(out, vertexOwner);
-    SerUtil::serialize(out, edgeOwner);
-    SerUtil::serialize(out, faceOwner);
-    */
 
     nSize = vertexInRank.size();
     SerUtil::serialize(out, nSize);
@@ -825,12 +854,7 @@ namespace AMDiS {
       faceLocalMap[data] = face;
     }
 
-
     ERROR_EXIT("REWRITE DESERIALIZATION!\n");
-    SerUtil::deserialize(in, vertexOwner);
-    SerUtil::deserialize(in, edgeOwner);
-    SerUtil::deserialize(in, faceOwner);
-
 
     SerUtil::deserialize(in, nSize);
     vertexInRank.clear();

AMDiS/src/parallel/ElementObjectDatabase.h

@@ -105,7 +105,9 @@ namespace AMDiS {
     ElementObjectDatabase()
       : feSpace(NULL),
 	mesh(NULL),
-	iterGeoPos(CENTER)
+	iterGeoPos(CENTER),
+	macroElementRankMap(NULL),
+	levelData(NULL)
     {}
 
     void setFeSpace(const FiniteElemSpace *fe)
@@ -119,7 +121,12 @@ namespace AMDiS {
       return mesh;
     }
 
-    void create();
+    /*
+     * \param[in]  macroElementRankMap   Maps to each macro element of the mesh
+     *                                   the rank that owns this macro element.
+     */
+    void create(map<int, int>& macroElementRankMap,
+		MeshLevelData& levelData);
 
     void createMacroElementInfo(vector<MacroElement*> &mel);
 
@@ -127,13 +134,8 @@ namespace AMDiS {
      * Create for a filled object database the membership information for all
      * element objects. An object is owned by a rank, if the rank has the
      * heighest rank number of all ranks where the object is part of.
-     *
-     * \param[in]  macroElementRankMap   Maps to each macro element of the mesh
-     *                                   the rank that owns this macro element.
      */
-    void createRankData(map<int, int>& macroElementRankMap,
-			MeshLevelData& levelData);
-
+    void updateRankData();
 
     /** \brief
      * Iterates over all elements for one geometrical index, i.e., over all
@@ -227,67 +229,11 @@ namespace AMDiS {
       }
     }
 
-
     /// Returns the rank owner of the current iterator position.
-    int getIterateOwner()
-    {
-      switch (iterGeoPos) {
-      case VERTEX:
-	return vertexOwner[vertexIter->first];
-	break;
-      case EDGE:
-	return edgeOwner[edgeIter->first];
-	break;
-      case FACE:
-	return faceOwner[faceIter->first];
-	break;
-      default:
-	ERROR_EXIT("Should not happen!\n");
-
-	// Will never be reached, just to avoid compiler warnings.
-	return -1;
-      }
-    }
+    int getIterateOwner();
 
     /// Returns the rank owner of the current iterator position.
-    int getIterateMaxLevel()
-    {
-      switch (iterGeoPos) {
-      case VERTEX:
-	return vertexMaxLevel[vertexIter->first];
-	break;
-      case EDGE:
-	return edgeMaxLevel[edgeIter->first];
-	break;
-      case FACE:
-	return faceMaxLevel[faceIter->first];
-	break;
-      default:
-	ERROR_EXIT("Should not happen!\n");
-
-	// Will never be reached, just to avoid compiler warnings.
-	return -1;
-      }
-    }
-
-    /// Returns the rank owner of a vertex DOF.
-    int getOwner(DegreeOfFreedom vertex)
-    {
-      return vertexOwner[vertex];
-    }
-
-    /// Returns the rank owner of an edge.
-    int getOwner(DofEdge edge)
-    {
-      return edgeOwner[edge];
-    }
-
-    /// Returns the rank owner of an face.
-    int getOwner(DofFace face, int level)
-    {
-      return faceOwner[face];
-    }
-
+    int getIterateMaxLevel();
 
     /// Checks if a given vertex DOF is in a given rank.
     int isInRank(DegreeOfFreedom vertex, int rank)
@@ -567,23 +513,7 @@ namespace AMDiS {
     /// Maps to an element object the corresponding face.
     map<ElementObjectData, DofFace> faceLocalMap;
 
-    /// Defines for all vertex DOFs the rank that ownes this vertex DOF.
-    map<DegreeOfFreedom, int> vertexOwner;
-
-    /// Defines for all edges the rank that ownes this edge.
-    map<DofEdge, int> edgeOwner;
-
-    /// Defines for all faces the rank that ownes this face.
-    map<DofFace, int> faceOwner;
-
-    
-    map<DegreeOfFreedom, int> vertexMaxLevel;
-
-    map<DofEdge, int> edgeMaxLevel;
-
-    map<DofFace, int> faceMaxLevel;
-
-
+   
     /// Defines to each vertex DOF a map that maps to each rank number the element
     /// objects that have this vertex DOF in common.
     map<DegreeOfFreedom, map<int, ElementObjectData> > vertexInRank;
@@ -636,11 +566,15 @@ namespace AMDiS {
 
     map<pair<ElementObjectData, ElementObjectData>, bool> faceReverseMode;
 
+    map<int, int> *macroElementRankMap;
+
     /// Maps to each macro element index a pointer to the corresponding element.
     map<int, Element*> macroElIndexMap;
     
     /// Maps to each macro element index the type of this element.
     map<int, int> macroElIndexTypeMap;
+
+    MeshLevelData* levelData;
   };
 
 }

AMDiS/src/parallel/InteriorBoundary.cc

@@ -19,28 +19,6 @@
 
 namespace AMDiS {
 
-  AtomicBoundary& InteriorBoundary::getNewAtomicOwn(int rank)
-  {
-    int size = own[rank].size();
-    own[rank].resize(size + 1);
-    return own[rank][size];
-  }
-
-  AtomicBoundary& InteriorBoundary::getNewAtomicOther(int rank)
-  {
-    int size = other[rank].size();
-    other[rank].resize(size + 1);
-    return other[rank][size];
-  }
-
-  AtomicBoundary& InteriorBoundary::getNewAtomicPer(int rank)
-  {
-    int size = periodic[rank].size();
-    periodic[rank].resize(size + 1);
-    return periodic[rank][size];
-  }
-
-
   void InteriorBoundary::create(MPI::Intracomm &mpiComm,
 				ElementObjectDatabase &elObjDb)
   { 
@@ -70,7 +48,6 @@ namespace AMDiS {
 	
 	AtomicBoundary bound;
 	bound.maxLevel = elObjDb.getIterateMaxLevel();
-
 	bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex);
 	bound.rankObj.elIndex = rankBoundEl.elIndex;
 	bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex);
@@ -101,7 +78,7 @@ namespace AMDiS {
 	    
 	    bound.type = INTERIOR;
 	    
-	    AtomicBoundary& b = getNewAtomicOwn(it2->first);
+	    AtomicBoundary& b = getNewOwn(it2->first);
 	    b = bound;
 	    if (geoIndex == EDGE)
 	      b.neighObj.reverseMode = 
@@ -125,7 +102,7 @@ namespace AMDiS {
 	  
 	  bound.type = INTERIOR;
 	  
-	  AtomicBoundary& b = getNewAtomicOther(owner);
+	  AtomicBoundary& b = getNewOther(owner);
 	  b = bound;	    
 	  if (geoIndex == EDGE)
 	    b.rankObj.reverseMode =
@@ -169,7 +146,7 @@ namespace AMDiS {
 
 	bound.type = it->second;
 
-	AtomicBoundary& b = getNewAtomicPer(otherElementRank);
+	AtomicBoundary& b = getNewPeriodic(otherElementRank);
 	b = bound;	    
       }
     }
@@ -203,7 +180,7 @@ namespace AMDiS {
 	
 	bound.type = it->second;
 	
-	AtomicBoundary& b = getNewAtomicPer(otherElementRank);
+	AtomicBoundary& b = getNewPeriodic(otherElementRank);
 	b = bound;
      
 	if (mpiRank > otherElementRank)
@@ -249,7 +226,7 @@ namespace AMDiS {
 	
 	bound.type = it->second;
 	
-	AtomicBoundary& b = getNewAtomicPer(otherElementRank);
+	AtomicBoundary& b = getNewPeriodic(otherElementRank);
 	b = bound;
      
 	if (mpiRank > otherElementRank)
@@ -465,6 +442,30 @@ namespace AMDiS {
   }
 
 
+  AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
+  {
+    int size = own[rank].size();
+    own[rank].resize(size + 1);
+    return own[rank][size];
+  }