And fixed the bug with multilevel interior boundary creation.

AMDiS/src/parallel/ElementObjectDatabase.cc

@@ -581,28 +581,30 @@ namespace AMDiS {
     int owner = -1;
     vector<ElementObjectData> *objData;
     
-   switch (iterGeoPos) {
-   case VERTEX:
-     objData = &(vertexElements[vertexIter->first]);
-     break;
-   case EDGE:
-     objData = &(edgeElements[edgeIter->first]);
-     break;
-   case FACE:
-     objData = &(faceElements[faceIter->first]);
-     break;
-   }
-
-   std::set<int> &levelRanks = levelData->getLevelRanks(level);
-   bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
-
-   for (vector<ElementObjectData>::iterator it = objData->begin();
-	it != objData->end(); ++it) {
-     int elRank = (*macroElementRankMap)[it->elIndex];
-     if (allRanks || levelData->getLevelRanks(level).count(elRank))
-       owner = std::max(owner, elRank);
-   }
-
+    switch (iterGeoPos) {
+    case VERTEX:
+      objData = &(vertexElements[vertexIter->first]);
+      break;
+    case EDGE:
+      objData = &(edgeElements[edgeIter->first]);
+      break;
+    case FACE:
+      objData = &(faceElements[faceIter->first]);
+      break;
+    }
+    
+    std::set<int> &levelRanks = levelData->getLevelRanks(level);
+    bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
+    
+    for (vector<ElementObjectData>::iterator it = objData->begin();
+	 it != objData->end(); ++it) {
+      int elRank = (*macroElementRankMap)[it->elIndex];
+      if (allRanks || levelRanks.count(elRank))
+	owner = std::max(owner, elRank);
+    }
+    
+    TEST_EXIT_DBG(owner >= 0)("Cannot find owner on level %d\n", level);
+    
     return owner;
   }
 

AMDiS/src/parallel/InteriorBoundary.cc

@@ -40,6 +40,7 @@ namespace AMDiS {
     int levelMpiRank = mpiComm.Get_rank();
     int globalMpiRank = MPI::COMM_WORLD.Get_rank();
     std::set<int> levelRanks = levelData.getLevelRanks(level);
+    bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
 
     // === Create interior boundary data structure. ===
     
@@ -53,11 +54,10 @@ namespace AMDiS {
 	if (!(objData.count(globalMpiRank) && objData.size() > 1))
 	  continue;
 
-#if 1
 	// Test, if the boundary object defines an interior boundary within the
 	// ranks of the MPI group. If not, go to next element.
 	bool boundaryWithinMpiGroup = false;
-	if (levelRanks.size() == 1 && *(levelRanks.begin()) == -1) {
+	if (allRanks) {
 	  boundaryWithinMpiGroup = true;
 	} else {
 	  for (map<int, ElementObjectData>::iterator it = objData.begin();
@@ -70,9 +70,9 @@ namespace AMDiS {
 	}
 	if (!boundaryWithinMpiGroup)
 	  continue;
-#endif	
 
 	int owner = elObjDb.getIterateOwner(level);
+	MSG("OWNER IN LEVEL %d IS %d\n",level, owner);
 	ElementObjectData& rankBoundEl = objData[globalMpiRank];
 	
 	AtomicBoundary bound;
@@ -98,7 +98,10 @@ namespace AMDiS {
 	       it2 != objData.end(); ++it2) {
 	    if (it2->first == globalMpiRank)
 	      continue;
-	    
+
+	    if (!allRanks && levelRanks.count(it2->first) == 0)
+	      continue;
+
 	    bound.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
 	    bound.neighObj.elIndex = it2->second.elIndex;
 	    bound.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
@@ -280,17 +283,11 @@ namespace AMDiS {
       stdMpi.recv(other);
     } else {
       for (RankToBoundMap::iterator rankIt = own.begin();	 
-	   rankIt != own.end(); ++rankIt) {
-	int r = levelData.mapRank(rankIt->first, 0, level);
-	TEST_EXIT_DBG(r >= 0)("Should not happen!\n");
-	stdMpi.send(r, rankIt->second);
-      }
+	   rankIt != own.end(); ++rankIt)
+	stdMpi.send(levelData.mapRank(rankIt->first, 0, level), rankIt->second);
       for (RankToBoundMap::iterator rankIt = other.begin();
-	   rankIt != other.end(); ++rankIt) {
-	int r = levelData.mapRank(rankIt->first, 0, level);
-	TEST_EXIT_DBG(r >= 0)("Should not happen!\n");
-	stdMpi.recv(r);
-      }
+	   rankIt != other.end(); ++rankIt)
+	stdMpi.recv(levelData.mapRank(rankIt->first, 0, level));
     }
 
     stdMpi.startCommunication();
@@ -304,16 +301,20 @@ namespace AMDiS {
     for (RankToBoundMap::iterator rankIt = other.begin();
 	 rankIt != other.end(); ++rankIt) {
 
-      // === We have received from rank "rankIt->first" the ordered list of   ===
-      // === element indices. Now, we have to sort the corresponding list in  ===
-      // === this rank to get the same order.                                 ===
+      int rank = rankIt->first;
+      if (level > 0)
+	rank = levelData.mapRank(rank, 0, level);
+
+      // === We have received from "rank" the ordered list of element       ===
+      // === indices. Now, we have to sort the corresponding list in this   ===
+      // === rank to get the same order.                                    ===
      
       for (unsigned int j = 0; j < rankIt->second.size(); j++) {
 
 	// If the expected object is not at place, search for it.
 
 	BoundaryObject &recvedBound = 
-	  stdMpi.getRecvData()[rankIt->first][j].rankObj;
+	  stdMpi.getRecvData()[rank][j].rankObj;
 
 	if ((rankIt->second)[j].neighObj != recvedBound) {
 	  unsigned int k = j + 1;

AMDiS/src/parallel/MeshLevelData.h

@@ -115,8 +115,9 @@ namespace AMDiS {
 
       MPI::Group::Translate_ranks(mpiGroups[fromLevel], 1, &fromRank, 
 				  mpiGroups[toLevel], &toRank);
-      if (toRank == MPI::UNDEFINED)
-	toRank = -1;
+
+      TEST_EXIT_DBG(toRank != MPI::UNDEFINED)
+	("Should not happen!\n");
 
       return toRank;			
     }