Fixed some issues for removing periodic boundary conditions.

AMDiS/src/ProblemStat.cc

@@ -505,6 +505,7 @@ namespace AMDiS {
   {
     FUNCNAME("ProblemStat::createSolver()");
 
+#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
     // === create solver ===
     string solverType("0");
     string initFileStr = name + "->solver";
@@ -515,6 +516,7 @@ namespace AMDiS {
     solverCreator->setName(initFileStr);
     solver = solverCreator->create();
     solver->initParameters();
+#endif
   }
 
 

AMDiS/src/io/MacroReader.cc

@@ -48,9 +48,7 @@ namespace AMDiS {
     DegreeOfFreedom **dof = macroInfo->dof;
 
     // === read periodic data =================================
-    if (periodicFile != "") {
-      WARNING("Periodic boundaries may lead to errors in small meshes if element neighbours are not set!\n");
-    
+    if (periodicFile != "") {   
       FILE *file = fopen(periodicFile.c_str(), "r");
       TEST_EXIT(file)("can't open file %s\n", periodicFile.c_str());
 

AMDiS/src/parallel/ElementObjectDatabase.cc

@@ -105,16 +105,15 @@ namespace AMDiS {
 	  periodicEdgeAssoc[edge0].insert(edge1);
 	  
 	  // Add both vertices of the edge to be periodic.
-	  periodicVertices[make_pair(edge0.first, edge1.first)] = boundaryType;
-	  periodicVertices[make_pair(edge0.second, edge1.second)] = boundaryType;
+	  DegreeOfFreedom mappedDof0 = 
+	    mesh->getPeriodicAssociations(boundaryType)[edge0.first];
+	  DegreeOfFreedom mappedDof1 = 
+	    mesh->getPeriodicAssociations(boundaryType)[edge0.second];
+	  periodicVertices[make_pair(edge0.first, mappedDof0)] = boundaryType;
+	  periodicVertices[make_pair(edge0.second, mappedDof1)] = boundaryType;
+
 	  periodicDofAssoc[edge0.first].insert(boundaryType);
-	  periodicDofAssoc[edge0.second].insert(boundaryType);
-	  
-	  TEST_EXIT_DBG(edge0.first == 
-			mesh->getPeriodicAssociations(boundaryType)[edge1.first] &&
-			edge0.second == 
-			mesh->getPeriodicAssociations(boundaryType)[edge1.second])
-	    ("Should not happen!\n");
+	  periodicDofAssoc[edge0.second].insert(boundaryType); 
 	}
       }
       break;
@@ -134,12 +133,16 @@ namespace AMDiS {
 	  periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
 	  
 	  /// Add all three vertices of the face to be periodic.
-	  periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] = 
-	    boundaryType;
-	  periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] = 
-	    boundaryType;
-	  periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] = 
-	    boundaryType;
+	  DegreeOfFreedom mappedDof0 = 
+	    mesh->getPeriodicAssociations(boundaryType)[face0.get<0>()];
+	  DegreeOfFreedom mappedDof1 = 
+	    mesh->getPeriodicAssociations(boundaryType)[face0.get<1>()];
+	  DegreeOfFreedom mappedDof2 = 
+	    mesh->getPeriodicAssociations(boundaryType)[face0.get<2>()];
+
+	  periodicVertices[make_pair(face0.get<0>(), mappedDof0)] = boundaryType;
+	  periodicVertices[make_pair(face0.get<1>(), mappedDof1)] = boundaryType;
+	  periodicVertices[make_pair(face0.get<2>(), mappedDof2)] = boundaryType;
 	  
 	  periodicDofAssoc[face0.get<0>()].insert(boundaryType);
 	  periodicDofAssoc[face0.get<1>()].insert(boundaryType);

AMDiS/src/parallel/ElementObjectDatabase.h

@@ -309,24 +309,32 @@ namespace AMDiS {
       return faceElements[face];
     }
 
-
-    
-    /// Returns a map that maps to each rank all macro elements in this rank that
-    /// have a given vertex DOF in common.
+    /// Returns, for a given vertex, a map that maps from rank numbers to 
+    /// element data objects, which identify on the rank the element which 
+    /// contains this vertex. If more than one element in one subdomain contains
+    /// the vertex, the element with the highest element index is given. If the
+    /// vertex is not contained in a rank's subdomain, it will not be considered
+    /// in this mapping.
     map<int, ElementObjectData>& getElementsInRank(DegreeOfFreedom vertex)
     {
       return vertexInRank[vertex];
     }
 
-    /// Returns a map that maps to each rank all macro elements in this rank that
-    /// have a given edge in common.
+    /// Returns, for a given edge, a map that maps from rank numbers to 
+    /// element data objects, which identify on the rank the element which 
+    /// contains this edge. If more than one element in one subdomain contains
+    /// the edge, the element with the highest element index is given. If the
+    /// edge is not contained in a rank's subdomain, it will not be considered
+    /// in this mapping.
     map<int, ElementObjectData>& getElementsInRank(DofEdge edge)
     {
       return edgeInRank[edge];
     }
 
-    /// Returns a map that maps to each rank all macro elements in this rank that
-    /// have a given face in common.
+    /// Returns, for a given face, a map that maps from rank numbers to 
+    /// element data objects, which identify on the rank the element which 
+    /// contains this face. If the face is not contained in a rank's subdomain, 
+    /// it will not be considered in this mapping.
     map<int, ElementObjectData>& getElementsInRank(DofFace face)
     {
       return faceInRank[face];

AMDiS/src/parallel/InteriorBoundary.cc

@@ -110,7 +110,7 @@ namespace AMDiS {
 	    bound.neighObj.ithObj = it2->second.ithObject;
 	    
 	    bound.type = INTERIOR;
-	    
+
 	    AtomicBoundary& b = getNewOwn(it2->first);
 	    b = bound;
 	    if (geoIndex == EDGE)
@@ -181,6 +181,7 @@ namespace AMDiS {
 	bound.neighObj.subObj = VERTEX;
 	bound.neighObj.ithObj = perDofEl1.ithObject;
 
+
 	if (removePeriodicBoundary) {
 	  bound.type = INTERIOR;
 	  
@@ -191,47 +192,56 @@ namespace AMDiS {
 	  
 	  int owner = std::max(elObjDb.getOwner(it->first.first, level),
 			       elObjDb.getOwner(it->first.second, level));
+
+	  // Consider the following configuration of four elements partitioned
+	  // to four ranks:
+	  //
+	  //  |--------|--------|(*)
+	  //  |       /|       /|
+	  //  |  3   / |  0   / |
+	  //  |     /  |     /  |
+	  //  |    /   |    /   |
+	  //  |   /    |   /    |
+	  //  |  /  2  |  /  1  |
+	  //  | /      | /      |
+	  //  |/-------|/-------|
+	  //
+	  // We are in now interested in vertex (*). For the first, rank 1 owns
+	  // the interior boundary with rank 0 on this vertex. When we remove 
+	  // periodic boundaries, which are applied on the left and right outer
+	  // boundary, rank 3 becomes the owner of this vertex. Rank 0 and rank 1
+	  // will have an interior boundary with rank 3 on this vertex, but rank
+	  // 0 and rank 1 have no more interior boundaries on vertex (*). Thus we
+	  // have to search and remove for this scenario.
+	  
+	  if (owner != globalMpiRank) {
+	    removeOwn(bound.rankObj);
+	    removeOther(bound.rankObj);
+	  }
+
+
+	  // And than we can add the boundary to either own or other part of the
+	  // interior database.
+
 	  if (owner == globalMpiRank) {
-	    for (map<int, ElementObjectData>::iterator it2 = objData.begin();
-	     	 it2 != objData.end(); ++it2) {
-	      if (it2->first == globalMpiRank)
-	     	continue;
-	      
-	      if (!allRanks && levelRanks.count(it2->first) == 0)
-	     	continue;
-	      
-	      AtomicBoundary& b = getNewOwn(it2->first);	  
-	      b = bound;	  
-	      b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
-	      b.neighObj.elIndex = it2->second.elIndex;
-	      b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
-	      b.neighObj.ithObj = it2->second.ithObject;
-	    }
+	    AtomicBoundary& b = getNewOwn(otherElementRank);	  
+	    b = bound;	  
 	  } else {
+	    ElementObjectData& ownerBoundEl = objData[owner];    
+	    bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
+	    bound.neighObj.elIndex = ownerBoundEl.elIndex;
+	    bound.neighObj.elType = -1;
+	    bound.neighObj.ithObj = ownerBoundEl.ithObject;
+
 	    AtomicBoundary& b = getNewOther(owner);
 	    b = bound;
-	    
-	    ElementObjectData& ownerBoundEl = objData[owner];    
-	    b.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex);
-	    b.neighObj.elIndex = ownerBoundEl.elIndex;
-	    b.neighObj.elType = -1;
-	    b.neighObj.ithObj = ownerBoundEl.ithObject;
 	  }
+
 	} else {
 	  bound.type = it->second;
 	  
-	  if (MeshDistributor::sebastianMode) {
-	    if (bound.rankObj.elIndex == 3 &&
-		bound.rankObj.ithObj == 1 ||
-		bound.rankObj.elIndex == 78 &&
-		bound.rankObj.ithObj == 2) {
-	      AtomicBoundary& b = getNewPeriodic(otherElementRank);
-	      b = bound;	    	    
-	    }
-	  } else {
-	    AtomicBoundary& b = getNewPeriodic(otherElementRank);
-	    b = bound;	    	  
-	  }
+	  AtomicBoundary& b = getNewPeriodic(otherElementRank);
+	  b = bound;	  
 	}
       }
     }
@@ -276,24 +286,15 @@ namespace AMDiS {
 			       elObjDb.getOwner(it->first.second, level));
 	  ElementObjectData& rankBoundEl = objData[globalMpiRank];
 
+	  // See comments in the same part of code for VERTEX
+	  if (owner != globalMpiRank) {
+	    removeOwn(bound.rankObj);
+	    removeOther(bound.rankObj);
+	  }
+
 	  if (owner == globalMpiRank) {
-	    for (map<int, ElementObjectData>::iterator it2 = objData.begin();
-	     	 it2 != objData.end(); ++it2) {
-	      if (it2->first == globalMpiRank)
-	     	continue;
-	      
-	      if (!allRanks && levelRanks.count(it2->first) == 0)
-	     	continue;
-	      
-	      AtomicBoundary& b = getNewOwn(it2->first);	  
-	      b = bound;	  
-	      b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
-	      b.neighObj.elIndex = it2->second.elIndex;
-	      b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
-	      b.neighObj.ithObj = it2->second.ithObject;
-	      b.neighObj.reverseMode = 
-		elObjDb.getEdgeReverseMode(rankBoundEl, it2->second);
-	    }
+	    AtomicBoundary& b = getNewOwn(otherElementRank); 
+	    b = bound;	  
 	  } else {
 	    AtomicBoundary& b = getNewOther(owner);
 	    b = bound;
@@ -368,23 +369,8 @@ namespace AMDiS {
 	  ElementObjectData& rankBoundEl = objData[globalMpiRank];
 
 	  if (owner == globalMpiRank) {
-	    for (map<int, ElementObjectData>::iterator it2 = objData.begin();
-	     	 it2 != objData.end(); ++it2) {
-	      if (it2->first == globalMpiRank)
-	     	continue;
-	      
-	      if (!allRanks && levelRanks.count(it2->first) == 0)
-	     	continue;
-	      
-	      AtomicBoundary& b = getNewOwn(it2->first);	  
-	      b = bound;	  
-	      b.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
-	      b.neighObj.elIndex = it2->second.elIndex;
-	      b.neighObj.elType = elObjDb.getElementType(it2->second.elIndex);
-	      b.neighObj.ithObj = it2->second.ithObject;
-	      b.neighObj.reverseMode = 
-		elObjDb.getFaceReverseMode(rankBoundEl, it2->second);
-	    }
+	    AtomicBoundary& b = getNewOwn(otherElementRank);
+	    b = bound;	  
 	  } else {
 	    AtomicBoundary& b = getNewOther(owner);
 	    b = bound;
@@ -413,6 +399,7 @@ namespace AMDiS {
       }
     }
 
+
     // === Once we have this information, we must care about the order of the ===
     // === atomic bounds in the three boundary handling object. Eventually    ===
     // === all the boundaries have to be in the same order on both ranks that ===
@@ -454,19 +441,25 @@ namespace AMDiS {
 
 	// If the expected object is not at place, search for it.
 
-	BoundaryObject &recvedBound = 
+	BoundaryObject &receivedBound = 
 	  stdMpi.getRecvData()[rank][j].rankObj;
 
-	if ((rankIt->second)[j].neighObj != recvedBound) {
+	if ((rankIt->second)[j].neighObj != receivedBound) {
 	  unsigned int k = j + 1;
 
-	  for (; k < rankIt->second.size(); k++)
- 	    if ((rankIt->second)[k].neighObj == recvedBound)
+	  for (; k < rankIt->second.size(); k++) {
+ 	    if ((rankIt->second)[k].neighObj == receivedBound)
 	      break;
+	  }
 
 	  // The element must always be found, because the list is just in
 	  // another order.
-	  TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
+	  TEST_EXIT_DBG(k < rankIt->second.size())
+	    ("Cannot find element %d/%d/%d received from rank %d!\n",
+	     receivedBound.elIndex, 
+	     receivedBound.subObj, 
+	     receivedBound.ithObj,
+	     rank);
 
 	  // Swap the current with the found element.
 	  AtomicBoundary tmpBound = (rankIt->second)[k];
@@ -531,6 +524,11 @@ namespace AMDiS {
 	}
       }     
     } // periodicBoundary.boundary.size() > 0
+
+
+    // === Run verification procedure. ===
+    
+    verifyBoundary();
   }
 
 
@@ -644,8 +642,37 @@ namespace AMDiS {
   }
 
 
+  void InteriorBoundary::verifyBoundary()
+  {
+    FUNCNAME("InteriorBoundary::verifyBoundary()");
+
+    // === Check if no other boundery rank object is also included in own ===
+    // === boundary part, which would make no sence.                      ===
+
+    std::set<BoundaryObject> allOwnBounds;
+    for (map<int, vector<AtomicBoundary> >::iterator it = own.begin(); 
+	 it != own.end(); ++it)
+      for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
+	   bIt != it->second.end(); ++bIt)
+	allOwnBounds.insert(bIt->rankObj);
+
+    for (map<int, vector<AtomicBoundary> >::iterator it = other.begin(); 
+	 it != other.end(); ++it)
+      for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
+	   bIt != it->second.end(); ++bIt)
+	if (allOwnBounds.count(bIt->rankObj)) {
+	  ERROR_EXIT("Boundary %d/%d/%d is included in both, own and other interior boundary part!\n",
+		     bIt->rankObj.elIndex,
+		     bIt->rankObj.subObj,
+		     bIt->rankObj.ithObj);
+	}
+  }
+
+
   AtomicBoundary& InteriorBoundary::getNewOwn(int rank)
   {
+    FUNCNAME("InteriorBoundary::getNewOwn()");
+
     int size = own[rank].size();
     own[rank].resize(size + 1);
     return own[rank][size];
@@ -654,6 +681,8 @@ namespace AMDiS {
 
   AtomicBoundary& InteriorBoundary::getNewOther(int rank)
   {
+    FUNCNAME("InteriorBoundary::getNewOther()");
+
     int size = other[rank].size();
     other[rank].resize(size + 1);
     return other[rank][size];
@@ -662,12 +691,86 @@ namespace AMDiS {
 
   AtomicBoundary& InteriorBoundary::getNewPeriodic(int rank)
   {
+    FUNCNAME("InteriorBoundary::getNewPeriodic()");
+
     int size = periodic[rank].size();
     periodic[rank].resize(size + 1);
     return periodic[rank][size];
   }
 
 
+  bool InteriorBoundary::checkOther(AtomicBoundary& bound, int rank)
+  {
+    FUNCNAME("InteriorBoundary::checkOther()");
+
+    int globalMpiRank = MPI::COMM_WORLD.Get_rank();
+    TEST_EXIT(rank > globalMpiRank)
+      ("Wrong MPI ranks: %d %d\n", rank, globalMpiRank);
+
+    bool found = false;
+
+    if (other.count(rank)) {
+      vector<AtomicBoundary>& otherBounds = other[rank];
+      for (int i = 0; i < static_cast<int>(otherBounds.size()); i++) {
+	if (otherBounds[i].rankObj == bound.rankObj) {
+	  if (otherBounds[i].neighObj != bound.neighObj) {
+	    ERROR_EXIT("If this really can happen, I have to thing about this!\n");
+	  } else {
+	    found = true;
+	    break;
+	  }
+	}
+      }
+    }
+
+    return found;
+  }
+
+
+  bool InteriorBoundary::removeOwn(BoundaryObject& bound)
+  {
+    FUNCNAME("InteriorBoundary::removeOwn()");
+
+    bool removed = false;
+
+    for (map<int, vector<AtomicBoundary> >::iterator it = own.begin();
+	 it != own.end(); ++it) {
+      for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
+	   bIt != it->second.end(); ++bIt) {
+	if (bIt->rankObj == bound) {
+	  it->second.erase(bIt);
+	  removed = true;
+	  break;
+	}
+      }
+    }
+
+    return removed;
+  }
+
+
+  bool InteriorBoundary::removeOther(BoundaryObject& bound)
+  {
+    FUNCNAME("InteriorBoundary::removeOther()");
+
+    bool removed = false;
+
+    for (map<int, vector<AtomicBoundary> >::iterator it = other.begin();
+	 it != other.end(); ++it) {
+      for (vector<AtomicBoundary>::iterator bIt = it->second.begin();
+	   bIt != it->second.end(); ++bIt) {
+	if (bIt->rankObj == bound) {
+	  it->second.erase(bIt);
+	  removed = true;
+	  break;
+	}
+      }
+    }
+
+    return removed;    
+  }
+
+
   void InteriorBoundary::serializeExcludeList(ostream &out, 
 					      ExcludeList &list)
   {

AMDiS/src/parallel/InteriorBoundary.h

@@ -86,12 +86,32 @@ namespace AMDiS {
     void deserialize(istream &in, Mesh *mesh);
 
   private:
+    /// In this function, we put some verification procedures to check for 
+    /// consistency if the created interior boundary data. This function is
+    /// enabled even for optimized mode to check all possible meshes. Thus,
+    /// everything herein should be fast. If the code is stable, we can think
+    /// to remove this function.
+    void verifyBoundary();
+
     AtomicBoundary& getNewOwn(int rank);
 
     AtomicBoundary& getNewOther(int rank);
 
     AtomicBoundary& getNewPeriodic(int rank);
 
+    /// Checks whether the given boundary is already a other boundary with 
+    /// given rank number. Returns true, if the bound is already in the other
+    /// boundary database.
+    bool checkOther(AtomicBoundary& bound, int rank);
+
+    /// Removes the given boundary object from all owned boundaries. Returns 
+    /// true, if at least one object has been removed, otherwise false.