Fixed some small bugs in 3d parallelization.

AMDiS/src/InteriorBoundary.h

@@ -83,7 +83,7 @@ namespace AMDiS {
 
     bool reverseMode;
 
-    std::vector<int> notIncludedSubStructures;
+    std::vector<std::pair<GeoIndex, int> > notIncludedSubStructures;
   };
 
   /** \brief 

AMDiS/src/ParallelDomainBase.cc

@@ -1047,6 +1047,9 @@ namespace AMDiS {
     // this edge.
     std::map<GlobalEdge, BoundaryObject> rankEdges;
 
+    std::map<DegreeOfFreedom, int> dofOwner;
+    std::map<DegreeOfFreedom, BoundaryObject> rankDofs;
+
     // === Traverse whole mesh and fill the maps edgeOwner and rankEdges. ===
 
     elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
@@ -1065,11 +1068,17 @@ namespace AMDiS {
 
 	// Update the owner of the current edge.
 	edgeOwner[edge] = max(edgeOwner[edge], partitionVec[el->getIndex()]);
+	dofOwner[dof0] = max(dofOwner[dof0], partitionVec[el->getIndex()]);
+	dofOwner[dof1] = max(dofOwner[dof1], partitionVec[el->getIndex()]);
 
 	// If the edge is part of an element that is part of rank's domain, add it
 	// to the set of all rank's edges.
-	if (partitionData && partitionData->getPartitionStatus() == IN)
-	  rankEdges[edge] = BoundaryObject(el, elInfo->getType(), EDGE, i);	
+	if (partitionData && partitionData->getPartitionStatus() == IN) {
+	  BoundaryObject b(el, elInfo->getType(), EDGE, i);
+	  rankEdges[edge] = b;
+	  rankDofs[dof0] = b;
+	  rankDofs[dof1] = b;
+	}
       }      
 
       elInfo = stack.traverseNext(elInfo);
@@ -1080,6 +1089,8 @@ namespace AMDiS {
     // Stores all edges at rank's interior boundaries.
     std::set<GlobalEdge> rankBoundaryEdges;
 
+    std::set<DegreeOfFreedom> rankBoundaryDofs;
+
     // First, traverse the rank owned elements af the interior boundaries.
     for (RankToBoundMap::iterator rankIt = myIntBoundary.boundary.begin();
 	 rankIt != myIntBoundary.boundary.end(); ++rankIt) {
@@ -1098,10 +1109,20 @@ namespace AMDiS {
 	  // Otherwise, it is part of the interior boundary and we add it to the set
 	  // rankBoundaryEdges.
 	  if (edgeOwner[edge] > mpiRank)
-	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(edgeNo);
+	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(std::pair<GeoIndex, int>(EDGE, edgeNo));
 	  else
 	    rankBoundaryEdges.insert(edge);
 	}
+      
+	for (int j = 0; j < 3; j++) {
+	  int dofNo = el->getVertexOfPosition(FACE, rankIt->second[i].rankObj.ithObj, j);
+	  DegreeOfFreedom dof = localIndices[dofNo];
+
+	  if (dofOwner[dof] > mpiRank)
+	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(std::pair<GeoIndex, int>(VERTEX, dofNo));
+	  else
+	    rankBoundaryDofs.insert(dof);
+	}
       }
     }
 
@@ -1119,13 +1140,25 @@ namespace AMDiS {
 	  GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));	
 
 	  if (edgeOwner[edge] > rankIt->first)
-	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(edgeNo);
+	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(std::pair<GeoIndex, int>(EDGE, edgeNo));
 	  else
 	    rankBoundaryEdges.insert(edge);
+
+	}
+
+	for (int j = 0; j < 3; j++) {
+	  int dofNo = el->getVertexOfPosition(FACE, rankIt->second[i].rankObj.ithObj, j);
+	  DegreeOfFreedom dof = localIndices[dofNo];
+	  
+	  if (dofOwner[dof] > rankIt->first)
+	    rankIt->second[i].rankObj.notIncludedSubStructures.push_back(std::pair<GeoIndex, int>(VERTEX, dofNo));
+	  else
+	    rankBoundaryDofs.insert(dof);
 	}	
       }
     }
 
+
     // === Create the new interior boundaries consisting only of edges. This    ===
     // === boundaries are created on that ranks, which do not own the boundary  ===
     // === but are on the other side of the edge. Than, theses ranks inform the ===
@@ -1156,10 +1189,14 @@ namespace AMDiS {
 	if (find(ownerEdges.begin(), ownerEdges.end(), it->first) == ownerEdges.end()) {
 	  ownerEdges.push_back(it->first);	
 	  recvEdgesObj[edgeOwner[it->first]].push_back(it->second);
+
+	  rankBoundaryDofs.insert(it->first.first);
+	  rankBoundaryDofs.insert(it->first.second);
 	}
       }
     }
 
+
     // === Send all edge interior boundary infos to the owner of the new edge === 
     // === interior boundaries.                                               ===
 
@@ -1195,6 +1232,9 @@ namespace AMDiS {
 	b.neighObj = stdMpiEdgeObj.getRecvData(it->first)[i];
 	
 	sendObjects[it->first].push_back(b.rankObj);
+
+	rankBoundaryDofs.insert(it->second[i].first);
+	rankBoundaryDofs.insert(it->second[i].second);
       }
     }
 
@@ -1222,6 +1262,14 @@ namespace AMDiS {
 	}
       }
     }
+
+
+    for (std::map<DegreeOfFreedom, BoundaryObject>::iterator it = rankDofs.begin();
+	 it != rankDofs.end(); ++it) {
+      if (dofOwner[it->first] > mpiRank && rankBoundaryDofs.count(it->first) == 0) {
+	MSG("HAB DICH: %d\n", it->first);
+      }
+    }
     
 
     // === Once we have this information, we must care about the order of the atomic ===
@@ -1317,6 +1365,8 @@ namespace AMDiS {
 	}
       }     
     } // periodicBoundary.boundary.size() > 0
+
+    //    exit(0);
   }
 
 

AMDiS/src/Tetrahedron.cc

@@ -4,6 +4,7 @@
 #include "CoarseningManager.h"
 #include "FixVec.h"
 #include "ElementDofIterator.h"
+#include "Global.h"
 
 namespace AMDiS {
 
@@ -260,9 +261,9 @@ namespace AMDiS {
 	nextBound1.reverseMode = false;
 	
 	bool addDof = true;
-	for (std::vector<int>::iterator it = bound.notIncludedSubStructures.begin(); 
+	for (std::vector<std::pair<GeoIndex, int> >::iterator it = bound.notIncludedSubStructures.begin(); 
 	     it != bound.notIncludedSubStructures.end(); ++it)
-	  if (*it == 0)
+	  if (it->first == EDGE && it->second == 0)
 	    addDof = false;
 	
 	if (bound.reverseMode) {
@@ -326,9 +327,11 @@ namespace AMDiS {
 
     switch (bound.subObj) {
     case FACE:      
-      for (std::vector<int>::iterator it = bound.notIncludedSubStructures.begin(); 
-	   it != bound.notIncludedSubStructures.end(); ++it)
-	*it = edgeOfChild[bound.elType][ithChild][*it];    
+      for (std::vector<std::pair<GeoIndex, int> >::iterator it = bound.notIncludedSubStructures.begin(); 
+   	   it != bound.notIncludedSubStructures.end(); ++it) {
+	if (it->first == EDGE)
+	  it->second = edgeOfChild[bound.elType][ithChild][it->second];
+      }
       
       bound.ithObj = sideOfChild[bound.elType][ithChild][bound.ithObj];
       bound.elType = (bound.elType + 1) % 3;