Periodic boundary conditions with mixed finite elements in parallel should work now.

AMDiS/src/DOFMatrix.cc

@@ -57,7 +57,7 @@ namespace AMDiS {
     FUNCNAME("DOFMatrix::DOFMatrix()");
 
     TEST_EXIT(rowFeSpace)("No fe space for row!\n");
-  
+
     if (!colFeSpace)
       colFeSpace = rowFeSpace;
 
@@ -211,16 +211,18 @@ namespace AMDiS {
     using namespace mtl;
 
 #if 0
-    std::cout << "----- PRINT MAT " << rowElInfo->getElement()->getIndex() << "--------" << std::endl;
-    std::cout << elMat << std::endl;
-    std::cout << "rows: ";
-    for (int i = 0; i < rowIndices.size(); i++)
-      std::cout << rowIndices[i] << " ";
-    std::cout << std::endl;
-    std::cout << "cols: ";
-    for (int i = 0; i < colIndices.size(); i++)
-      std::cout << colIndices[i] << " ";
-    std::cout << std::endl;
+    if (MPI::COMM_WORLD.Get_rank() == 0) {
+      std::cout << "----- PRINT MAT " << rowElInfo->getElement()->getIndex() << "--------" << std::endl;
+      std::cout << elMat << std::endl;
+      std::cout << "rows: ";
+      for (int i = 0; i < rowIndices.size(); i++)
+	std::cout << rowIndices[i] << " ";
+      std::cout << std::endl;
+      std::cout << "cols: ";
+      for (int i = 0; i < colIndices.size(); i++)
+	std::cout << colIndices[i] << " ";
+      std::cout << std::endl;
+    }
 #endif
          
     for (int i = 0; i < nRow; i++)  {
@@ -229,7 +231,7 @@ namespace AMDiS {
       BoundaryCondition *condition = 
 	bound ? boundaryManager->getBoundaryCondition(bound[i]) : NULL;
 
-      if (condition && condition->isDirichlet()) {
+      if (condition && condition->isDirichlet()) {	
 	if (condition->applyBoundaryCondition()) {
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
 	  if ((*rankDofs)[rowIndices[i]]) 
@@ -241,8 +243,9 @@ namespace AMDiS {
       } else {
 	for (int j = 0; j < nCol; j++) {
 	  DegreeOfFreedom col = colIndices[j];
-	  if (fabs(elMat[i][j]) > 1e-10)
+	  if (fabs(elMat[i][j]) > 1e-10) {
 	    ins[row][col] += elMat[i][j];
+	  }
 	}
       }
     }
@@ -259,7 +262,9 @@ namespace AMDiS {
   {}
 
 
-  void DOFMatrix::assemble(double factor, ElInfo *elInfo, const BoundaryType *bound)
+  void DOFMatrix::assemble(double factor, 
+			   ElInfo *elInfo, 
+			   const BoundaryType *bound)
   {
     FUNCNAME("DOFMatrix::assemble()");
 
@@ -278,7 +283,9 @@ namespace AMDiS {
   }
 
 
-  void DOFMatrix::assemble(double factor, ElInfo *elInfo, const BoundaryType *bound,
+  void DOFMatrix::assemble(double factor, 
+			   ElInfo *elInfo, 
+			   const BoundaryType *bound,
 			   Operator *op)
   {
       FUNCNAME("DOFMatrix::assemble()");

AMDiS/src/ProblemStat.cc

@@ -1356,6 +1356,10 @@ namespace AMDiS {
   {
     FUNCNAME("ProblemStat::addMatrixOperator()");
 
+    TEST_EXIT(i < nComponents && j < nComponents)
+      ("Cannot add matrix operator at position %d/%d. The stationary problem has only %d components!\n",
+       i, j, nComponents);
+
     TEST_EXIT(!boundaryConditionSet)
       ("Do not add operators after boundary conditions were set!\n");
    
@@ -1402,6 +1406,10 @@ namespace AMDiS {
   {
     FUNCNAME("ProblemStat::addVectorOperator()");
 
+    TEST_EXIT(i < nComponents)
+      ("Cannot add vector operator at position %d. The stationary problem has only %d components!\n",
+       i, nComponents);
+
     TEST_EXIT(!boundaryConditionSet)
       ("Do not add operators after boundary conditions were set!\n");
 

AMDiS/src/SubAssembler.h

@@ -64,17 +64,13 @@ namespace AMDiS {
     /// Destructor
     virtual ~SubAssembler() {}
 
-    /** \brief
-     * Calculates the element matrix for elInfo and adds it to mat. Memory
-     * for mat must be provided by the caller.
-     */
+    /// Calculates the element matrix for elInfo and adds it to mat. Memory for
+    /// mat must be provided by the caller.
     virtual void calculateElementMatrix(const ElInfo *elInfo,
 					ElementMatrix& mat) = 0;
 
-    /** \brief
-     * Calculates the element vector for elInfo and adds it to vec. Memory
-     * for vec must be provided by the caller.
-     */
+    /// Calculates the element vector for elInfo and adds it to vec. Memory for
+    /// vec must be provided by the caller.
     virtual void calculateElementVector(const ElInfo *elInfo,
 					ElementVector& vec) = 0;
 
@@ -104,10 +100,8 @@ namespace AMDiS {
     WorldVector<double>* getCoordsAtQPs(const ElInfo* elInfo, 
 					Quadrature *quad = NULL);
 
-    /** \brief
-     * DOFVector dv evaluated at quadrature points.
-     * Used by \ref OperatorTerm::initElement().
-     */
+    /// DOFVector dv evaluated at quadrature points.
+    /// Used by \ref OperatorTerm::initElement().
     template<typename T>
     void getVectorAtQPs(DOFVectorBase<T>* dv, 
 			const ElInfo* elInfo,
@@ -122,10 +116,8 @@ namespace AMDiS {
 			Quadrature *quad,
 			mtl::dense_vector<T>& vecAtQPs);
     
-    /** \brief
-     * Gradients of DOFVector dv evaluated at quadrature points.
-     * Used by \ref OperatorTerm::initElement().
-     */
+    /// Gradients of DOFVector dv evaluated at quadrature points.
+    /// Used by \ref OperatorTerm::initElement().
     template<typename T>
     void getGradientsAtQPs( DOFVectorBase<T>* dv,
 			    const ElInfo* elInfo,
@@ -139,11 +131,10 @@ namespace AMDiS {
 			    Quadrature *quad,
 			    mtl::dense_vector<typename GradientType<T>::type>& grdAtQPs);
 
-    /** \brief
-     * The comp'th component of the derivative of DOFVector dv evaluated at quadrature points.
-     * Used by \ref OperatorTerm::initElement().
-     * Attention: not caching at the moment! Using cache if gradients for read but not for write
-     */
+    /// The comp'th component of the derivative of DOFVector dv evaluated at
+    /// quadrature points. Used by \ref OperatorTerm::initElement().
+    /// Attention: not caching at the moment! Using cache if gradients for read 
+    /// but not for write.
     template<typename T>
     void getDerivativeAtQPs(DOFVectorBase<T>* dv,
 			    const ElInfo* elInfo,
@@ -159,11 +150,9 @@ namespace AMDiS {
 			    int comp,
 			    mtl::dense_vector<T>& grdAtQPs);
 
-    /** \brief
-     * Called once for each ElInfo when \ref calculateElementMatrix() or
-     * \ref calculateElementVector() is called for the first time for this
-     * Element.
-     */
+    /// Called once for each ElInfo when \ref calculateElementMatrix() or
+    /// \ref calculateElementVector() is called for the first time for this
+    /// Element.
     virtual void initElement(const ElInfo *smallElInfo,
 			     const ElInfo *largeElInfo = NULL,
 			     Quadrature *quad = NULL);
@@ -202,16 +191,12 @@ namespace AMDiS {
     /// Column FiniteElemSpace.
     const FiniteElemSpace *colFeSpace;
 
-    /** \brief
-     * Number of rows of the element matrix and length of the element
-     * vector. Is equal to the number of row basis functions
-     */
+    /// Number of rows of the element matrix and length of the element
+    /// vector. Is equal to the number of row basis functions
     int nRow;
 
-    /** \brief
-     * Number of columns of the element matrix. Is equal to the number
-     * of column basis functions
-     */
+    /// Number of columns of the element matrix. Is equal to the number
+    /// of column basis functions
     int nCol;
 
     /// Used for \ref getVectorAtQPs() and \ref getGradientsAtQPs().
@@ -230,16 +215,15 @@ namespace AMDiS {
     std::map<const void*, ValuesAtQPs* > cachedValuesAtQPs;
     std::map<const void*, ValuesAtQPs* > cachedGradientsAtQPs;
     
-    /** \brief
-     * Set and updated by \ref initElement() for each ElInfo. 
-     * coordsAtQPs[i] points to the coordinates of the i-th quadrature point.
-     */
+    /// Set and updated by \ref initElement() for each ElInfo. 
+    /// coordsAtQPs[i] points to the coordinates of the i-th quadrature point.
     WorldVector<double> *coordsAtQPs;
 
     /// Used for \ref getCoordsAtQPs().
     bool coordsValid;
 
-    /// Used for \ref getCoordsAtQP(). Stores the number of allocated WorldVectors.
+    /// Used for \ref getCoordsAtQP(). Stores the number of allocated 
+    /// WorldVectors.
     int coordsNumAllocated;
 
     /// Quadrature object to be used for assembling.

AMDiS/src/Tetrahedron.cc

@@ -376,6 +376,48 @@ namespace AMDiS {
       return;
       break;
     case EDGE:
+      {
+	// === Create boundary information objects for children elements. ===
+
+	BoundaryObject nextBound0 = bound;
+	prepareNextBound(nextBound0, 0);
+
+	BoundaryObject nextBound1 = bound;
+	prepareNextBound(nextBound1, 1);
+
+	// === Check for boundary on children elements. ===
+
+	if ((nextBound0.ithObj >= 0 || nextBound1.ithObj >= 0) &&  child[0]) {
+	  // So, the edge is contained in at least on of the children and the
+	  // element is also refined. Then we have go down further in refinement
+	  // hierarchie.
+
+	  if (bound.reverseMode) {
+	    if (nextBound1.ithObj >= 0)
+	      child[1]->getHigherOrderDofs(feSpace, nextBound1, dofs);	    
+	    if (nextBound0.ithObj >= 0)
+	      child[0]->getHigherOrderDofs(feSpace, nextBound0, dofs);
+	  } else {
+	    if (nextBound0.ithObj >= 0)
+	      child[0]->getHigherOrderDofs(feSpace, nextBound0, dofs);	    
+	    if (nextBound1.ithObj >= 0)
+	      child[1]->getHigherOrderDofs(feSpace, nextBound1, dofs);
+	  }
+	} else {
+	  // Either the edge is not contained in further refined children, or
+	  // the element is not refined further on this edge. Then we can get
+	  // all the DOFs on this edge.
+
+	  ElementDofIterator elDofIter(feSpace, true);
+	  elDofIter.reset(this);
+	  do {
+	    if (elDofIter.getCurrentPos() == 1 && 
+		elDofIter.getCurrentElementPos() == bound.ithObj)
+	      dofs.push_back(elDofIter.getDofPtr());	
+	  } while(elDofIter.next());      
+	}
+      }
+
       break;
     case FACE:
       {      
@@ -403,10 +445,8 @@ namespace AMDiS {
 	  elDofIter.reset(this);
 	  do {
 	    if (elDofIter.getCurrentPos() == 2 && 
-		elDofIter.getCurrentElementPos() == bound.ithObj) {
-	      ERROR_EXIT("Check this, if it will really work!\n");
+		elDofIter.getCurrentElementPos() == bound.ithObj)
 	      dofs.push_back(elDofIter.getDofPtr());	
-	    }	
 	  } while(elDofIter.next());      
 	}
       }

AMDiS/src/Tetrahedron.h

@@ -324,10 +324,10 @@ namespace AMDiS {
     static const int sideOfChild[3][2][4];
 
     /** \brief
-     * edgeOfChild[elType][i][j] is the local edge number of the j-th edge within the 
-     * i-th children of an element of elType. If the value is -1, the edge is not 
-     * included in the element's child. Note that the 0 edge is included in both
-     * children only by its half.
+     * edgeOfChild[elType][i][j] is the local edge number of the j-th edge within
+     * the i-th children of an element of elType. If the value is -1, the edge is
+     * not included in the element's child. Note that the 0 edge is included in
+     * both children only by its half.
      */
     static const int edgeOfChild[3][2][6];
 

AMDiS/src/Triangle.cc

@@ -169,7 +169,7 @@ namespace AMDiS {
       break;      
     default:
       ERROR_EXIT("Should never happen!\n");
-    }
+    }  
   }
 
 

AMDiS/src/ZeroOrderAssembler.cc

@@ -77,9 +77,9 @@ namespace AMDiS {
       newAssembler = new StandardZOA(op, assembler, quad);
     } else {
       if (pwConst)
-       	newAssembler = new PrecalcZOA(op, assembler, quad);
+	newAssembler = new PrecalcZOA(op, assembler, quad);
       else
-    	newAssembler = new FastQuadZOA(op, assembler, quad);      
+     	newAssembler = new FastQuadZOA(op, assembler, quad);      
     }
 
     subAssemblers->push_back(newAssembler);

AMDiS/src/parallel/ElementObjectData.cc

@@ -85,9 +85,12 @@ namespace AMDiS {
 	  periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
 	  
 	  /// Add all three vertices of the face to be periodic.
-	  periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] = boundaryType;
-	  periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] = boundaryType;
-	  periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] = boundaryType;
+	  periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] = 
+	    boundaryType;
+	  periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] = 
+	    boundaryType;
+	  periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] = 
+	    boundaryType;
 	  
 	  periodicDofAssoc[face0.get<0>()].insert(boundaryType);
 	  periodicDofAssoc[face0.get<1>()].insert(boundaryType);
@@ -131,8 +134,8 @@ namespace AMDiS {
 
     TEST_EXIT_DBG(mesh)("Mesh not set!\n");
 
-    // === Return, if there are no periodic vertices, i.e., there are no no  ===
-    // === periodic boundaries in the mesh.                                  ===
+    // === Return, if there are no periodic vertices, i.e., there are no  ===
+    // === periodic boundaries in the mesh.                               ===
     
     if (periodicVertices.size() == 0)
       return;
@@ -141,9 +144,9 @@ namespace AMDiS {
     // === Get all vertex DOFs that have multiple periodic associations. ===
     
     // We group all vertices together, that have either two or three periodic
-    // associations. For rectangular domains in 2D, the four corner vertices have all
-    // two periodic associations. For box domains in 3D, the eight corner vertices
-    // have all three periodic associations.
+    // associations. For rectangular domains in 2D, the four corner vertices have
+    // all two periodic associations. For box domains in 3D, the eight corner
+    // vertices have all three periodic associations.
 
     vector<DegreeOfFreedom> multPeriodicDof2, multPeriodicDof3;
     for (map<DegreeOfFreedom, std::set<BoundaryType> >::iterator it = periodicDofAssoc.begin();

AMDiS/src/parallel/ElementObjectData.h

@@ -126,11 +126,11 @@ namespace AMDiS {
     /** \brief
      * Creates final data of the periodic boundaries. Must be called after all
      * elements of the mesh are added to the object database. Then this functions
-     * search for interectly connected vertices in periodic boundaries. This is only
-     * the case, if there are more than one boundary conditions. Then, e.g., in 2D, 
-     * all edges of a square are iterectly connected. In 3D, if the macro mesh is a
-     * box, all eight vertex nodes and always four of the 12 edges are iterectly 
-     * connected.
+     * search for indirectly connected vertices in periodic boundaries. This is
+     * only the case, if there are more than one boundary conditions. Then, e.g., 
+     * in 2D, all edges of a square are iterectly connected. In 3D, if the macro 
+     * mesh is a box, all eight vertex nodes and always four of the 12 edges are 
+     * indirectly connected.
      */
     void createPeriodicData(const FiniteElemSpace *feSpace);
 

AMDiS/src/parallel/MeshDistributor.cc

@@ -289,17 +289,17 @@ namespace AMDiS {
 
 
     // We have to remove the VertexVectors, which contain periodic assoiciations, 
-    // because they are not valid anymore after some macro elements have been removed
-    // and the corresponding DOFs were deleted.
+    // because they are not valid anymore after some macro elements have been
+    // removed and the corresponding DOFs were deleted.
     for (map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
 	 it != mesh->getPeriodicAssociations().end(); ++it)
       const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
 
     updateLocalGlobalNumbering();
 
-    // === In 3D we have to make some test, if the resulting mesh is valid. If   ===
-    // === it is not valid, there is no possiblity yet to fix this problem, just ===
-    // === exit with an error message.                                           ===
+    // === In 3D we have to make some test, if the resulting mesh is valid.  ===
+    // === If it is not valid, there is no possiblity yet to fix this        ===
+    // === problem, just exit with an error message.                         ===
     
     check3dValidMesh();
 
@@ -320,7 +320,7 @@ namespace AMDiS {
 
     // === Create periodic DOF mapping, if there are periodic boundaries. ===
 
-    createPeriodicMap(feSpaces[0]);
+    createPeriodicMap();
 
 #if (DEBUG != 0)
     ParallelDebug::testPeriodicBoundary(*this);
@@ -335,25 +335,20 @@ namespace AMDiS {
       refineManager->globalRefine(mesh, globalRefinement);
 
       updateLocalGlobalNumbering();
-
      
       // === Update periodic mapping, if there are periodic boundaries. ===     
 
-      createPeriodicMap(feSpaces[0]);
+      createPeriodicMap();
 
 #if (DEBUG != 0)
     ParallelDebug::testPeriodicBoundary(*this);
 #endif
     }
 
-
-    /// === Set DOF rank information to all matrices and vectors. ===
-
+    // Set DOF rank information to all matrices and vectors.
     setRankDofs();
 
-
-    // === Remove periodic boundary conditions in sequential problem definition. ===
-
+    // Remove periodic boundary conditions in sequential problem definition.
     removePeriodicBoundaryConditions();
 
     initialized = true;
@@ -364,6 +359,9 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::addProblemStat()");
 
+    TEST_EXIT_DBG(probStat->getFeSpaces().size())
+      ("No FE spaces in stationary problem!\n");
+
     // === Add FE spaces from stationary problem to mesh distributor. ===
 
     for (unsigned int i = 0; i < probStat->getFeSpaces().size(); i++) {
@@ -845,21 +843,20 @@ namespace AMDiS {
 #endif
 
 
-    // === Because the mesh has been changed, update the DOF numbering and mappings. ===
-
+    // Because the mesh has been changed, update the DOF numbering and mappings.
     updateLocalGlobalNumbering();
 
+    // Update periodic mapping, if there are periodic boundaries.
+    createPeriodicMap();
 
-    // === Update periodic mapping, if there are periodic boundaries. ===
-
-    createPeriodicMap(feSpaces[0]);
 
 #if (DEBUG != 0)
     ParallelDebug::testPeriodicBoundary(*this);
 #endif
 
 
-    // === The mesh has changed, so check if it is required to repartition the mesh. ===
+    // === The mesh has changed, so check if it is required to repartition ===
+    // === the mesh.                                                       ===
 
     nMeshChangesAfterLastRepartitioning++;
 
@@ -900,7 +897,8 @@ namespace AMDiS {
 	maxDofs = std::max(maxDofs, nDofsInRank[i]);
       }      
       int avrgDofs = nOverallDofs / mpiSize;
-      double imbalance = (static_cast<double>(maxDofs - avrgDofs) /  avrgDofs) * 100.0;
+      double imbalance = 
+	(static_cast<double>(maxDofs - avrgDofs) /  avrgDofs) * 100.0;
 
       MSG("Imbalancing factor: %.1f\%\n", imbalance);
     }
@@ -1202,7 +1200,8 @@ namespace AMDiS {
 	 it != newMacroEl.end(); ++it) {
       MacroElement *mel = *it;
 
-      // First, reset all neighbour relations. The correct neighbours will be set later.
+      // First, reset all neighbour relations. The correct neighbours will be 
+      // set later.
       for (int i = 0; i < mesh->getGeo(NEIGH); i++)
 	mel->setNeighbour(i, NULL);
 
@@ -1259,7 +1258,8 @@ namespace AMDiS {
     stdMpi2.startCommunication();
 
 
-    // === Adapte the new macro elements due to the received mesh structure codes. ===
+    // === Adapte the new macro elements due to the received mesh   ===
+    // === structure codes.                                         ===
 
     for (map<int, vector<int> >::iterator it = partitioner->getRecvElements().begin();
 	 it != partitioner->getRecvElements().end(); ++it) {
@@ -1348,7 +1348,7 @@ namespace AMDiS {
 
     // === Update periodic mapping, if there are periodic boundaries. ===
 
-    createPeriodicMap(feSpaces[0]);
+    createPeriodicMap();
 
 
 #if (DEBUG != 0)
@@ -1435,25 +1435,21 @@ namespace AMDiS {
       macroElIndexMap[el->getIndex()] = el;
       macroElIndexTypeMap[el->getIndex()] = elInfo->getType();
      
-      // === Add all sub object of the element to the variable elObjects. ===
+      // Add all sub object of the element to the variable elObjects.
       elObjects.addElement(elInfo);
 
       elInfo = stack.traverseNext(elInfo);
     }
 
 
-    // === Create periodic data, if there are periodic boundary conditions. ===
-    if (elObjects.hasPeriodicData()) {
-      TEST_EXIT(feSpaces.size() == 1)
-	("Sebastian: Na, dass funktioniert auch noch nicht mit mehreren FE spaces. Du weisst schon, wen du jetzt mobben kannst :)!\n");
-    }
+    // Create periodic data, if there are periodic boundary conditions.
     elObjects.createPeriodicData(feSpaces[0]);
 
-    // === Create data about the reverse modes of neighbouring elements. ===
+    // Create data about the reverse modes of neighbouring elements.
     elObjects.createReverseModeData(feSpaces[0], macroElIndexMap, 
 				    macroElIndexTypeMap);
 
-    // === Create mesh element data for this rank. ===
+    // Create mesh element data for this rank.
     elObjects.createRankData(partitionMap);
   }
 
@@ -1482,7 +1478,8 @@ namespace AMDiS {
 	int owner = elObjects.getIterateOwner();
 	ElementObjectData& rankBoundEl = objData[mpiRank];
 	
-	TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])("Should not happen!\n");
+	TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])
+	  ("Should not happen!\n");
 	
 	AtomicBoundary bound;	    	    
 	bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
@@ -1555,12 +1552,8 @@ namespace AMDiS {
       if (elObjects.isInRank(it->first.first, mpiRank) == false)
 	continue;
 
-      TEST_EXIT(feSpaces.size() == 1)("Does not work for multiple fe spaces!\n");
-      WorldVector<double> c0, c1;
-      mesh->getDofIndexCoords(it->first.first, feSpaces[0], c0);
-      mesh->getDofIndexCoords(it->first.second, feSpaces[0], c1);
-
-      ElementObjectData& perDofEl0 = elObjects.getElementsInRank(it->first.first)[mpiRank];
+      ElementObjectData& perDofEl0 = 
+	elObjects.getElementsInRank(it->first.first)[mpiRank];
 
       for (map<int, ElementObjectData>::iterator elIt = elObjects.getElementsInRank(it->first.second).begin();
 	   elIt != elObjects.getElementsInRank(it->first.second).end(); ++elIt) {
@@ -1665,16 +1658,19 @@ namespace AMDiS {
 	b = bound;
      
 	if (mpiRank > otherElementRank)
-	  b.neighObj.reverseMode = elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
+	  b.neighObj.reverseMode = 
+	    elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
 	else
-	  b.rankObj.reverseMode = elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
+	  b.rankObj.reverseMode = 
+	    elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
       }
     }
     
 
-    // === Once we have this information, we must care about the order of the atomic ===
-    // === bounds in the three boundary handling object. Eventually all the bound-   ===
-    // === aries have to be in the same order on both ranks that share the bounday.  ===
+    // === Once we have this information, we must care about the order of the ===
+    // === atomic bounds in the three boundary handling object. Eventually    ===
+    // === all the boundaries have to be in the same order on both ranks that ===
+    // === share the bounday.                                                 ===
 
     StdMpi<vector<AtomicBoundary> > stdMpi(mpiComm);
     stdMpi.send(myIntBoundary.boundary);
@@ -1682,16 +1678,17 @@ namespace AMDiS {
     stdMpi.startCommunication();
 
 
-    // === The information about all neighbouring boundaries has been received. So ===
-    // === the rank tests if its own atomic boundaries are in the same order. If   ===
-    // === not, the atomic boundaries are swaped to the correct order.             ===
+    // === The information about all neighbouring boundaries has been         ===
+    // === received. So the rank tests if its own atomic boundaries are in    ===
+    // === the same order. If not, the atomic boundaries are swaped to the    ===
+    // === correct order.                                                     ===
 
     for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
 	 rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
 
-      // === We have received from rank "rankIt->first" the ordered list of element ===
-      // === indices. Now, we have to sort the corresponding list in this rank to   ===
-      // === get the same order.                                                    ===
+      // === We have received from rank "rankIt->first" the ordered list of   ===
+      // === element indices. Now, we have to sort the corresponding list in  ===
+      // === this rank to get the same order.                                 ===
      
       for (unsigned int j = 0; j < rankIt->second.size(); j++) {
 
@@ -1706,7 +1703,8 @@ namespace AMDiS {
  	    if ((rankIt->second)[k].neighObj == recvedBound)
 	      break;
 
-	  // The element must always be found, because the list is just in another order.
+	  // The element must always be found, because the list is just in
+	  // another order.
 	  TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");