FETI-DP uses subdomains consisting of multiple ranks. Still cannot run a solve on these systems.

AMDiS/src/parallel/DofComm.cc

@@ -46,7 +46,7 @@ namespace AMDiS {
   {
     FUNCNAME("DofComm::Iterator::setNextFeMap()");
 
-    if (dataIter != dofComm.data[0].end()) {
+    if (dataIter != dofComm.data[traverseLevel].end()) {
       TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
 
       feMapIter = dataIter->second.begin();

AMDiS/src/parallel/DofComm.h

@@ -78,19 +78,26 @@ namespace AMDiS {
 	       const FiniteElemSpace *fe = NULL)
 	: dofComm(dc),
 	  dofCounter(-1),
-	  traverseFeSpace(fe)
+	  traverseFeSpace(fe),
+	  traverseLevel(0)
       {
-	FUNCNAME("DofComm::Iterator::Iterator()");
-
-	dataIter = dofComm.data[0].begin();
+	goFirst();
+      }
 
-	while (setNextFeMap() == false)
-	  ++dataIter;
+      Iterator(DofComm &dc,
+	       int level,
+	       const FiniteElemSpace *fe = NULL)
+	: dofComm(dc),
+	  dofCounter(-1),
+	  traverseFeSpace(fe),
+	  traverseLevel(level)
+      {
+	goFirst();
       }
 
       inline bool end()
       {
-	return (dataIter == dofComm.data[0].end());
+	return (dataIter == dofComm.data[traverseLevel].end());
       }
       
       inline void nextRank()
@@ -181,6 +188,14 @@ namespace AMDiS {
       }
 
     protected:
+      void goFirst()
+      {
+	dataIter = dofComm.data[traverseLevel].begin();
+
+	while (setNextFeMap() == false)
+	  ++dataIter;
+      }
+
       bool setNextFeMap();
 
     protected:
@@ -195,6 +210,8 @@ namespace AMDiS {
       int dofCounter;
 
       const FiniteElemSpace *traverseFeSpace;
+
+      int traverseLevel;
     };
 
 

AMDiS/src/parallel/InteriorBoundary.h

@@ -216,14 +216,18 @@ namespace AMDiS {
 	  TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");
 	  TEST_EXIT_DBG(level == 1)("Only 2-level method supported!\n");
 
-	  int rankInLevel = levelData->mapRank(mapIt->first, level - 1, level);
-	  MSG("rankInLevel %d\n", rankInLevel);
-	}
-
- 	while (mapIt->second.size() == 0) {
- 	  ++mapIt;
-	  if (mapIt == bound.boundary.end())
-	    return;
+	  while (levelData->rankInSubdomain(mapIt->first, level) ||
+		 mapIt->second.size() == 0) {
+	    ++mapIt;
+	    if (mapIt == bound.boundary.end())
+	      return;	  
+	  }
+	} else {
+	  while (mapIt->second.size() == 0) {
+	    ++mapIt;
+	    if (mapIt == bound.boundary.end())
+	      return;
+	  }
 	}
       }
 

AMDiS/src/parallel/MeshDistributor.cc

@@ -665,14 +665,15 @@ namespace AMDiS {
 
 
   void MeshDistributor::getAllBoundaryDofs(const FiniteElemSpace *feSpace,
+					   int level,
 					   DofContainer& dofs)
   {
     FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
 
     DofContainerSet dofSet;
-    for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(sendDofs, level, feSpace); !it.end(); it.nextRank())
       dofSet.insert(it.getDofs().begin(), it.getDofs().end());
-    for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
       dofSet.insert(it.getDofs().begin(), it.getDofs().end());
 
     dofs.clear();
@@ -1036,23 +1037,6 @@ namespace AMDiS {
   }
 
 
-  void MeshDistributor::createBoundaryDofs(const FiniteElemSpace *feSpace,
-					   std::set<DegreeOfFreedom> &boundaryDofs)
-  {
-    FUNCNAME("MeshDistributor::createBoundaryDofs()");
-
-    boundaryDofs.clear();
-
-    for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
-      for (; !it.endDofIter(); it.nextDof())
-	boundaryDofs.insert(it.getDofIndex());
-
-    for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
-      for (; !it.endDofIter(); it.nextDof())
-	boundaryDofs.insert(it.getDofIndex());
-  }
-
-
   void MeshDistributor::serialize(ostream &out, DofContainer &data)
   {    
     int vecSize = data.size();

AMDiS/src/parallel/MeshDistributor.h

@@ -373,11 +373,6 @@ namespace AMDiS {
       return initialized;
     }
 
-    /// Creates a set of all DOFs that are on interior boundaries of rank's
-    /// domain. Thus, it creates the union of \ref sendDofs and \ref recvDofs.
-    void createBoundaryDofs(const FiniteElemSpace *feSpace,
-			    std::set<DegreeOfFreedom> &boundaryDofs);
-
     // Writes all data of this object to an output stream.
     void serialize(ostream &out);
 
@@ -436,7 +431,8 @@ namespace AMDiS {
       createBoundaryDofFlag = flag;
     }
 
-    BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, int level = 0)
+    BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, 
+					int level = 0)
     {
       FUNCNAME("MeshDistributor::getBoundaryDofInfo()");
 
@@ -447,7 +443,8 @@ namespace AMDiS {
       return boundaryDofInfo[level][feSpace];
     }
 
-    void getAllBoundaryDofs(const FiniteElemSpace *feSpace,
+    void getAllBoundaryDofs(const FiniteElemSpace *feSpace, 
+			    int level,
 			    DofContainer& dofs);
 
     const ElementObjectDatabase& getElementObjectDb() 

AMDiS/src/parallel/MeshLevelData.h

@@ -98,6 +98,13 @@ namespace AMDiS {
       return toRank;			
     }
 
+    bool rankInSubdomain(int rank, int level)
+    {
+      TEST_EXIT_DBG(level < nLevel)("Should not happen!\n");
+
+      return static_cast<bool>(levelRanks[level].count(rank));
+    }
+
   protected:
     int nLevel;
 

AMDiS/src/parallel/ParallelDebug.h

@@ -63,8 +63,8 @@ namespace AMDiS {
 
     /** \brief
      * This function is used for debugging only. It traverses all interior boundaries
-     * and compares the DOF indices on them with the dof indices of the boundarys
-     * neighbours. The function fails, when dof indices on an interior boundary do
+     * and compares the DOF indices on them with the DOF indices of the boundarys
+     * neighbours. The function fails, when DOF indices on an interior boundary do
      * not fit together.
      *
      * \param[in]  pdb           Parallel problem definition used for debugging.

AMDiS/src/parallel/PetscSolverFeti.cc

@@ -10,6 +10,7 @@
 // See also license.opensource.txt in the distribution.
 
 
+#include "AMDiS.h"
 #include "parallel/PetscSolverFeti.h"
 #include "parallel/PetscSolverFetiStructs.h"
 #include "parallel/StdMpi.h"
@@ -315,7 +316,6 @@ namespace AMDiS {
     DofIndexSet primals;
     DofContainerSet& vertices = 
       meshDistributor->getBoundaryDofInfo(feSpace, meshLevel).geoDofs[VERTEX];
-    TEST_EXIT_DBG(vertices.size())("No primal vertices on this rank!\n");
     for (DofContainerSet::iterator it = vertices.begin(); 
 	 it != vertices.end(); ++it)
       primals.insert(**it);
@@ -339,7 +339,7 @@ namespace AMDiS {
     // === Create global index of the dual nodes on each rank. ===
 
     DofContainer allBoundaryDofs;
-    meshDistributor->getAllBoundaryDofs(feSpace, allBoundaryDofs);
+    meshDistributor->getAllBoundaryDofs(feSpace, meshLevel, allBoundaryDofs);
 
     for (DofContainer::iterator it = allBoundaryDofs.begin();
 	 it != allBoundaryDofs.end(); ++it)
@@ -357,7 +357,7 @@ namespace AMDiS {
 
     boundaryDofRanks[feSpace].clear();
 
-    for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace); 
+    for (DofComm::Iterator it(meshDistributor->getSendDofs(), meshLevel, feSpace); 
 	 !it.end(); it.nextRank())
       for (; !it.endDofIter(); it.nextDof()) {
 	if (!isPrimal(feSpace, it.getDofIndex())) {
@@ -372,20 +372,23 @@ namespace AMDiS {
 
     StdMpi<vector<std::set<int> > > stdMpi(meshDistributor->getMpiComm());
 
-    for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
+    for (DofComm::Iterator it(meshDistributor->getSendDofs(), meshLevel, feSpace);
 	 !it.end(); it.nextRank())
       for (; !it.endDofIter(); it.nextDof())
-	if (!isPrimal(feSpace, it.getDofIndex()))
+	if (!isPrimal(feSpace, it.getDofIndex())) {
+	  MSG("SEND TO RANK %d\n", it.getRank());
 	  stdMpi.getSendData(it.getRank()).push_back(boundaryDofRanks[feSpace][it.getDofIndex()]);
+	}
 
     stdMpi.updateSendDataSize();
 
-    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace); 
+    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), meshLevel, feSpace); 
 	 !it.end(); it.nextRank()) {
       bool recvFromRank = false;
       for (; !it.endDofIter(); it.nextDof()) {
 	if (!isPrimal(feSpace, it.getDofIndex())) {
 	  recvFromRank = true;
+	  MSG("RECV FROM RANK %d\n", it.getRank());
 	  break;
 	}
       }
@@ -396,7 +399,7 @@ namespace AMDiS {
 
     stdMpi.startCommunication();
 
-    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace); 
+    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), meshLevel, feSpace); 
 	 !it.end(); it.nextRank()) {
       int i = 0;
       for (; !it.endDofIter(); it.nextDof())
@@ -405,7 +408,6 @@ namespace AMDiS {
 	    stdMpi.getRecvData(it.getRank())[i++];
     }
 
-
     // === Reserve for each dual node, on the rank that owns this node, the ===
     // === appropriate number of Lagrange constraints.                      ===
 

AMDiS/src/parallel/PetscSolverFeti.h

@@ -190,7 +190,7 @@ namespace AMDiS {
     /// in this map. Is used for the Dirichlet preconditioner only.
     ParallelDofMapping interiorDofMap;
 
-    /// Stores to each dual boundary DOF in each finite elment space the set of
+    /// Stores to each dual boundary DOF in each FE space the set of
     /// ranks in which the DOF is contained in.
     map<const FiniteElemSpace*, DofIndexToPartitions> boundaryDofRanks;