Fixed Lagrange computation problem for multilevle FETI-DP.

AMDiS/src/parallel/PetscSolverFeti.cc

@@ -447,7 +447,12 @@ namespace AMDiS {
     for (DofContainer::iterator it = allBoundaryDofs.begin();
 	 it != allBoundaryDofs.end(); ++it)
       if (!isPrimal(feSpace, **it))
-	dualDofMap[feSpace].insertRankDof(**it);
+	if (meshLevel == 0) {
+	  dualDofMap[feSpace].insertRankDof(**it);
+	} else {
+	  if (meshDistributor->getDofMapSd()[feSpace].isRankDof(**it))
+	    dualDofMap[feSpace].insertRankDof(**it);
+	}	  
   }
 
   
@@ -463,14 +468,69 @@ namespace AMDiS {
     // === Create for each dual node that is owned by the rank, the set ===
     // === of ranks that contain this node (denoted by W(x_j)).         ===
 
-    for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), meshLevel, feSpace); 
-	 !it.end(); it.nextRank())
-      for (; !it.endDofIter(); it.nextDof()) {
-	if (!isPrimal(feSpace, it.getDofIndex())) {
-	  boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank);
-	  boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank());
+    if (meshLevel == 0) {
+      for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), 
+				feSpace); 
+	   !it.end(); it.nextRank()) {
+	for (; !it.endDofIter(); it.nextDof()) {
+	  if (!isPrimal(feSpace, it.getDofIndex())) {
+	    boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank);
+	    boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank());
+	  }
 	}
       }
+    } else {
+      StdMpi<vector<int> > stdMpi(mpiComm);
+
+      for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), 
+				meshLevel, feSpace);
+	   !it.end(); it.nextRank()) {
+
+	vector<int> subdomainRankDofs;
+	subdomainRankDofs.reserve(it.getDofs().size());
+
+	for (; !it.endDofIter(); it.nextDof()) {
+	  if (meshDistributor->getDofMapSd()[feSpace].isRankDof(it.getDofIndex())) {
+	    MSG("IN SD DOF\n");
+	    subdomainRankDofs.push_back(1);
+	  } else {
+	    MSG("NOT IN SD DOF\n");
+	    subdomainRankDofs.push_back(0);
+	  }
+	}
+	
+	MSG("SEND SIZE %d TO RANK %d\n", subdomainRankDofs.size(), it.getRank());
+
+	stdMpi.send(it.getRank(), subdomainRankDofs);
+      }	     
+
+      for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), 
+				meshLevel, feSpace);
+	   !it.end(); it.nextRank())
+	stdMpi.recv(it.getRank());
+
+      stdMpi.startCommunication();
+
+      for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), 
+				meshLevel, feSpace); 
+	   !it.end(); it.nextRank()) {
+	for (; !it.endDofIter(); it.nextDof()) {
+	  if (!isPrimal(feSpace, it.getDofIndex())) {
+	    boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank);
+
+	    if (it.getDofCounter() >= stdMpi.getRecvData(it.getRank()).size())
+	      MSG("VERY BIG SHIT: %d from %d\n", stdMpi.getRecvData(it.getRank()).size(), it.getRank());
+
+	    if (stdMpi.getRecvData(it.getRank())[it.getDofCounter()])
+	      boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank());
+	  }
+	}
+      }
+    }
+
+    for (DofIndexToPartitions::iterator it = boundaryDofRanks[feSpace].begin();
+	 it != boundaryDofRanks[feSpace].end(); ++it)
+      MSG("DOF %d IS DUAL IN %d RANKS\n", it->first, it->second.size());
 	
 
     // === Communicate these sets for all rank owned dual nodes to other ===