diff --git a/AMDiS/src/parallel/PetscSolverFeti.cc b/AMDiS/src/parallel/PetscSolverFeti.cc index d9660b959091c77798f725ad882fcdb8d6697fea..5d875198b8e0643cf14b8c59fd102dfed9a2ef71 100644 --- a/AMDiS/src/parallel/PetscSolverFeti.cc +++ b/AMDiS/src/parallel/PetscSolverFeti.cc @@ -447,7 +447,12 @@ namespace AMDiS { for (DofContainer::iterator it = allBoundaryDofs.begin(); it != allBoundaryDofs.end(); ++it) if (!isPrimal(feSpace, **it)) - dualDofMap[feSpace].insertRankDof(**it); + if (meshLevel == 0) { + dualDofMap[feSpace].insertRankDof(**it); + } else { + if (meshDistributor->getDofMapSd()[feSpace].isRankDof(**it)) + dualDofMap[feSpace].insertRankDof(**it); + } } @@ -463,14 +468,69 @@ namespace AMDiS { // === Create for each dual node that is owned by the rank, the set === // === of ranks that contain this node (denoted by W(x_j)). === - for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), meshLevel, feSpace); - !it.end(); it.nextRank()) - for (; !it.endDofIter(); it.nextDof()) { - if (!isPrimal(feSpace, it.getDofIndex())) { - boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank); - boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank()); + if (meshLevel == 0) { + for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), + feSpace); + !it.end(); it.nextRank()) { + for (; !it.endDofIter(); it.nextDof()) { + if (!isPrimal(feSpace, it.getDofIndex())) { + boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank); + boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank()); + } } } + } else { + StdMpi<vector<int> > stdMpi(mpiComm); + + for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), + meshLevel, feSpace); + !it.end(); it.nextRank()) { + + vector<int> subdomainRankDofs; + subdomainRankDofs.reserve(it.getDofs().size()); + + for (; !it.endDofIter(); it.nextDof()) { + if (meshDistributor->getDofMapSd()[feSpace].isRankDof(it.getDofIndex())) { + MSG("IN SD DOF\n"); + subdomainRankDofs.push_back(1); + } else { + MSG("NOT IN SD DOF\n"); + subdomainRankDofs.push_back(0); + } + } + + MSG("SEND SIZE %d TO RANK %d\n", subdomainRankDofs.size(), it.getRank()); + + stdMpi.send(it.getRank(), subdomainRankDofs); + } + + for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), + meshLevel, feSpace); + !it.end(); it.nextRank()) + stdMpi.recv(it.getRank()); + + stdMpi.startCommunication(); + + for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), + meshLevel, feSpace); + !it.end(); it.nextRank()) { + for (; !it.endDofIter(); it.nextDof()) { + if (!isPrimal(feSpace, it.getDofIndex())) { + boundaryDofRanks[feSpace][it.getDofIndex()].insert(mpiRank); + + if (it.getDofCounter() >= stdMpi.getRecvData(it.getRank()).size()) + MSG("VERY BIG SHIT: %d from %d\n", stdMpi.getRecvData(it.getRank()).size(), it.getRank()); + + if (stdMpi.getRecvData(it.getRank())[it.getDofCounter()]) + boundaryDofRanks[feSpace][it.getDofIndex()].insert(it.getRank()); + } + } + } + } + + for (DofIndexToPartitions::iterator it = boundaryDofRanks[feSpace].begin(); + it != boundaryDofRanks[feSpace].end(); ++it) + MSG("DOF %d IS DUAL IN %d RANKS\n", it->first, it->second.size()); // === Communicate these sets for all rank owned dual nodes to other ===