diff --git a/AMDiS/src/parallel/ElementObjectDatabase.cc b/AMDiS/src/parallel/ElementObjectDatabase.cc index bd310a88cc7a757ff91f6efa59671fb5a0bb5c37..2450ab3605d6d00ea3f35e468b4d6144ff7cc53c 100644 --- a/AMDiS/src/parallel/ElementObjectDatabase.cc +++ b/AMDiS/src/parallel/ElementObjectDatabase.cc @@ -581,28 +581,30 @@ namespace AMDiS { int owner = -1; vector<ElementObjectData> *objData; - switch (iterGeoPos) { - case VERTEX: - objData = &(vertexElements[vertexIter->first]); - break; - case EDGE: - objData = &(edgeElements[edgeIter->first]); - break; - case FACE: - objData = &(faceElements[faceIter->first]); - break; - } - - std::set<int> &levelRanks = levelData->getLevelRanks(level); - bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); - - for (vector<ElementObjectData>::iterator it = objData->begin(); - it != objData->end(); ++it) { - int elRank = (*macroElementRankMap)[it->elIndex]; - if (allRanks || levelData->getLevelRanks(level).count(elRank)) - owner = std::max(owner, elRank); - } - + switch (iterGeoPos) { + case VERTEX: + objData = &(vertexElements[vertexIter->first]); + break; + case EDGE: + objData = &(edgeElements[edgeIter->first]); + break; + case FACE: + objData = &(faceElements[faceIter->first]); + break; + } + + std::set<int> &levelRanks = levelData->getLevelRanks(level); + bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); + + for (vector<ElementObjectData>::iterator it = objData->begin(); + it != objData->end(); ++it) { + int elRank = (*macroElementRankMap)[it->elIndex]; + if (allRanks || levelRanks.count(elRank)) + owner = std::max(owner, elRank); + } + + TEST_EXIT_DBG(owner >= 0)("Cannot find owner on level %d\n", level); + return owner; } diff --git a/AMDiS/src/parallel/InteriorBoundary.cc b/AMDiS/src/parallel/InteriorBoundary.cc index 3436156df0ae4403ffc1791fc9300895ee8426a8..020a3f295c1d0c516c768a629794c3ed989fcf94 100644 --- a/AMDiS/src/parallel/InteriorBoundary.cc +++ b/AMDiS/src/parallel/InteriorBoundary.cc @@ -40,6 +40,7 @@ namespace AMDiS { int levelMpiRank = mpiComm.Get_rank(); int globalMpiRank = MPI::COMM_WORLD.Get_rank(); std::set<int> levelRanks = levelData.getLevelRanks(level); + bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); // === Create interior boundary data structure. === @@ -53,11 +54,10 @@ namespace AMDiS { if (!(objData.count(globalMpiRank) && objData.size() > 1)) continue; -#if 1 // Test, if the boundary object defines an interior boundary within the // ranks of the MPI group. If not, go to next element. bool boundaryWithinMpiGroup = false; - if (levelRanks.size() == 1 && *(levelRanks.begin()) == -1) { + if (allRanks) { boundaryWithinMpiGroup = true; } else { for (map<int, ElementObjectData>::iterator it = objData.begin(); @@ -70,9 +70,9 @@ namespace AMDiS { } if (!boundaryWithinMpiGroup) continue; -#endif int owner = elObjDb.getIterateOwner(level); + MSG("OWNER IN LEVEL %d IS %d\n",level, owner); ElementObjectData& rankBoundEl = objData[globalMpiRank]; AtomicBoundary bound; @@ -98,7 +98,10 @@ namespace AMDiS { it2 != objData.end(); ++it2) { if (it2->first == globalMpiRank) continue; - + + if (!allRanks && levelRanks.count(it2->first) == 0) + continue; + bound.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex); bound.neighObj.elIndex = it2->second.elIndex; bound.neighObj.elType = elObjDb.getElementType(it2->second.elIndex); @@ -280,17 +283,11 @@ namespace AMDiS { stdMpi.recv(other); } else { for (RankToBoundMap::iterator rankIt = own.begin(); - rankIt != own.end(); ++rankIt) { - int r = levelData.mapRank(rankIt->first, 0, level); - TEST_EXIT_DBG(r >= 0)("Should not happen!\n"); - stdMpi.send(r, rankIt->second); - } + rankIt != own.end(); ++rankIt) + stdMpi.send(levelData.mapRank(rankIt->first, 0, level), rankIt->second); for (RankToBoundMap::iterator rankIt = other.begin(); - rankIt != other.end(); ++rankIt) { - int r = levelData.mapRank(rankIt->first, 0, level); - TEST_EXIT_DBG(r >= 0)("Should not happen!\n"); - stdMpi.recv(r); - } + rankIt != other.end(); ++rankIt) + stdMpi.recv(levelData.mapRank(rankIt->first, 0, level)); } stdMpi.startCommunication(); @@ -304,16 +301,20 @@ namespace AMDiS { for (RankToBoundMap::iterator rankIt = other.begin(); rankIt != other.end(); ++rankIt) { - // === We have received from rank "rankIt->first" the ordered list of === - // === element indices. Now, we have to sort the corresponding list in === - // === this rank to get the same order. === + int rank = rankIt->first; + if (level > 0) + rank = levelData.mapRank(rank, 0, level); + + // === We have received from "rank" the ordered list of element === + // === indices. Now, we have to sort the corresponding list in this === + // === rank to get the same order. === for (unsigned int j = 0; j < rankIt->second.size(); j++) { // If the expected object is not at place, search for it. BoundaryObject &recvedBound = - stdMpi.getRecvData()[rankIt->first][j].rankObj; + stdMpi.getRecvData()[rank][j].rankObj; if ((rankIt->second)[j].neighObj != recvedBound) { unsigned int k = j + 1; diff --git a/AMDiS/src/parallel/MeshLevelData.h b/AMDiS/src/parallel/MeshLevelData.h index babd55106095def445c4d49faaa209c81e4249c1..a68f5e77a1168a415f63b79f206c22da2507141c 100644 --- a/AMDiS/src/parallel/MeshLevelData.h +++ b/AMDiS/src/parallel/MeshLevelData.h @@ -115,8 +115,9 @@ namespace AMDiS { MPI::Group::Translate_ranks(mpiGroups[fromLevel], 1, &fromRank, mpiGroups[toLevel], &toRank); - if (toRank == MPI::UNDEFINED) - toRank = -1; + + TEST_EXIT_DBG(toRank != MPI::UNDEFINED) + ("Should not happen!\n"); return toRank; }