Commit 7a605e6e authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on parallelization.

parent d81aa41d
...@@ -2,36 +2,54 @@ ...@@ -2,36 +2,54 @@
namespace AMDiS { namespace AMDiS {
void ElementObjects::createRankData() void ElementObjects::createRankData(std::map<int, int>& macroElementRankMap)
{ {
FUNCNAME("ElementObjects::createRankData()");
vertexOwner.clear();
vertexInRank.clear();
for (std::map<DegreeOfFreedom, std::vector<ElementObjectData> >::iterator it = vertexElements.begin(); for (std::map<DegreeOfFreedom, std::vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) { it != vertexElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
int elOwner = elementInRank[it2->elIndex]; it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex) if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elOwner] = *it2; vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
} }
} }
edgeOwner.clear();
edgeInRank.clear();
for (std::map<DofEdge, std::vector<ElementObjectData> >::iterator it = edgeElements.begin(); for (std::map<DofEdge, std::vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) { it != edgeElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
int elOwner = elementInRank[it2->elIndex]; it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex) if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elOwner] = *it2; edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
} }
} }
faceOwner.clear();
faceInRank.clear();
for (std::map<DofFace, std::vector<ElementObjectData> >::iterator it = faceElements.begin(); for (std::map<DofFace, std::vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) { it != faceElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
int elOwner = elementInRank[it2->elIndex]; it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex) if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elOwner] = *it2; faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
} }
} }
} }
......
...@@ -50,33 +50,29 @@ namespace AMDiS { ...@@ -50,33 +50,29 @@ namespace AMDiS {
class ElementObjects { class ElementObjects {
public: public:
ElementObjects(std::map<int, int> vec) ElementObjects()
: elementInRank(vec), : iterGeoPos(CENTER)
iterGeoPos(CENTER)
{} {}
void addVertex(DegreeOfFreedom vertex, void addVertex(DegreeOfFreedom vertex,
int elIndex, int ith, BoundaryType bound = INTERIOR) int elIndex, int ith, BoundaryType bound = INTERIOR)
{ {
vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound)); vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound));
vertexOwner[vertex] = std::max(vertexOwner[vertex], elementInRank[elIndex]);
} }
void addEdge(DofEdge edge, void addEdge(DofEdge edge,
int elIndex, int ith, BoundaryType bound = INTERIOR) int elIndex, int ith, BoundaryType bound = INTERIOR)
{ {
edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound)); edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound));
edgeOwner[edge] = std::max(edgeOwner[edge], elementInRank[elIndex]);
} }
void addFace(DofFace face, void addFace(DofFace face,
int elIndex, int ith, BoundaryType bound = INTERIOR) int elIndex, int ith, BoundaryType bound = INTERIOR)
{ {
faceElements[face].push_back(ElementObjectData(elIndex, ith, bound)); faceElements[face].push_back(ElementObjectData(elIndex, ith, bound));
faceOwner[face] = std::max(faceOwner[face], elementInRank[elIndex]);
} }
void createRankData(); void createRankData(std::map<int, int>& macroElementRankMap);
bool iterate(GeoIndex pos) bool iterate(GeoIndex pos)
{ {
...@@ -244,8 +240,6 @@ namespace AMDiS { ...@@ -244,8 +240,6 @@ namespace AMDiS {
} }
private: private:
std::map<int, int> elementInRank;
std::map<DegreeOfFreedom, std::vector<ElementObjectData> > vertexElements; std::map<DegreeOfFreedom, std::vector<ElementObjectData> > vertexElements;
std::map<DofEdge, std::vector<ElementObjectData> > edgeElements; std::map<DofEdge, std::vector<ElementObjectData> > edgeElements;
std::map<DofFace, std::vector<ElementObjectData> > faceElements; std::map<DofFace, std::vector<ElementObjectData> > faceElements;
......
...@@ -216,6 +216,11 @@ namespace AMDiS { ...@@ -216,6 +216,11 @@ namespace AMDiS {
public: public:
InteriorBoundary() {} InteriorBoundary() {}
void clear()
{
boundary.clear();
}
AtomicBoundary& getNewAtomic(int rank); AtomicBoundary& getNewAtomic(int rank);
/// Writes this object to a file. /// Writes this object to a file.
......
...@@ -141,7 +141,7 @@ namespace AMDiS { ...@@ -141,7 +141,7 @@ namespace AMDiS {
// === Create new global and local DOF numbering. === // === Create new global and local DOF numbering. ===
createLocalGlobalNumbering(); // createLocalGlobalNumbering();
// === Remove all macro elements that are not part of the rank partition. === // === Remove all macro elements that are not part of the rank partition. ===
...@@ -150,6 +150,8 @@ namespace AMDiS { ...@@ -150,6 +150,8 @@ namespace AMDiS {
macroElementStructureConsisten = true; macroElementStructureConsisten = true;
updateLocalGlobalNumbering(true);
// === Reset all DOFAdmins of the mesh. === // === Reset all DOFAdmins of the mesh. ===
updateDofAdmins(); updateDofAdmins();
...@@ -187,7 +189,7 @@ namespace AMDiS { ...@@ -187,7 +189,7 @@ namespace AMDiS {
#endif #endif
mesh->dofCompress(); mesh->dofCompress();
updateLocalGlobalNumbering(); updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. === // === Update periodic mapping, if there are periodic boundaries. ===
...@@ -563,7 +565,7 @@ namespace AMDiS { ...@@ -563,7 +565,7 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. === // === Because the mesh has been changed, update the DOF numbering and mappings. ===
mesh->dofCompress(); mesh->dofCompress();
updateLocalGlobalNumbering(); updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. === // === Update periodic mapping, if there are periodic boundaries. ===
...@@ -1142,6 +1144,8 @@ namespace AMDiS { ...@@ -1142,6 +1144,8 @@ namespace AMDiS {
#endif #endif
partitioner->fillCoarsePartitionVec(&partitionVec); partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
} }
...@@ -1149,127 +1153,28 @@ namespace AMDiS { ...@@ -1149,127 +1153,28 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()"); FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
createBoundaryDataStructure(); createMeshElementData();
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
TEST_EXIT_DBG(rankIt->first != mpiRank)
("It is not allowed to have an interior boundary within a rank partition!\n");
if (rankIt->first < mpiRank)
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[rankIt->first] = rankIt->second;
}
stdMpi.send(sendBounds.boundary); createBoundaryData();
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
} // periodicBoundary.boundary.size() > 0
} }
void MeshDistributor::createBoundaryDataStructure() void MeshDistributor::updateInteriorBoundaryInfo()
{ {
FUNCNAME("MeshDistributor::createBoundaryDataStructure()"); FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
// Data structure to store all sub-objects of all elements of the mesh. elObjects.createRankData(partitionVec);
ElementObjects elObjects(partitionVec);
// Maps to each element index a pointer to the corresponding element.
std::map<int, Element*> elIndexMap;
// Maps to each element index the type of this element.
std::map<int, int> elIndexTypeMap;
// The following three data structures store periodic DOFs, edges and faces, createBoundaryData();
// i.e., }
std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
// Stores to each DOF all its periodic associations.
std::map<DegreeOfFreedom, std::set<BoundaryType> > periodicDofAssoc;
void MeshDistributor::createMeshElementData()
{
FUNCNAME("MeshDistributor::createMeshElementData()");
// === Phase 1, fills the data structures defined above. === // === Fills macro element data structures. ===
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = ElInfo *elInfo =
...@@ -1279,8 +1184,8 @@ namespace AMDiS { ...@@ -1279,8 +1184,8 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n"); TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n");
Element *el = elInfo->getElement(); Element *el = elInfo->getElement();
elIndexMap[el->getIndex()] = el; macroElIndexMap[el->getIndex()] = el;
elIndexTypeMap[el->getIndex()] = elInfo->getType(); macroElIndexTypeMap[el->getIndex()] = elInfo->getType();
// === Add all sub object of the element to the variable elObjects. === // === Add all sub object of the element to the variable elObjects. ===
...@@ -1295,7 +1200,7 @@ namespace AMDiS { ...@@ -1295,7 +1200,7 @@ namespace AMDiS {
elObjects.addFace(el->getFace(i), el->getIndex(), i); elObjects.addFace(el->getFace(i), el->getIndex(), i);
// === === // === Get periodic boundary information. ===
switch (mesh->getDim()) { switch (mesh->getDim()) {
case 2: case 2:
...@@ -1346,9 +1251,9 @@ namespace AMDiS { ...@@ -1346,9 +1251,9 @@ namespace AMDiS {
} }
// === PHASE 2 === // === Create mesh element data for this rank. ===
elObjects.createRankData(); elObjects.createRankData(partitionVec);
// === Search for interectly connected vertices in periodic boundaries. === // === Search for interectly connected vertices in periodic boundaries. ===
...@@ -1383,9 +1288,22 @@ namespace AMDiS { ...@@ -1383,9 +1288,22 @@ namespace AMDiS {
} }
} }
} }
}
void MeshDistributor::createBoundaryData()
{
FUNCNAME("MeshDistributor::createBoundaryData()");
// === PHASE 3 ===
// === Clear all relevant data structures, ===
myIntBoundary.clear();
otherIntBoundary.clear();
periodicBoundary.clear();
// === Create interior boundary data structure. ===
for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) { for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) {
GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim()); GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim());
...@@ -1397,9 +1315,9 @@ namespace AMDiS { ...@@ -1397,9 +1315,9 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[mpiRank]; ElementObjectData& rankBoundEl = objData[mpiRank];
AtomicBoundary bound; AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex]; bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex; bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex]; bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = geoIndex; bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject; bound.rankObj.ithObj = rankBoundEl.ithObject;
...@@ -1419,9 +1337,9 @@ namespace AMDiS { ...@@ -1419,9 +1337,9 @@ namespace AMDiS {
if (it2->first == mpiRank) if (it2->first == mpiRank)
continue; continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex]; bound.neighObj.el = macroElIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex; bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = elIndexTypeMap[it2->second.elIndex]; bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex];
bound.neighObj.subObj = geoIndex; bound.neighObj.subObj = geoIndex;
bound.neighObj.ithObj = it2->second.ithObject; bound.neighObj.ithObj = it2->second.ithObject;
...@@ -1443,7 +1361,7 @@ namespace AMDiS { ...@@ -1443,7 +1361,7 @@ namespace AMDiS {
ElementObjectData& ownerBoundEl = objData[owner]; ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex]; bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex; bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = -1;
bound.neighObj.subObj = geoIndex; bound.neighObj.subObj = geoIndex;
...@@ -1464,7 +1382,7 @@ namespace AMDiS { ...@@ -1464,7 +1382,7 @@ namespace AMDiS {
} }
// === PHASE 4 === // === Create periodic boundary data structure. ===
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin(); for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin();
it != periodicDofs.end(); ++it) { it != periodicDofs.end(); ++it) {
...@@ -1479,13 +1397,13 @@ namespace AMDiS { ...@@ -1479,13 +1397,13 @@ namespace AMDiS {
ElementObjectData& perDofEl1 = elIt->second; ElementObjectData& perDofEl1 = elIt->second;
AtomicBoundary bound; AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perDofEl0.elIndex]; bound.rankObj.el = macroElIndexMap[perDofEl0.elIndex];
bound.rankObj.elIndex = perDofEl0.elIndex; bound.rankObj.elIndex = perDofEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perDofEl0.elIndex]; bound.rankObj.elType = macroElIndexTypeMap[perDofEl0.elIndex];
bound.rankObj.subObj = VERTEX; bound.rankObj.subObj = VERTEX;
bound.rankObj.ithObj = perDofEl0.ithObject; bound.rankObj.ithObj = perDofEl0.ithObject;
bound.neighObj.el = elIndexMap[perDofEl1.elIndex]; bound.neighObj.el = macroElIndexMap[perDofEl1.elIndex];
bound.neighObj.elIndex = perDofEl1.elIndex; bound.neighObj.elIndex = perDofEl1.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = -1;
bound.neighObj.subObj = VERTEX; bound.neighObj.subObj = VERTEX;
...@@ -1519,13 +1437,13 @@ namespace AMDiS { ...@@ -1519,13 +1437,13 @@ namespace AMDiS {
ElementObjectData& perEdgeEl1 = elObjects.getElements(it->first.second)[0]; ElementObjectData& perEdgeEl1 = elObjects.getElements(it->first.second)[0];
AtomicBoundary bound; AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perEdgeEl0.elIndex]; bound.rankObj.el = macroElIndexMap[perEdgeEl0.elIndex];
bound.rankObj.elIndex = perEdgeEl0.elIndex; bound.rankObj.elIndex = perEdgeEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perEdgeEl0.elIndex]; bound.rankObj.elType = macroElIndexTypeMap[perEdgeEl0.elIndex];
bound.rankObj.subObj = EDGE; bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = perEdgeEl0.ithObject; bound.rankObj.ithObj = perEdgeEl0.ithObject;
bound.neighObj.el = elIndexMap[perEdgeEl1.elIndex]; bound.neighObj.el = macroElIndexMap[perEdgeEl1.elIndex];
bound.neighObj.elIndex = perEdgeEl1.elIndex; bound.neighObj.elIndex = perEdgeEl1.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = -1;
bound.neighObj.subObj = EDGE; bound.neighObj.subObj = EDGE;
...@@ -1542,6 +1460,102 @@ namespace AMDiS { ...@@ -1542,6 +1460,102 @@ namespace AMDiS {
b.neighObj.setReverseMode(b.rankObj, feSpace); b.neighObj.setReverseMode(b.rankObj, feSpace);
} }
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neig