Skip to content

GitLab

Commit 7a605e6e authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on parallelization.

parent d81aa41d
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/ElementObjectData.cc
View file @ 7a605e6e
......@@ -2,36 +2,54 @@
namespace AMDiS {
void ElementObjects::createRankData()
void ElementObjects::createRankData(std::map<int, int>& macroElementRankMap)
{
FUNCNAME("ElementObjects::createRankData()");
vertexOwner.clear();
vertexInRank.clear();
for (std::map<DegreeOfFreedom, std::vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
vertexInRank[it->first][elOwner] = *it2;
if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
}
}
edgeOwner.clear();
edgeInRank.clear();
for (std::map<DofEdge, std::vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
edgeInRank[it->first][elOwner] = *it2;
if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
}
}
faceOwner.clear();
faceInRank.clear();
for (std::map<DofFace, std::vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex)
faceInRank[it->first][elOwner] = *it2;
if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
}
}
}
......
AMDiS/src/parallel/ElementObjectData.h
View file @ 7a605e6e
......@@ -50,33 +50,29 @@ namespace AMDiS {
class ElementObjects {
public:
ElementObjects(std::map<int, int> vec)
: elementInRank(vec),
iterGeoPos(CENTER)
ElementObjects()
: iterGeoPos(CENTER)
{}
void addVertex(DegreeOfFreedom vertex,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound));
vertexOwner[vertex] = std::max(vertexOwner[vertex], elementInRank[elIndex]);
}
void addEdge(DofEdge edge,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound));
edgeOwner[edge] = std::max(edgeOwner[edge], elementInRank[elIndex]);
}
void addFace(DofFace face,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
faceElements[face].push_back(ElementObjectData(elIndex, ith, bound));
faceOwner[face] = std::max(faceOwner[face], elementInRank[elIndex]);
}
void createRankData();
void createRankData(std::map<int, int>& macroElementRankMap);
bool iterate(GeoIndex pos)
{
......@@ -244,8 +240,6 @@ namespace AMDiS {
}
private:
std::map<int, int> elementInRank;
std::map<DegreeOfFreedom, std::vector<ElementObjectData> > vertexElements;
std::map<DofEdge, std::vector<ElementObjectData> > edgeElements;
std::map<DofFace, std::vector<ElementObjectData> > faceElements;
......
AMDiS/src/parallel/InteriorBoundary.h
View file @ 7a605e6e
......@@ -216,6 +216,11 @@ namespace AMDiS {
public:
InteriorBoundary() {}
void clear()
{
boundary.clear();
}
AtomicBoundary& getNewAtomic(int rank);
/// Writes this object to a file.
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 7a605e6e
......@@ -141,7 +141,7 @@ namespace AMDiS {
// === Create new global and local DOF numbering. ===
createLocalGlobalNumbering();
// createLocalGlobalNumbering();
// === Remove all macro elements that are not part of the rank partition. ===
......@@ -150,6 +150,8 @@ namespace AMDiS {
macroElementStructureConsisten = true;
updateLocalGlobalNumbering(true);
// === Reset all DOFAdmins of the mesh. ===
updateDofAdmins();
......@@ -187,7 +189,7 @@ namespace AMDiS {
#endif
mesh->dofCompress();
updateLocalGlobalNumbering();
updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -563,7 +565,7 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
mesh->dofCompress();
updateLocalGlobalNumbering();
updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -1142,6 +1144,8 @@ namespace AMDiS {
#endif
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
}
......@@ -1149,127 +1153,28 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
createBoundaryDataStructure();
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
TEST_EXIT_DBG(rankIt->first != mpiRank)
("It is not allowed to have an interior boundary within a rank partition!\n");
if (rankIt->first < mpiRank)
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[rankIt->first] = rankIt->second;
}
createMeshElementData();
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
createBoundaryData();
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
} // periodicBoundary.boundary.size() > 0
}
void MeshDistributor::createBoundaryDataStructure()
void MeshDistributor::updateInteriorBoundaryInfo()
{
FUNCNAME("MeshDistributor::createBoundaryDataStructure()");
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
// Data structure to store all sub-objects of all elements of the mesh.
ElementObjects elObjects(partitionVec);
// Maps to each element index a pointer to the corresponding element.
std::map<int, Element*> elIndexMap;
// Maps to each element index the type of this element.
std::map<int, int> elIndexTypeMap;
elObjects.createRankData(partitionVec);
// The following three data structures store periodic DOFs, edges and faces,
// i.e.,
std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
createBoundaryData();
}
// Stores to each DOF all its periodic associations.
std::map<DegreeOfFreedom, std::set<BoundaryType> > periodicDofAssoc;
void MeshDistributor::createMeshElementData()
{
FUNCNAME("MeshDistributor::createMeshElementData()");
// === Phase 1, fills the data structures defined above. ===
// === Fills macro element data structures. ===
TraverseStack stack;
ElInfo *elInfo =
......@@ -1279,8 +1184,8 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n");
Element *el = elInfo->getElement();
elIndexMap[el->getIndex()] = el;
elIndexTypeMap[el->getIndex()] = elInfo->getType();
macroElIndexMap[el->getIndex()] = el;
macroElIndexTypeMap[el->getIndex()] = elInfo->getType();
// === Add all sub object of the element to the variable elObjects. ===
......@@ -1295,7 +1200,7 @@ namespace AMDiS {
elObjects.addFace(el->getFace(i), el->getIndex(), i);
// === ===
// === Get periodic boundary information. ===
switch (mesh->getDim()) {
case 2:
......@@ -1346,9 +1251,9 @@ namespace AMDiS {
}
// === PHASE 2 ===
// === Create mesh element data for this rank. ===
elObjects.createRankData();
elObjects.createRankData(partitionVec);
// === Search for interectly connected vertices in periodic boundaries. ===
......@@ -1383,9 +1288,22 @@ namespace AMDiS {
}
}
}
}
void MeshDistributor::createBoundaryData()
{
FUNCNAME("MeshDistributor::createBoundaryData()");
// === PHASE 3 ===
// === Clear all relevant data structures, ===
myIntBoundary.clear();
otherIntBoundary.clear();
periodicBoundary.clear();
// === Create interior boundary data structure. ===
for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) {
GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim());
......@@ -1397,9 +1315,9 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[mpiRank];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex];
bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject;
......@@ -1419,9 +1337,9 @@ namespace AMDiS {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex];
bound.neighObj.el = macroElIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = elIndexTypeMap[it2->second.elIndex];
bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex];
bound.neighObj.subObj = geoIndex;
bound.neighObj.ithObj = it2->second.ithObject;
......@@ -1443,7 +1361,7 @@ namespace AMDiS {
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex];
bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = geoIndex;
......@@ -1464,7 +1382,7 @@ namespace AMDiS {
}
// === PHASE 4 ===
// === Create periodic boundary data structure. ===
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin();
it != periodicDofs.end(); ++it) {
......@@ -1479,13 +1397,13 @@ namespace AMDiS {
ElementObjectData& perDofEl1 = elIt->second;
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perDofEl0.elIndex];
bound.rankObj.el = macroElIndexMap[perDofEl0.elIndex];
bound.rankObj.elIndex = perDofEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perDofEl0.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[perDofEl0.elIndex];
bound.rankObj.subObj = VERTEX;
bound.rankObj.ithObj = perDofEl0.ithObject;
bound.neighObj.el = elIndexMap[perDofEl1.elIndex];
bound.neighObj.el = macroElIndexMap[perDofEl1.elIndex];
bound.neighObj.elIndex = perDofEl1.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = VERTEX;
......@@ -1519,13 +1437,13 @@ namespace AMDiS {
ElementObjectData& perEdgeEl1 = elObjects.getElements(it->first.second)[0];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perEdgeEl0.elIndex];
bound.rankObj.el = macroElIndexMap[perEdgeEl0.elIndex];
bound.rankObj.elIndex = perEdgeEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perEdgeEl0.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[perEdgeEl0.elIndex];
bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = perEdgeEl0.ithObject;
bound.neighObj.el = elIndexMap[perEdgeEl1.elIndex];
bound.neighObj.el = macroElIndexMap[perEdgeEl1.elIndex];
bound.neighObj.elIndex = perEdgeEl1.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = EDGE;
......@@ -1542,6 +1460,102 @@ namespace AMDiS {
b.neighObj.setReverseMode(b.rankObj, feSpace);
}
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
TEST_EXIT_DBG(rankIt->first != mpiRank)
("It is not allowed to have an interior boundary within a rank partition!\n");
if (rankIt->first < mpiRank)
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[rankIt->first] = rankIt->second;
}
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
} // periodicBoundary.boundary.size() > 0
}
......@@ -1733,7 +1747,7 @@ namespace AMDiS {
}
void MeshDistributor::updateLocalGlobalNumbering()
void MeshDistributor::updateLocalGlobalNumbering(bool b)
{
FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()");
......@@ -1767,6 +1781,10 @@ namespace AMDiS {
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
int nRankAllDofs = rankDofs.size();
if (b)
for (unsigned int i = 0; i < rankDofs.size(); i++)
*const_cast<DegreeOfFreedom*>(rankDofs[i]) = i;
// === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDofs to boundaryDofs. ===
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 7a605e6e
......@@ -28,11 +28,11 @@
#include <vector>
#include <mpi.h>
#include "parallel/InteriorBoundary.h"
#include "Global.h"
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "FiniteElemSpace.h"
#include "parallel/InteriorBoundary.h"
#include "Serializer.h"
#include "BoundaryManager.h"
#include "ElementObjectData.h"
......@@ -240,7 +240,11 @@ namespace AMDiS {
*/
void createInteriorBoundaryInfo();
void createBoundaryDataStructure();
void updateInteriorBoundaryInfo();
void createMeshElementData();
void createBoundaryData();
/// Removes all macro elements from the mesh that are not part of ranks partition.
void removeMacroElements();
......@@ -249,7 +253,7 @@ namespace AMDiS {
void createLocalGlobalNumbering();
/// Updates the local and global DOF numbering after the mesh has been changed.
void updateLocalGlobalNumbering();
void updateLocalGlobalNumbering(bool b);
/** \brief
* Creates to all dofs in rank's partition that are on a periodic boundary the
......@@ -498,6 +502,24 @@ namespace AMDiS {
/// Number of DOFs in the whole domain.
int nOverallDofs;
// Data structure to store all sub-objects of all elements of the macro mesh.
ElementObjects elObjects;