Skip to content

GitLab

Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit 7a605e6e authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on parallelization.

parent d81aa41d
Hide whitespace changes
Inline Side-by-side
Showing with 215 additions and 158 deletions
AMDiS/src/parallel/ElementObjectData.cc
View file @ 7a605e6e
......@@ -2,36 +2,54 @@
namespace AMDiS {
void ElementObjects::createRankData()
void ElementObjects::createRankData(std::map<int, int>& macroElementRankMap)
{
FUNCNAME("ElementObjects::createRankData()");
vertexOwner.clear();
vertexInRank.clear();
for (std::map<DegreeOfFreedom, std::vector<ElementObjectData> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
vertexInRank[it->first][elOwner] = *it2;
if (it2->elIndex > vertexInRank[it->first][elementInRank].elIndex)
vertexInRank[it->first][elementInRank] = *it2;
vertexOwner[it->first] = std::max(vertexOwner[it->first], elementInRank);
}
}
edgeOwner.clear();
edgeInRank.clear();
for (std::map<DofEdge, std::vector<ElementObjectData> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
edgeInRank[it->first][elOwner] = *it2;
if (it2->elIndex > edgeInRank[it->first][elementInRank].elIndex)
edgeInRank[it->first][elementInRank] = *it2;
edgeOwner[it->first] = std::max(edgeOwner[it->first], elementInRank);
}
}
faceOwner.clear();
faceInRank.clear();
for (std::map<DofFace, std::vector<ElementObjectData> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = elementInRank[it2->elIndex];
for (std::vector<ElementObjectData>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
int elementInRank = macroElementRankMap[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex)
faceInRank[it->first][elOwner] = *it2;
if (it2->elIndex > faceInRank[it->first][elementInRank].elIndex)
faceInRank[it->first][elementInRank] = *it2;
faceOwner[it->first] = std::max(faceOwner[it->first], elementInRank);
}
}
}
......
AMDiS/src/parallel/ElementObjectData.h
View file @ 7a605e6e
......@@ -50,33 +50,29 @@ namespace AMDiS {
class ElementObjects {
public:
ElementObjects(std::map<int, int> vec)
: elementInRank(vec),
iterGeoPos(CENTER)
ElementObjects()
: iterGeoPos(CENTER)
{}
void addVertex(DegreeOfFreedom vertex,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
vertexElements[vertex].push_back(ElementObjectData(elIndex, ith, bound));
vertexOwner[vertex] = std::max(vertexOwner[vertex], elementInRank[elIndex]);
}
void addEdge(DofEdge edge,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
edgeElements[edge].push_back(ElementObjectData(elIndex, ith, bound));
edgeOwner[edge] = std::max(edgeOwner[edge], elementInRank[elIndex]);
}
void addFace(DofFace face,
int elIndex, int ith, BoundaryType bound = INTERIOR)
{
faceElements[face].push_back(ElementObjectData(elIndex, ith, bound));
faceOwner[face] = std::max(faceOwner[face], elementInRank[elIndex]);
}
void createRankData();
void createRankData(std::map<int, int>& macroElementRankMap);
bool iterate(GeoIndex pos)
{
......@@ -244,8 +240,6 @@ namespace AMDiS {
}
private:
std::map<int, int> elementInRank;
std::map<DegreeOfFreedom, std::vector<ElementObjectData> > vertexElements;
std::map<DofEdge, std::vector<ElementObjectData> > edgeElements;
std::map<DofFace, std::vector<ElementObjectData> > faceElements;
......
AMDiS/src/parallel/InteriorBoundary.h
View file @ 7a605e6e
......@@ -216,6 +216,11 @@ namespace AMDiS {
public:
InteriorBoundary() {}
void clear()
{
boundary.clear();
}
AtomicBoundary& getNewAtomic(int rank);
/// Writes this object to a file.
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 7a605e6e
......@@ -141,7 +141,7 @@ namespace AMDiS {
// === Create new global and local DOF numbering. ===
createLocalGlobalNumbering();
// createLocalGlobalNumbering();
// === Remove all macro elements that are not part of the rank partition. ===
......@@ -150,6 +150,8 @@ namespace AMDiS {
macroElementStructureConsisten = true;
updateLocalGlobalNumbering(true);
// === Reset all DOFAdmins of the mesh. ===
updateDofAdmins();
......@@ -187,7 +189,7 @@ namespace AMDiS {
#endif
mesh->dofCompress();
updateLocalGlobalNumbering();
updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -563,7 +565,7 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
mesh->dofCompress();
updateLocalGlobalNumbering();
updateLocalGlobalNumbering(false);
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -1142,6 +1144,8 @@ namespace AMDiS {
#endif
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
}
......@@ -1149,127 +1153,28 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
createBoundaryDataStructure();
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
TEST_EXIT_DBG(rankIt->first != mpiRank)
("It is not allowed to have an interior boundary within a rank partition!\n");
if (rankIt->first < mpiRank)
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[rankIt->first] = rankIt->second;
}
createMeshElementData();
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
createBoundaryData();
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
} // periodicBoundary.boundary.size() > 0
}
void MeshDistributor::createBoundaryDataStructure()
void MeshDistributor::updateInteriorBoundaryInfo()
{
FUNCNAME("MeshDistributor::createBoundaryDataStructure()");
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
// Data structure to store all sub-objects of all elements of the mesh.
ElementObjects elObjects(partitionVec);
// Maps to each element index a pointer to the corresponding element.
std::map<int, Element*> elIndexMap;
// Maps to each element index the type of this element.
std::map<int, int> elIndexTypeMap;
elObjects.createRankData(partitionVec);
// The following three data structures store periodic DOFs, edges and faces,
// i.e.,
std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
createBoundaryData();
}
// Stores to each DOF all its periodic associations.
std::map<DegreeOfFreedom, std::set<BoundaryType> > periodicDofAssoc;
void MeshDistributor::createMeshElementData()
{
FUNCNAME("MeshDistributor::createMeshElementData()");
// === Phase 1, fills the data structures defined above. ===
// === Fills macro element data structures. ===
TraverseStack stack;
ElInfo *elInfo =
......@@ -1279,8 +1184,8 @@ namespace AMDiS {
TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n");
Element *el = elInfo->getElement();
elIndexMap[el->getIndex()] = el;
elIndexTypeMap[el->getIndex()] = elInfo->getType();
macroElIndexMap[el->getIndex()] = el;
macroElIndexTypeMap[el->getIndex()] = elInfo->getType();
// === Add all sub object of the element to the variable elObjects. ===
......@@ -1295,7 +1200,7 @@ namespace AMDiS {
elObjects.addFace(el->getFace(i), el->getIndex(), i);
// === ===
// === Get periodic boundary information. ===
switch (mesh->getDim()) {
case 2:
......@@ -1346,9 +1251,9 @@ namespace AMDiS {
}
// === PHASE 2 ===
// === Create mesh element data for this rank. ===
elObjects.createRankData();
elObjects.createRankData(partitionVec);
// === Search for interectly connected vertices in periodic boundaries. ===
......@@ -1383,9 +1288,22 @@ namespace AMDiS {
}
}
}
}
void MeshDistributor::createBoundaryData()
{
FUNCNAME("MeshDistributor::createBoundaryData()");
// === PHASE 3 ===
// === Clear all relevant data structures, ===
myIntBoundary.clear();
otherIntBoundary.clear();
periodicBoundary.clear();
// === Create interior boundary data structure. ===
for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) {
GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim());
......@@ -1397,9 +1315,9 @@ namespace AMDiS {
ElementObjectData& rankBoundEl = objData[mpiRank];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex];
bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject;
......@@ -1419,9 +1337,9 @@ namespace AMDiS {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex];
bound.neighObj.el = macroElIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = elIndexTypeMap[it2->second.elIndex];
bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex];
bound.neighObj.subObj = geoIndex;
bound.neighObj.ithObj = it2->second.ithObject;
......@@ -1443,7 +1361,7 @@ namespace AMDiS {
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex];
bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = geoIndex;
......@@ -1464,7 +1382,7 @@ namespace AMDiS {
}
// === PHASE 4 ===
// === Create periodic boundary data structure. ===
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin();
it != periodicDofs.end(); ++it) {
......@@ -1479,13 +1397,13 @@ namespace AMDiS {
ElementObjectData& perDofEl1 = elIt->second;
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perDofEl0.elIndex];
bound.rankObj.el = macroElIndexMap[perDofEl0.elIndex];
bound.rankObj.elIndex = perDofEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perDofEl0.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[perDofEl0.elIndex];
bound.rankObj.subObj = VERTEX;
bound.rankObj.ithObj = perDofEl0.ithObject;
bound.neighObj.el = elIndexMap[perDofEl1.elIndex];
bound.neighObj.el = macroElIndexMap[perDofEl1.elIndex];
bound.neighObj.elIndex = perDofEl1.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = VERTEX;
......@@ -1519,13 +1437,13 @@ namespace AMDiS {
ElementObjectData& perEdgeEl1 = elObjects.getElements(it->first.second)[0];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[perEdgeEl0.elIndex];
bound.rankObj.el = macroElIndexMap[perEdgeEl0.elIndex];
bound.rankObj.elIndex = perEdgeEl0.elIndex;
bound.rankObj.elType = elIndexTypeMap[perEdgeEl0.elIndex];
bound.rankObj.elType = macroElIndexTypeMap[perEdgeEl0.elIndex];
bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = perEdgeEl0.ithObject;
bound.neighObj.el = elIndexMap[perEdgeEl1.elIndex];
bound.neighObj.el = macroElIndexMap[perEdgeEl1.elIndex];
bound.neighObj.elIndex = perEdgeEl1.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = EDGE;
......@@ -1542,6 +1460,102 @@ namespace AMDiS {
b.neighObj.setReverseMode(b.rankObj, feSpace);
}
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
StdMpi<std::vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if ((rankIt->second)[j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
// === Do the same for the periodic boundaries. ===
if (periodicBoundary.boundary.size() > 0) {
stdMpi.clear();
InteriorBoundary sendBounds, recvBounds;
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
TEST_EXIT_DBG(rankIt->first != mpiRank)
("It is not allowed to have an interior boundary within a rank partition!\n");
if (rankIt->first < mpiRank)
sendBounds.boundary[rankIt->first] = rankIt->second;
else
recvBounds.boundary[rankIt->first] = rankIt->second;
}
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
if (rankIt->first <= mpiRank)
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvedBound) {
unsigned int k = j + 1;
for (; k < rankIt->second.size(); k++)
if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
AtomicBoundary tmpBound = (rankIt->second)[k];
(rankIt->second)[k] = (rankIt->second)[j];
(rankIt->second)[j] = tmpBound;
}
}
}
} // periodicBoundary.boundary.size() > 0
}
......@@ -1733,7 +1747,7 @@ namespace AMDiS {
}
void MeshDistributor::updateLocalGlobalNumbering()
void MeshDistributor::updateLocalGlobalNumbering(bool b)
{
FUNCNAME("MeshDistributor::updateLocalGlobalNumbering()");
......@@ -1767,6 +1781,10 @@ namespace AMDiS {
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
int nRankAllDofs = rankDofs.size();
if (b)
for (unsigned int i = 0; i < rankDofs.size(); i++)
*const_cast<DegreeOfFreedom*>(rankDofs[i]) = i;
// === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDofs to boundaryDofs. ===
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 7a605e6e
......@@ -28,11 +28,11 @@
#include <vector>
#include <mpi.h>
#include "parallel/InteriorBoundary.h"
#include "Global.h"
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "FiniteElemSpace.h"
#include "parallel/InteriorBoundary.h"
#include "Serializer.h"
#include "BoundaryManager.h"
#include "ElementObjectData.h"
......@@ -240,7 +240,11 @@ namespace AMDiS {
*/
void createInteriorBoundaryInfo();
void createBoundaryDataStructure();
void updateInteriorBoundaryInfo();
void createMeshElementData();
void createBoundaryData();
/// Removes all macro elements from the mesh that are not part of ranks partition.
void removeMacroElements();
......@@ -249,7 +253,7 @@ namespace AMDiS {
void createLocalGlobalNumbering();
/// Updates the local and global DOF numbering after the mesh has been changed.
void updateLocalGlobalNumbering();
void updateLocalGlobalNumbering(bool b);
/** \brief
* Creates to all dofs in rank's partition that are on a periodic boundary the
......@@ -498,6 +502,24 @@ namespace AMDiS {
/// Number of DOFs in the whole domain.
int nOverallDofs;
// Data structure to store all sub-objects of all elements of the macro mesh.
ElementObjects elObjects;
// Maps to each macro element index a pointer to the corresponding element.
std::map<int, Element*> macroElIndexMap;