Skip to content

GitLab

Commit 5682e383 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Refactoring of StdMpi and documentation of this class.

parent b1bd6c98
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/MeshDistributor.cc
View file @ 5682e383
......@@ -419,7 +419,7 @@ namespace AMDiS {
recvIt != recvDofs.end(); ++recvIt)
stdMpi.recv(recvIt->first, recvIt->second.size());
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvDofs.end(); ++recvIt)
......@@ -452,7 +452,7 @@ namespace AMDiS {
recvIt != recvDofs.end(); ++recvIt)
stdMpi.recv(recvIt->first, recvIt->second.size() * nComponents);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvDofs.end(); ++recvIt) {
......@@ -579,14 +579,8 @@ namespace AMDiS {
MeshStructure elCode;
elCode.init(it->rankObj);
MeshManipulation meshManipulation(feSpace);
meshChanged |=
!(meshManipulation.startFitElementToMeshCode(elCode,
it->neighObj.el,
it->neighObj.subObj,
it->neighObj.ithObj,
it->neighObj.elType,
it->neighObj.reverseMode));
MeshManipulation mm(feSpace);
meshChanged |= !(mm.fitElementToMeshCode(elCode, it->neighObj));
}
} else {
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
......@@ -663,8 +657,9 @@ namespace AMDiS {
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(allBound);
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication();
// === Compare received mesh structure codes. ===
......@@ -684,14 +679,8 @@ namespace AMDiS {
if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace);
meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->rankObj.reverseMode);
MeshManipulation mm(feSpace);
meshChanged |= mm.fitElementToMeshCode(recvCodes[i], boundIt->rankObj);
}
}
}
......@@ -940,9 +929,9 @@ namespace AMDiS {
elCode.init(mesh, *elIt);
sendCodes[it->first].push_back(elCode);
for (unsigned int i = 0; i < testVec.size(); i++) {
for (unsigned int i = 0; i < interchangeVectors.size(); i++) {
vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec[i], valVec);
elCode.getMeshStructureValues(mesh, *elIt, interchangeVectors[i], valVec);
sendValues[it->first].push_back(valVec);
}
}
......@@ -950,13 +939,17 @@ namespace AMDiS {
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(partitioner->getRecvElements());
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication();
StdMpi<vector<vector<double> > > stdMpi2(mpiComm, true);
stdMpi2.send(sendValues);
stdMpi2.recv(partitioner->getRecvElements());
stdMpi2.startCommunication<double>(MPI_DOUBLE);
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it)
stdMpi2.recv(it->first);
stdMpi2.startCommunication();
// === Adapte the new macro elements due to the received mesh structure codes. ===
......@@ -974,8 +967,11 @@ namespace AMDiS {
elIt != it->second.end(); ++elIt) {
recvCodes[i].fitMeshToStructure(mesh, refineManager, false, *elIt);
for (unsigned int k = 0; k < testVec.size(); k++)
recvCodes[i].setMeshStructureValues(mesh, *elIt, testVec[k], recvValues[j++]);
for (unsigned int k = 0; k < interchangeVectors.size(); k++)
recvCodes[i].setMeshStructureValues(mesh,
*elIt,
interchangeVectors[k],
recvValues[j++]);
i++;
}
......@@ -1327,7 +1323,7 @@ namespace AMDiS {
StdMpi<vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === The information about all neighbouring boundaries has been received. So ===
......@@ -1386,7 +1382,7 @@ namespace AMDiS {
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
......@@ -1548,7 +1544,7 @@ namespace AMDiS {
}
stdMpi.updateSendDataSize();
stdMpi.recv(recvDofs);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
......@@ -1677,7 +1673,7 @@ namespace AMDiS {
}
stdMpi.updateSendDataSize();
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === The rank has received the dofs from the rank on the other side of ===
......@@ -1749,7 +1745,7 @@ namespace AMDiS {
stdMpi2.recv(it->first);
}
stdMpi2.startCommunication<int>(MPI_INT);
stdMpi2.startCommunication();
for (std::map<int, PeriodicDofMap>::iterator it = stdMpi2.getRecvData().begin();
it != stdMpi2.getRecvData().end(); ++it)
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 5682e383
......@@ -88,6 +88,13 @@ namespace AMDiS {
void addProblemStat(ProblemVec *probVec);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector<double> *vec)
{
interchangeVectors.push_back(vec);
}
/** \brief
* This function checks if the mesh has changed on at least on rank. In this case,
* the interior boundaries are adapted on all ranks such that they fit together on
......@@ -392,9 +399,6 @@ namespace AMDiS {
}
}
public:
vector<DOFVector<double>* > testVec;
protected:
///
vector<ProblemVec*> probStat;
......@@ -528,6 +532,11 @@ namespace AMDiS {
*/
map<int, std::set<BoundaryType> > periodicDofAssociations;
/// This set of values must be interchanged between ranks when the mesh is
/// repartitioned.
vector<DOFVector<double>*> interchangeVectors;
/// Is the index of the first row of the linear system, which is owned by the rank.
int rstart;
......
AMDiS/src/parallel/MeshManipulation.cc
View file @ 5682e383
......@@ -201,16 +201,12 @@ namespace AMDiS {
}
bool MeshManipulation::startFitElementToMeshCode(MeshStructure &code,
Element *el,
GeoIndex subObj,
int ithObj,
int elType,
bool reverseMode)
bool MeshManipulation::fitElementToMeshCode(MeshStructure &code,
BoundaryObject &boundEl)
{
FUNCNAME("MeshManipulation::startFitElementToMeshCode()");
FUNCNAME("MeshManipulation::fitElementToMeshCode()");
TEST_EXIT_DBG(el)("No element given!\n");
TEST_EXIT_DBG(boundEl.el)("No element given!\n");
// If the code is empty, the element does not matter and the function can
// return without chaning the mesh.
......@@ -220,8 +216,10 @@ namespace AMDiS {
// s0 and s1 are the number of the edge/face in both child of the element,
// which contain the edge/face the function has to traverse through. If the
// edge/face is not contained in one of the children, s0 or s1 is -1.
int s0 = el->getSubObjOfChild(0, subObj, ithObj, elType);
int s1 = el->getSubObjOfChild(1, subObj, ithObj, elType);
int s0 =
boundEl.el->getSubObjOfChild(0, boundEl.subObj, boundEl.ithObj, boundEl.elType);
int s1 =
boundEl.el->getSubObjOfChild(1, boundEl.subObj, boundEl.ithObj, boundEl.elType);
TEST_EXIT_DBG(s0 != -1 || s1 != -1)("This should not happen!\n");
......@@ -231,29 +229,32 @@ namespace AMDiS {
// Test for reverse mode, in which the left and right children of elements
// are flipped.
if (reverseMode)
if (boundEl.reverseMode)
traverseFlag |= Mesh::CALL_REVERSE_MODE;
// === If the edge/face is contained in both children. ===
Mesh *mesh = boundEl.el->getMesh();
if (s0 != -1 && s1 != -1) {
// Create traverse stack and traverse within the mesh until the element,
// which should be fitted to the mesh structure code, is reached.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
while (elInfo && elInfo->getElement() != el)
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
while (elInfo && elInfo->getElement() != boundEl.el)
elInfo = stack.traverseNext(elInfo);
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
TEST_EXIT_DBG(elInfo->getElement() == boundEl.el)("This should not happen!\n");
return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return fitElementToMeshCode(code, stack, boundEl.subObj,
boundEl.ithObj, boundEl.reverseMode);
}
// === The edge/face is contained in only on of the both children. ===
if (el->isLeaf()) {
if (boundEl.el->isLeaf()) {
// If element is leaf and code contains only one leaf element, we are finished.
if (code.getNumElements() == 1 && code.isLeafElement())
......@@ -261,23 +262,23 @@ namespace AMDiS {
// Create traverse stack and traverse the mesh to the element.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
while (elInfo && elInfo->getElement() != el)
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
while (elInfo && elInfo->getElement() != boundEl.el)
elInfo = stack.traverseNext(elInfo);
TEST_EXIT_DBG(elInfo)("This should not happen!\n");
// Code is not leaf, therefore refine the element.
el->setMark(1);
refineManager->setMesh(el->getMesh());
boundEl.el->setMark(1);
refineManager->setMesh(boundEl.el->getMesh());
refineManager->setStack(&stack);
refineManager->refineFunction(elInfo);
meshChanged = true;
}
Element *child0 = el->getFirstChild();
Element *child1 = el->getSecondChild();
if (reverseMode) {
Element *child0 = boundEl.el->getFirstChild();
Element *child1 = boundEl.el->getSecondChild();
if (boundEl.reverseMode) {
swap(s0, s1);
swap(child0, child1);
}
......@@ -287,18 +288,20 @@ namespace AMDiS {
// === To the mesh structure code. ===
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
if (s0 != -1) {
while (elInfo && elInfo->getElement() != child0)
elInfo = stack.traverseNext(elInfo);
meshChanged |= fitElementToMeshCode(code, stack, subObj, s0, reverseMode);
meshChanged |=
fitElementToMeshCode(code, stack, boundEl.subObj, s0, boundEl.reverseMode);
} else {
while (elInfo && elInfo->getElement() != child1)
elInfo = stack.traverseNext(elInfo);
meshChanged |= fitElementToMeshCode(code, stack, subObj, s1, reverseMode);
meshChanged |=
fitElementToMeshCode(code, stack, boundEl.subObj, s1, boundEl.reverseMode);
}
......
AMDiS/src/parallel/MeshManipulation.h
View file @ 5682e383
......@@ -45,25 +45,15 @@ namespace AMDiS {
* then \ref fitElementToMeshCode, that mainly refines the element such that
* it fits to the mesh structure code.
*
* \param[in] code The mesh structure code to which the edge/face of
* an element must be fitted.
* \param[in] el Pointer to the element.
* \param[in] subObj Defines whether an edge or a face must be fitted.
* \param[in] ithObj Defines which edge/face must be fitted.
* \param[in] elType Element type of the element (only important in 3D).
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
* \param[in] code The mesh structure code to which the edge/face of
* an element must be fitted.
* \param[in] boundEl Defines the element and its sub-object, i.e., edge or
* face that must be fitted.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool startFitElementToMeshCode(MeshStructure &code,
Element *el,
GeoIndex subObj,
int ithObj,
int elType,
bool reverseMode);
bool fitElementToMeshCode(MeshStructure &code, BoundaryObject &boundEl);
private:
/** \brief
......
AMDiS/src/parallel/ParallelDebug.cc
View file @ 5682e383
......@@ -172,7 +172,7 @@ namespace AMDiS {
stdMpi.send(0, pdb.periodicDof);
}
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
int foundError = 0;
......@@ -280,7 +280,7 @@ namespace AMDiS {
StdMpi<CoordsVec> stdMpiCoords(pdb.mpiComm, true);
stdMpiCoords.send(sendCoords);
stdMpiCoords.recv(recvCoords);
stdMpiCoords.startCommunication<double>(MPI_DOUBLE);
stdMpiCoords.startCommunication();
for (RankToCoords::iterator it = sendCoords.begin();
......@@ -408,7 +408,7 @@ namespace AMDiS {
StdMpi<CoordsVec> stdMpi(pdb.mpiComm, true);
stdMpi.send(sendCoords);
stdMpi.recv(recvCoords);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
int dimOfWorld = Global::getGeo(WORLD);
......@@ -478,7 +478,7 @@ namespace AMDiS {
it != pdb.recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
int foundError = 0;
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
......@@ -565,7 +565,7 @@ namespace AMDiS {
stdMpi.send(sendNumber);
for (RankToDofContainer::iterator it = recvDofs.begin(); it != recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
int foundError = 0;
for (std::map<int, int>::iterator it = stdMpi.getRecvData().begin();
......
AMDiS/src/parallel/PetscSolver.cc
View file @ 5682e383
......@@ -403,7 +403,7 @@ namespace AMDiS {
for (std::set<int>::iterator it = recvFromRank.begin();
it != recvFromRank.end(); ++it)
stdMpi.recv(*it);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
......
AMDiS/src/parallel/StdMpi.cc
View file @ 5682e383
......@@ -14,66 +14,91 @@
namespace AMDiS {
int intSizeOf(int &data)
MPI_Datatype StdMpiHelper<int>::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<int> >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<double> >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::vector<std::vector<double> > >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::vector<MeshStructure> >::mpiDataType = MPI_UNSIGNED_LONG;
MPI_Datatype StdMpiHelper<std::vector<AtomicBoundary> >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<std::pair<int, int> > >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<WorldVector<double> > >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::map<WorldVector<double>, int> >::mpiDataType = MPI_DOUBLE;
// T = int
int StdMpiHelper<int>::getBufferSize(int &data)
{
return 1;
}
int intSizeOf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data)
void StdMpiHelper<int>::createBuffer(int &data, int *buf)
{
return data.size() * 2;
buf[0] = data;
}
int intSizeOf(std::map<WorldVector<double>, int> &data)
void StdMpiHelper<int>::makeFromBuffer(int &data, int *buf, int bufSize)
{
return data.size() * (Global::getGeo(WORLD) + 1);
data = buf[0];
}
int intSizeOf(std::vector<MeshStructure> &data)
{
int s = 0;
for (unsigned int i = 0; i < data.size(); i++)
s += data[i].getCode().size() + 2;
return s;
}
int intSizeOf(std::vector<WorldVector<double> > &data)
// T = std::vector<int>
int StdMpiHelper<std::vector<int> >::getBufferSize(std::vector<int> &data)
{
return data.size() * Global::getGeo(WORLD);
return data.size();
}
int intSizeOf(std::vector<int> &data)
int StdMpiHelper<std::vector<int> >::getBufferSize(std::vector<const int*> &data)
{
return data.size();
}
int intSizeOf(std::vector<double> &data)
void StdMpiHelper<std::vector<int> >::createBuffer(std::vector<int> &data, int *buf)
{
return data.size();
for (unsigned int i = 0; i < data.size(); i++)
buf[i] = data[i];
}
int intSizeOf(std::vector<std::pair<int, int> > &data)
void StdMpiHelper<std::vector<int> >::makeFromBuffer(std::vector<int> &data, int *buf, int bufSize)
{
return data.size() * 2;
data.resize(bufSize);
for (int i = 0; i < bufSize; i++)
data[i] = buf[i];
}
int intSizeOf(std::vector<AtomicBoundary> &data)
// T = std::vector<double>
int StdMpiHelper<std::vector<double> >::getBufferSize(std::vector<double> &data)
{
return data.size() * 6;
return data.size();
}
int intSizeOf(std::vector<BoundaryObject> &data)
void StdMpiHelper<std::vector<double> >::createBuffer(std::vector<double> &data, double *buf)
{
return data.size() * 5;
for (unsigned int i = 0; i < data.size(); i++)
buf[i] = data[i];
}
int intSizeOf(std::vector<const DegreeOfFreedom*> &data)
void StdMpiHelper<std::vector<double> >::makeFromBuffer(std::vector<double> &data, double *buf, int bufSize)
{
return data.size();
data.resize(bufSize);
for (int i = 0; i < bufSize; i++)
data[i] = buf[i];
}
int intSizeOf(std::vector<std::vector<double> > &data)
// T = std::vector<std::vector<double> >
int StdMpiHelper<std::vector<std::vector<double> > >::getBufferSize(std::vector<std::vector<double> > &data)
{
int size = 1;
......@@ -83,86 +108,45 @@ namespace AMDiS {
return size;
}
int intSizeOf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data)
void StdMpiHelper<std::vector<std::vector<double> > >::createBuffer(std::vector<std::vector<double> > &data, double *buf)
{
int size = 1;
buf[0] = data.size();
int counter = 1;
for (std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
it != data.end(); ++it) {
size += 2 + it->second.size() * 2;
for (unsigned int i = 0; i < data.size(); i++) {
buf[counter++] = data[i].size();
for (unsigned int j = 0; j < data[i].size(); j++)
buf[counter++] = data[i][j];
}
return size;
}
void makeBuf(int &data, int *buf)
void StdMpiHelper<std::vector<std::vector<double> > >::makeFromBuffer(std::vector<std::vector<double> > &data, double *buf, int bufSize)
{
buf[0] = data;
}
void makeFromBuf(int &data, int *buf, int bufSize)
{
data = buf[0];
}
data.resize(static_cast<unsigned int>(buf[0]));
int counter = 1;
void makeBuf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data, int* buf)
{
int i = 0;
for (std::map<const DegreeOfFreedom*, DegreeOfFreedom>::iterator it = data.begin();
it != data.end(); ++it) {
buf[i++] = *(it->first);
buf[i++] = it->second;
for (unsigned int i = 0; i < data.size(); i++) {
data[i].resize(static_cast<unsigned int>(buf[counter++]));
for (unsigned int j = 0; j < data[i].size(); j++)
data[i][j] = buf[counter++];
}
}
void makeFromBuf(std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > &data,
int *buf, int bufSize)
{
if (bufSize == 0)
return;
TEST_EXIT(bufSize % 2 == 0)("This should not happen!\n");
data.clear();
data.reserve(bufSize / 2);
for (int i = 0; i < (bufSize / 2); i++)
data.push_back(std::make_pair(buf[i * 2], buf[i * 2 + 1]));
}
// T = std::vector<MeshStructure>
void makeBuf(std::map<WorldVector<double>, int> &data, double* buf)
int StdMpiHelper<std::vector<MeshStructure> >::getBufferSize(std::vector<MeshStructure> &data)