Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit 5682e383 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Refactoring of StdMpi and documentation of this class.

parent b1bd6c98
......@@ -419,7 +419,7 @@ namespace AMDiS {
recvIt != recvDofs.end(); ++recvIt)
stdMpi.recv(recvIt->first, recvIt->second.size());
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvDofs.end(); ++recvIt)
......@@ -452,7 +452,7 @@ namespace AMDiS {
recvIt != recvDofs.end(); ++recvIt)
stdMpi.recv(recvIt->first, recvIt->second.size() * nComponents);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvDofs.end(); ++recvIt) {
......@@ -579,14 +579,8 @@ namespace AMDiS {
MeshStructure elCode;
elCode.init(it->rankObj);
MeshManipulation meshManipulation(feSpace);
meshChanged |=
!(meshManipulation.startFitElementToMeshCode(elCode,
it->neighObj.el,
it->neighObj.subObj,
it->neighObj.ithObj,
it->neighObj.elType,
it->neighObj.reverseMode));
MeshManipulation mm(feSpace);
meshChanged |= !(mm.fitElementToMeshCode(elCode, it->neighObj));
}
} else {
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
......@@ -663,8 +657,9 @@ namespace AMDiS {
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(allBound);
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication();
// === Compare received mesh structure codes. ===
......@@ -684,14 +679,8 @@ namespace AMDiS {
if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
MeshManipulation meshManipulation(feSpace);
meshChanged |=
meshManipulation.startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->rankObj.reverseMode);
MeshManipulation mm(feSpace);
meshChanged |= mm.fitElementToMeshCode(recvCodes[i], boundIt->rankObj);
}
}
}
......@@ -940,9 +929,9 @@ namespace AMDiS {
elCode.init(mesh, *elIt);
sendCodes[it->first].push_back(elCode);
for (unsigned int i = 0; i < testVec.size(); i++) {
for (unsigned int i = 0; i < interchangeVectors.size(); i++) {
vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec[i], valVec);
elCode.getMeshStructureValues(mesh, *elIt, interchangeVectors[i], valVec);
sendValues[it->first].push_back(valVec);
}
}
......@@ -950,13 +939,17 @@ namespace AMDiS {
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(partitioner->getRecvElements());
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication();
StdMpi<vector<vector<double> > > stdMpi2(mpiComm, true);
stdMpi2.send(sendValues);
stdMpi2.recv(partitioner->getRecvElements());
stdMpi2.startCommunication<double>(MPI_DOUBLE);
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it)
stdMpi2.recv(it->first);
stdMpi2.startCommunication();
// === Adapte the new macro elements due to the received mesh structure codes. ===
......@@ -974,8 +967,11 @@ namespace AMDiS {
elIt != it->second.end(); ++elIt) {
recvCodes[i].fitMeshToStructure(mesh, refineManager, false, *elIt);
for (unsigned int k = 0; k < testVec.size(); k++)
recvCodes[i].setMeshStructureValues(mesh, *elIt, testVec[k], recvValues[j++]);
for (unsigned int k = 0; k < interchangeVectors.size(); k++)
recvCodes[i].setMeshStructureValues(mesh,
*elIt,
interchangeVectors[k],
recvValues[j++]);
i++;
}
......@@ -1327,7 +1323,7 @@ namespace AMDiS {
StdMpi<vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === The information about all neighbouring boundaries has been received. So ===
......@@ -1386,7 +1382,7 @@ namespace AMDiS {
stdMpi.send(sendBounds.boundary);
stdMpi.recv(recvBounds.boundary);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin();
rankIt != periodicBoundary.boundary.end(); ++rankIt) {
......@@ -1548,7 +1544,7 @@ namespace AMDiS {
}
stdMpi.updateSendDataSize();
stdMpi.recv(recvDofs);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
for (RankToDofContainer::iterator recvIt = recvDofs.begin();
......@@ -1677,7 +1673,7 @@ namespace AMDiS {
}
stdMpi.updateSendDataSize();
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === The rank has received the dofs from the rank on the other side of ===
......@@ -1749,7 +1745,7 @@ namespace AMDiS {
stdMpi2.recv(it->first);
}
stdMpi2.startCommunication<int>(MPI_INT);
stdMpi2.startCommunication();
for (std::map<int, PeriodicDofMap>::iterator it = stdMpi2.getRecvData().begin();
it != stdMpi2.getRecvData().end(); ++it)
......
......@@ -88,6 +88,13 @@ namespace AMDiS {
void addProblemStat(ProblemVec *probVec);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector<double> *vec)
{
interchangeVectors.push_back(vec);
}
/** \brief
* This function checks if the mesh has changed on at least on rank. In this case,
* the interior boundaries are adapted on all ranks such that they fit together on
......@@ -392,9 +399,6 @@ namespace AMDiS {
}
}
public:
vector<DOFVector<double>* > testVec;
protected:
///
vector<ProblemVec*> probStat;
......@@ -528,6 +532,11 @@ namespace AMDiS {
*/
map<int, std::set<BoundaryType> > periodicDofAssociations;
/// This set of values must be interchanged between ranks when the mesh is
/// repartitioned.
vector<DOFVector<double>*> interchangeVectors;
/// Is the index of the first row of the linear system, which is owned by the rank.
int rstart;
......
......@@ -201,16 +201,12 @@ namespace AMDiS {
}
bool MeshManipulation::startFitElementToMeshCode(MeshStructure &code,
Element *el,
GeoIndex subObj,
int ithObj,
int elType,
bool reverseMode)
bool MeshManipulation::fitElementToMeshCode(MeshStructure &code,
BoundaryObject &boundEl)
{
FUNCNAME("MeshManipulation::startFitElementToMeshCode()");
FUNCNAME("MeshManipulation::fitElementToMeshCode()");
TEST_EXIT_DBG(el)("No element given!\n");
TEST_EXIT_DBG(boundEl.el)("No element given!\n");
// If the code is empty, the element does not matter and the function can
// return without chaning the mesh.
......@@ -220,8 +216,10 @@ namespace AMDiS {
// s0 and s1 are the number of the edge/face in both child of the element,
// which contain the edge/face the function has to traverse through. If the
// edge/face is not contained in one of the children, s0 or s1 is -1.
int s0 = el->getSubObjOfChild(0, subObj, ithObj, elType);
int s1 = el->getSubObjOfChild(1, subObj, ithObj, elType);
int s0 =
boundEl.el->getSubObjOfChild(0, boundEl.subObj, boundEl.ithObj, boundEl.elType);
int s1 =
boundEl.el->getSubObjOfChild(1, boundEl.subObj, boundEl.ithObj, boundEl.elType);
TEST_EXIT_DBG(s0 != -1 || s1 != -1)("This should not happen!\n");
......@@ -231,29 +229,32 @@ namespace AMDiS {
// Test for reverse mode, in which the left and right children of elements
// are flipped.
if (reverseMode)
if (boundEl.reverseMode)
traverseFlag |= Mesh::CALL_REVERSE_MODE;
// === If the edge/face is contained in both children. ===
Mesh *mesh = boundEl.el->getMesh();
if (s0 != -1 && s1 != -1) {
// Create traverse stack and traverse within the mesh until the element,
// which should be fitted to the mesh structure code, is reached.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
while (elInfo && elInfo->getElement() != el)
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
while (elInfo && elInfo->getElement() != boundEl.el)
elInfo = stack.traverseNext(elInfo);
TEST_EXIT_DBG(elInfo->getElement() == el)("This should not happen!\n");
TEST_EXIT_DBG(elInfo->getElement() == boundEl.el)("This should not happen!\n");
return fitElementToMeshCode(code, stack, subObj, ithObj, reverseMode);
return fitElementToMeshCode(code, stack, boundEl.subObj,
boundEl.ithObj, boundEl.reverseMode);
}
// === The edge/face is contained in only on of the both children. ===
if (el->isLeaf()) {
if (boundEl.el->isLeaf()) {
// If element is leaf and code contains only one leaf element, we are finished.
if (code.getNumElements() == 1 && code.isLeafElement())
......@@ -261,23 +262,23 @@ namespace AMDiS {
// Create traverse stack and traverse the mesh to the element.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
while (elInfo && elInfo->getElement() != el)
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
while (elInfo && elInfo->getElement() != boundEl.el)
elInfo = stack.traverseNext(elInfo);
TEST_EXIT_DBG(elInfo)("This should not happen!\n");
// Code is not leaf, therefore refine the element.
el->setMark(1);
refineManager->setMesh(el->getMesh());
boundEl.el->setMark(1);
refineManager->setMesh(boundEl.el->getMesh());
refineManager->setStack(&stack);
refineManager->refineFunction(elInfo);
meshChanged = true;
}
Element *child0 = el->getFirstChild();
Element *child1 = el->getSecondChild();
if (reverseMode) {
Element *child0 = boundEl.el->getFirstChild();
Element *child1 = boundEl.el->getSecondChild();
if (boundEl.reverseMode) {
swap(s0, s1);
swap(child0, child1);
}
......@@ -287,18 +288,20 @@ namespace AMDiS {
// === To the mesh structure code. ===
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(el->getMesh(), -1, traverseFlag);
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag);
if (s0 != -1) {
while (elInfo && elInfo->getElement() != child0)
elInfo = stack.traverseNext(elInfo);
meshChanged |= fitElementToMeshCode(code, stack, subObj, s0, reverseMode);
meshChanged |=
fitElementToMeshCode(code, stack, boundEl.subObj, s0, boundEl.reverseMode);
} else {
while (elInfo && elInfo->getElement() != child1)
elInfo = stack.traverseNext(elInfo);
meshChanged |= fitElementToMeshCode(code, stack, subObj, s1, reverseMode);
meshChanged |=
fitElementToMeshCode(code, stack, boundEl.subObj, s1, boundEl.reverseMode);
}
......
......@@ -45,25 +45,15 @@ namespace AMDiS {
* then \ref fitElementToMeshCode, that mainly refines the element such that
* it fits to the mesh structure code.
*
* \param[in] code The mesh structure code to which the edge/face of
* an element must be fitted.
* \param[in] el Pointer to the element.
* \param[in] subObj Defines whether an edge or a face must be fitted.
* \param[in] ithObj Defines which edge/face must be fitted.
* \param[in] elType Element type of the element (only important in 3D).
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
* \param[in] code The mesh structure code to which the edge/face of
* an element must be fitted.
* \param[in] boundEl Defines the element and its sub-object, i.e., edge or
* face that must be fitted.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool startFitElementToMeshCode(MeshStructure &code,
Element *el,
GeoIndex subObj,
int ithObj,
int elType,
bool reverseMode);
bool fitElementToMeshCode(MeshStructure &code, BoundaryObject &boundEl);
private:
/** \brief
......
......@@ -172,7 +172,7 @@ namespace AMDiS {
stdMpi.send(0, pdb.periodicDof);
}
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
int foundError = 0;
......@@ -280,7 +280,7 @@ namespace AMDiS {
StdMpi<CoordsVec> stdMpiCoords(pdb.mpiComm, true);
stdMpiCoords.send(sendCoords);
stdMpiCoords.recv(recvCoords);
stdMpiCoords.startCommunication<double>(MPI_DOUBLE);
stdMpiCoords.startCommunication();
for (RankToCoords::iterator it = sendCoords.begin();
......@@ -408,7 +408,7 @@ namespace AMDiS {
StdMpi<CoordsVec> stdMpi(pdb.mpiComm, true);
stdMpi.send(sendCoords);
stdMpi.recv(recvCoords);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
int dimOfWorld = Global::getGeo(WORLD);
......@@ -478,7 +478,7 @@ namespace AMDiS {
it != pdb.recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<double>(MPI_DOUBLE);
stdMpi.startCommunication();
int foundError = 0;
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
......@@ -565,7 +565,7 @@ namespace AMDiS {
stdMpi.send(sendNumber);
for (RankToDofContainer::iterator it = recvDofs.begin(); it != recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
int foundError = 0;
for (std::map<int, int>::iterator it = stdMpi.getRecvData().begin();
......
......@@ -403,7 +403,7 @@ namespace AMDiS {
for (std::set<int>::iterator it = recvFromRank.begin();
it != recvFromRank.end(); ++it)
stdMpi.recv(*it);
stdMpi.startCommunication<int>(MPI_INT);
stdMpi.startCommunication();
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
......
......@@ -14,66 +14,91 @@
namespace AMDiS {
int intSizeOf(int &data)
MPI_Datatype StdMpiHelper<int>::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<int> >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<double> >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::vector<std::vector<double> > >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::vector<MeshStructure> >::mpiDataType = MPI_UNSIGNED_LONG;
MPI_Datatype StdMpiHelper<std::vector<AtomicBoundary> >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<std::pair<int, int> > >::mpiDataType = MPI_INT;
MPI_Datatype StdMpiHelper<std::vector<WorldVector<double> > >::mpiDataType = MPI_DOUBLE;
MPI_Datatype StdMpiHelper<std::map<WorldVector<double>, int> >::mpiDataType = MPI_DOUBLE;
// T = int
int StdMpiHelper<int>::getBufferSize(int &data)
{
return 1;
}
int intSizeOf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data)
void StdMpiHelper<int>::createBuffer(int &data, int *buf)
{
return data.size() * 2;
buf[0] = data;
}
int intSizeOf(std::map<WorldVector<double>, int> &data)
void StdMpiHelper<int>::makeFromBuffer(int &data, int *buf, int bufSize)
{
return data.size() * (Global::getGeo(WORLD) + 1);
data = buf[0];
}
int intSizeOf(std::vector<MeshStructure> &data)
{
int s = 0;
for (unsigned int i = 0; i < data.size(); i++)
s += data[i].getCode().size() + 2;
return s;
}
int intSizeOf(std::vector<WorldVector<double> > &data)
// T = std::vector<int>
int StdMpiHelper<std::vector<int> >::getBufferSize(std::vector<int> &data)
{
return data.size() * Global::getGeo(WORLD);
return data.size();
}
int intSizeOf(std::vector<int> &data)
int StdMpiHelper<std::vector<int> >::getBufferSize(std::vector<const int*> &data)
{
return data.size();
}
int intSizeOf(std::vector<double> &data)
void StdMpiHelper<std::vector<int> >::createBuffer(std::vector<int> &data, int *buf)
{
return data.size();
for (unsigned int i = 0; i < data.size(); i++)
buf[i] = data[i];
}
int intSizeOf(std::vector<std::pair<int, int> > &data)
void StdMpiHelper<std::vector<int> >::makeFromBuffer(std::vector<int> &data, int *buf, int bufSize)
{
return data.size() * 2;
data.resize(bufSize);
for (int i = 0; i < bufSize; i++)
data[i] = buf[i];
}
int intSizeOf(std::vector<AtomicBoundary> &data)
// T = std::vector<double>
int StdMpiHelper<std::vector<double> >::getBufferSize(std::vector<double> &data)
{
return data.size() * 6;
return data.size();
}
int intSizeOf(std::vector<BoundaryObject> &data)
void StdMpiHelper<std::vector<double> >::createBuffer(std::vector<double> &data, double *buf)
{
return data.size() * 5;
for (unsigned int i = 0; i < data.size(); i++)
buf[i] = data[i];
}
int intSizeOf(std::vector<const DegreeOfFreedom*> &data)
void StdMpiHelper<std::vector<double> >::makeFromBuffer(std::vector<double> &data, double *buf, int bufSize)
{
return data.size();
data.resize(bufSize);
for (int i = 0; i < bufSize; i++)
data[i] = buf[i];
}
int intSizeOf(std::vector<std::vector<double> > &data)
// T = std::vector<std::vector<double> >
int StdMpiHelper<std::vector<std::vector<double> > >::getBufferSize(std::vector<std::vector<double> > &data)
{
int size = 1;
......@@ -83,86 +108,45 @@ namespace AMDiS {
return size;
}
int intSizeOf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data)
void StdMpiHelper<std::vector<std::vector<double> > >::createBuffer(std::vector<std::vector<double> > &data, double *buf)
{
int size = 1;
buf[0] = data.size();
int counter = 1;
for (std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
it != data.end(); ++it) {
size += 2 + it->second.size() * 2;
for (unsigned int i = 0; i < data.size(); i++) {
buf[counter++] = data[i].size();
for (unsigned int j = 0; j < data[i].size(); j++)
buf[counter++] = data[i][j];
}
return size;
}
void makeBuf(int &data, int *buf)
void StdMpiHelper<std::vector<std::vector<double> > >::makeFromBuffer(std::vector<std::vector<double> > &data, double *buf, int bufSize)
{
buf[0] = data;
}
void makeFromBuf(int &data, int *buf, int bufSize)
{
data = buf[0];
}
data.resize(static_cast<unsigned int>(buf[0]));
int counter = 1;
void makeBuf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data, int* buf)
{
int i = 0;
for (std::map<const DegreeOfFreedom*, DegreeOfFreedom>::iterator it = data.begin();
it != data.end(); ++it) {
buf[i++] = *(it->first);
buf[i++] = it->second;
for (unsigned int i = 0; i < data.size(); i++) {
data[i].resize(static_cast<unsigned int>(buf[counter++]));
for (unsigned int j = 0; j < data[i].size(); j++)
data[i][j] = buf[counter++];
}
}
void makeFromBuf(std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > &data,
int *buf, int bufSize)
{
if (bufSize == 0)
return;
TEST_EXIT(bufSize % 2 == 0)("This should not happen!\n");
data.clear();
data.reserve(bufSize / 2);
for (int i = 0; i < (bufSize / 2); i++)
data.push_back(std::make_pair(buf[i * 2], buf[i * 2 + 1]));
}
// T = std::vector<MeshStructure>
void makeBuf(std::map<WorldVector<double>, int> &data, double* buf)
int StdMpiHelper<std::vector<MeshStructure> >::getBufferSize(std::vector<MeshStructure> &data)
{
int i = 0;