Skip to content
Snippets Groups Projects
Commit b227d4cf authored by Reuther, Sebastian's avatar Reuther, Sebastian
Browse files

added more synch methods in MeshDistributor

parent e01dbece
No related branches found
No related tags found
No related merge requests found
......@@ -282,6 +282,147 @@ namespace AMDiS { namespace Parallel {
synchVector(vec);
}
/** \brief
* This function must be used if the values of a set of DOFVectors must be
* synchronized over all ranks. That means, that each rank sends the
* values of the DOFs, which are owned by the rank and lie on an interior
* boundary, to all other ranks also having these DOFs.
*
* The synchronization direction is from major to minor rank. This means
* that the value of the rank with the higher number sends its value
* to the rank with the lower number.
*/
// majorRank => minorRank
template<typename T>
void synchVector(std::vector<DOFVector<T>*> &vecs)
{
if (vecs.size() > 0)
{
// get FE space
const FiniteElemSpace *fe = vecs[0]->getFeSpace();
// TODO: check equal FE space
// The lines below do not work!
// for ( typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt)
// TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
int nLevels = levelData.getNumberOfLevels();
for (int level = nLevels - 1; level >= 0; level--)
{
StdMpi<std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
{
std::vector<std::vector<T> > dofs;
dofs.reserve(it.getDofs().size());
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values;
values.reserve(vecs.size());
for (typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt )
values.push_back((**vecIt)[it.getDofIndex()]);
dofs.push_back(values);
}
stdMpi.send(it.getRank(), dofs);
}
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
{
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin();
typename std::vector<T>::iterator valuesIt = values.begin();
for (; vecIt != vecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
(**vecIt)[it.getDofIndex()] = *valuesIt;
}
}
}
}
}
/** \brief
* Works quite similar to the function \ref synchVector with an operator/
* assigner for the values on the subdomain boundaries of the DOFVector vec.
* Additionally, the values stored in additionalVecs are synchronized in
* the same way (direction (minor to major or major to minor rank)) as the
* DOFs of the variable vec.
*/
template<typename T, typename Operator>
void synchVectorSameWay(DOFVector<T> &vec, std::vector<DOFVector<T>*> additionalVecs, Operator op)
{
// get FE space and check equal FE space
const FiniteElemSpace *fe = vec.getFeSpace();
// TODO: check equal FE space
// The lines below do not work!
// for ( typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt)
// TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
int nLevels = levelData.getNumberOfLevels();
for (int level = nLevels - 1; level >= 0; level--)
{
StdMpi < std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
{
std::vector<std::vector<T> > dofs;
dofs.reserve(it.getDofs().size());
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values;
values.reserve(additionalVecs.size() + 1);
values.push_back( vec[it.getDofIndex()] );
for (typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt )
values.push_back( (**vecIt)[it.getDofIndex()] );
dofs.push_back( values );
}
stdMpi.send(it.getRank(), dofs);
}
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
{
for (; !it.endDofIter(); it.nextDof())
{
DegreeOfFreedom idx = it.getDofIndex();
std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
T minorRankValue = vec[idx];
T majorRankValue = values[0];
op(vec[idx], values[0]);
T synchValue = vec[idx];
TEST_EXIT(additionalVecs.size() == values.size()-1)("The number of additional vectors and the received values do not match!\n");
typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin();
typename std::vector<T>::iterator valuesIt = values.begin();
++valuesIt; // exclude the first one since it belongs to the variable vec
for (; vecIt != additionalVecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
if (synchValue == majorRankValue)
(**vecIt)[idx] = *valuesIt;
}
}
}
// call simple sync method
std::vector<DOFVector<T>*> allDOFVectors;
allDOFVectors.push_back(&vec);
for ( int i = 0; i < additionalVecs.size(); ++i )
allDOFVectors.push_back(additionalVecs[i]);
synchVector(allDOFVectors);
}
/** \brief
* This function must be used if the values of a DOFVector must be
* synchronised over all ranks. That means, that each rank sends the
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment