Commit b227d4cf authored by Reuther, Sebastian's avatar Reuther, Sebastian

added more synch methods in MeshDistributor

parent e01dbece
......@@ -282,6 +282,147 @@ namespace AMDiS { namespace Parallel {
synchVector(vec);
}
/** \brief
* This function must be used if the values of a set of DOFVectors must be
* synchronized over all ranks. That means, that each rank sends the
* values of the DOFs, which are owned by the rank and lie on an interior
* boundary, to all other ranks also having these DOFs.
*
* The synchronization direction is from major to minor rank. This means
* that the value of the rank with the higher number sends its value
* to the rank with the lower number.
*/
// majorRank => minorRank
template<typename T>
void synchVector(std::vector<DOFVector<T>*> &vecs)
{
if (vecs.size() > 0)
{
// get FE space
const FiniteElemSpace *fe = vecs[0]->getFeSpace();
// TODO: check equal FE space
// The lines below do not work!
// for ( typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt)
// TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
int nLevels = levelData.getNumberOfLevels();
for (int level = nLevels - 1; level >= 0; level--)
{
StdMpi<std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
{
std::vector<std::vector<T> > dofs;
dofs.reserve(it.getDofs().size());
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values;
values.reserve(vecs.size());
for (typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin(); vecIt != vecs.end(); ++vecIt )
values.push_back((**vecIt)[it.getDofIndex()]);
dofs.push_back(values);
}
stdMpi.send(it.getRank(), dofs);
}
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
{
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
typename std::vector<DOFVector<T>*>::iterator vecIt = vecs.begin();
typename std::vector<T>::iterator valuesIt = values.begin();
for (; vecIt != vecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
(**vecIt)[it.getDofIndex()] = *valuesIt;
}
}
}
}
}
/** \brief
* Works quite similar to the function \ref synchVector with an operator/
* assigner for the values on the subdomain boundaries of the DOFVector vec.
* Additionally, the values stored in additionalVecs are synchronized in
* the same way (direction (minor to major or major to minor rank)) as the
* DOFs of the variable vec.
*/
template<typename T, typename Operator>
void synchVectorSameWay(DOFVector<T> &vec, std::vector<DOFVector<T>*> additionalVecs, Operator op)
{
// get FE space and check equal FE space
const FiniteElemSpace *fe = vec.getFeSpace();
// TODO: check equal FE space
// The lines below do not work!
// for ( typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt)
// TEST_EXIT( (*vecIt)->getFeSpace()->getBasisFcts()->getDegree() == fe->getBasisFcts()->getDegree() )("FE space of vectors to synch not equal!\n");
MultiLevelDofComm& dofComm = dofComms[fe->getMesh()];
int nLevels = levelData.getNumberOfLevels();
for (int level = nLevels - 1; level >= 0; level--)
{
StdMpi < std::vector<std::vector<T> > > stdMpi(levelData.getMpiComm(level));
for (DofComm::Iterator it(dofComm[level].getRecvDofs(), fe); !it.end(); it.nextRank())
{
std::vector<std::vector<T> > dofs;
dofs.reserve(it.getDofs().size());
for (; !it.endDofIter(); it.nextDof())
{
std::vector<T> values;
values.reserve(additionalVecs.size() + 1);
values.push_back( vec[it.getDofIndex()] );
for (typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin(); vecIt != additionalVecs.end(); ++vecIt )
values.push_back( (**vecIt)[it.getDofIndex()] );
dofs.push_back( values );
}
stdMpi.send(it.getRank(), dofs);
}
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
for (DofComm::Iterator it(dofComm[level].getSendDofs(), fe); !it.end(); it.nextRank())
{
for (; !it.endDofIter(); it.nextDof())
{
DegreeOfFreedom idx = it.getDofIndex();
std::vector<T> values = stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
T minorRankValue = vec[idx];
T majorRankValue = values[0];
op(vec[idx], values[0]);
T synchValue = vec[idx];
TEST_EXIT(additionalVecs.size() == values.size()-1)("The number of additional vectors and the received values do not match!\n");
typename std::vector<DOFVector<T>*>::iterator vecIt = additionalVecs.begin();
typename std::vector<T>::iterator valuesIt = values.begin();
++valuesIt; // exclude the first one since it belongs to the variable vec
for (; vecIt != additionalVecs.end(), valuesIt != values.end(); ++vecIt , ++valuesIt)
if (synchValue == majorRankValue)
(**vecIt)[idx] = *valuesIt;
}
}
}
// call simple sync method
std::vector<DOFVector<T>*> allDOFVectors;
allDOFVectors.push_back(&vec);
for ( int i = 0; i < additionalVecs.size(); ++i )
allDOFVectors.push_back(additionalVecs[i]);
synchVector(allDOFVectors);
}
/** \brief
* This function must be used if the values of a DOFVector must be
* synchronised over all ranks. That means, that each rank sends the
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment