Commit e0139072 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Changed sendDofs and recvDofs data structure in parallel code.

parent 1f381921
...@@ -220,7 +220,7 @@ if(ENABLE_PARALLEL_DOMAIN) ...@@ -220,7 +220,7 @@ if(ENABLE_PARALLEL_DOMAIN)
list(APPEND COMPILEFLAGS "-DHAVE_PARALLEL_DOMAIN_AMDIS=1") list(APPEND COMPILEFLAGS "-DHAVE_PARALLEL_DOMAIN_AMDIS=1")
SET(PARALLEL_DOMAIN_AMDIS_SRC SET(PARALLEL_DOMAIN_AMDIS_SRC
${SOURCE_DIR}/parallel/ParMetisPartitioner.cc ${SOURCE_DIR}/parallel/DofComm.cc
${SOURCE_DIR}/parallel/CheckerPartitioner.cc ${SOURCE_DIR}/parallel/CheckerPartitioner.cc
${SOURCE_DIR}/parallel/ElementObjectData.cc ${SOURCE_DIR}/parallel/ElementObjectData.cc
${SOURCE_DIR}/parallel/MeshDistributor.cc ${SOURCE_DIR}/parallel/MeshDistributor.cc
......
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.
#include "DofComm.h"
namespace AMDiS {
using namespace std;
void DofComm::removeEmpty()
{
for (DataIter dit = data.begin(); dit != data.end(); ++dit) {
FeMapIter it = dit->second.begin();
while (it != dit->second.end()) {
if (it->second.size() == 0) {
const FiniteElemSpace *fe = it->first;
++it;
dit->second.erase(fe);
} else
++it;
}
}
}
void DofComm::Iterator::setNextFeMap()
{
if (dataIter != dofComm.data.end()) {
feMapIter = dataIter->second.begin();
if (traverseFeSpace != NULL) {
TEST_EXIT_DBG(dataIter->second.count(traverseFeSpace))
("Should not happen!\n");
while (feMapIter->first != traverseFeSpace &&
feMapIter != dataIter->second.end())
++feMapIter;
TEST_EXIT_DBG(feMapIter != dataIter->second.end() &&
feMapIter->first == traverseFeSpace)
("Should not happen!\n");
}
if (feMapIter != dataIter->second.end())
dofIter = feMapIter->second.begin();
dofCounter = 0;
}
}
}
// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// == http://www.amdis-fem.org ==
// == ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.
/** \file DofComm.h */
#ifndef AMDIS_DOF_COMM_H
#define AMDIS_DOF_COMM_H
#include <map>
#include "FiniteElemSpace.h"
#include "Global.h"
namespace AMDiS {
using namespace std;
class DofComm
{
public:
DofComm() {}
typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
typedef FeMapType::iterator FeMapIter;
typedef map<int, FeMapType> DataType;
typedef DataType::iterator DataIter;
DofContainer& getDofCont(int rank, const FiniteElemSpace *feSpace)
{
return data[rank][feSpace];
}
void removeEmpty();
void clear()
{
data.clear();
}
DataType& getData()
{
return data;
}
protected:
DataType data;
friend class Iterator;
public:
class Iterator
{
public:
Iterator(DofComm &dc,
const FiniteElemSpace *fe = NULL)
: dofComm(dc),
dofCounter(-1),
traverseFeSpace(fe)
{
FUNCNAME("DofComm::Iterator::Iterator()");
dataIter = dofComm.data.begin();
setNextFeMap();
}
inline bool end()
{
return (dataIter == dofComm.data.end());
}
inline void nextRank()
{
++dataIter;
setNextFeMap();
}
inline void nextFeSpace()
{
++feMapIter;
}
inline void beginDofIter(const FiniteElemSpace *fe = NULL)
{
FUNCNAME("DofComm::Iterator::beginDofIter()");
if (fe != NULL) {
feMapIter = dataIter->second.begin();
while (feMapIter->first != fe &&
feMapIter != dataIter->second.end())
++feMapIter;
}
TEST_EXIT_DBG(feMapIter != dataIter->second.end())
("Should not happen!\n");
dofIter = feMapIter->second.begin();
dofCounter = 0;
}
inline bool endDofIter()
{
return (dofIter == feMapIter->second.end());
}
inline void nextDof()
{
++dofIter;
++dofCounter;
}
inline int getRank()
{
return dataIter->first;
}
inline const FiniteElemSpace* getFeSpace()
{
return feMapIter->first;
}
inline DofContainer& getDofs()
{
return feMapIter->second;
}
inline const DegreeOfFreedom* getDof()
{
return *dofIter;
}
inline DegreeOfFreedom getDofIndex()
{
return **dofIter;
}
inline int getDofCounter()
{
return dofCounter;
}
protected:
void setNextFeMap();
protected:
DofComm &dofComm;
DofComm::DataIter dataIter;
DofComm::FeMapIter feMapIter;
DofContainer::iterator dofIter;
int dofCounter;
const FiniteElemSpace *traverseFeSpace;
};
};
}
#endif // AMDIS_DOF_COMM_H
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include "parallel/SimplePartitioner.h" #include "parallel/SimplePartitioner.h"
#include "parallel/CheckerPartitioner.h" #include "parallel/CheckerPartitioner.h"
#include "parallel/MpiHelper.h" #include "parallel/MpiHelper.h"
#include "parallel/DofComm.h"
#include "io/ElementFileWriter.h" #include "io/ElementFileWriter.h"
#include "io/MacroInfo.h" #include "io/MacroInfo.h"
#include "io/VtkWriter.h" #include "io/VtkWriter.h"
...@@ -522,44 +523,34 @@ namespace AMDiS { ...@@ -522,44 +523,34 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::synchVector()"); FUNCNAME("MeshDistributor::synchVector()");
int nComponents = vec.getSize();
StdMpi<vector<double> > stdMpi(mpiComm); StdMpi<vector<double> > stdMpi(mpiComm);
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; for (DofComm::Iterator it(sendDofs); !it.end(); it.nextRank()) {
for (it_type sendIt = sendDofs.begin(); sendIt != sendDofs.end(); ++sendIt) {
vector<double> dofs; vector<double> dofs;
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < vec.getSize(); i++) {
TEST_EXIT_DBG(sendIt->second.count(vec.getFeSpace(i))) DOFVector<double> &dofVec = *(vec.getDOFVector(i));
("Should not happen!\n");
DofContainer &feDofs = sendIt->second[vec.getFeSpace(i)];
DOFVector<double>& dofVec = *(vec.getDOFVector(i));
int nFeDofs = feDofs.size(); for (it.beginDofIter(vec.getFeSpace(i)); !it.endDofIter(); it.nextDof())
for (int j = 0; j < nFeDofs; j++) dofs.push_back(dofVec[it.getDofIndex()]);
dofs.push_back(dofVec[*(feDofs[j])]);
} }
stdMpi.send(sendIt->first, dofs); stdMpi.send(it.getRank(), dofs);
} }
for (it_type recvIt = recvDofs.begin(); recvIt != recvDofs.end(); ++recvIt) for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank())
stdMpi.recv(recvIt->first); stdMpi.recv(it.getRank());
stdMpi.startCommunication(); stdMpi.startCommunication();
for (it_type recvIt = recvDofs.begin(); recvIt != recvDofs.end(); ++recvIt) { for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (int i = 0; i < nComponents; i++) {
DofContainer &feDofs = recvIt->second[vec.getFeSpace(i)];
DOFVector<double>& dofVec = *(vec.getDOFVector(i));
int nFeDofs = feDofs.size();
for (int j = 0; j < nFeDofs; j++) for (int i = 0; i < vec.getSize(); i++) {
dofVec[*(feDofs[j])] = stdMpi.getRecvData(recvIt->first)[counter++]; DOFVector<double> &dofVec = *(vec.getDOFVector(i));
for (it.beginDofIter(vec.getFeSpace(i)); !it.endDofIter(); it.nextDof())
dofVec[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[counter++];
} }
} }
} }
...@@ -685,14 +676,10 @@ namespace AMDiS { ...@@ -685,14 +676,10 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::getAllBoundaryDofs()"); FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
DofContainerSet dofSet; DofContainerSet dofSet;
for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; dofSet.insert(it.getDofs().begin(), it.getDofs().end());
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (it_type it = sendDofs.begin(); it != sendDofs.end(); ++it) dofSet.insert(it.getDofs().begin(), it.getDofs().end());
dofSet.insert(it->second[feSpace].begin(), it->second[feSpace].end());
for (it_type it = recvDofs.begin(); it != recvDofs.end(); ++it)
dofSet.insert(it->second[feSpace].begin(), it->second[feSpace].end());
dofs.clear(); dofs.clear();
dofs.insert(dofs.begin(), dofSet.begin(), dofSet.end()); dofs.insert(dofs.begin(), dofSet.begin(), dofSet.end());
...@@ -993,17 +980,13 @@ namespace AMDiS { ...@@ -993,17 +980,13 @@ namespace AMDiS {
boundaryDofs.clear(); boundaryDofs.clear();
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
for (it_type it = sendDofs.begin(); it != sendDofs.end(); ++it) boundaryDofs.insert(it.getDofIndex());
for (DofContainer::iterator dofIt = it->second[feSpace].begin();
dofIt != it->second[feSpace].begin(); ++dofIt)
boundaryDofs.insert(**dofIt);
for (it_type it = recvDofs.begin(); it != recvDofs.end(); ++it) for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (DofContainer::iterator dofIt = it->second[feSpace].begin(); for (; !it.endDofIter(); it.nextDof())
dofIt != it->second[feSpace].begin(); ++dofIt) boundaryDofs.insert(it.getDofIndex());
boundaryDofs.insert(**dofIt);
} }
...@@ -1820,7 +1803,7 @@ namespace AMDiS { ...@@ -1820,7 +1803,7 @@ namespace AMDiS {
DofContainer dofs; DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
DofContainer& tmp = sendDofs[it.getRank()][feSpace]; DofContainer& tmp = sendDofs.getDofCont(it.getRank(), feSpace);
tmp.insert(tmp.end(), dofs.begin(), dofs.end()); tmp.insert(tmp.end(), dofs.begin(), dofs.end());
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS)) if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
...@@ -1836,7 +1819,7 @@ namespace AMDiS { ...@@ -1836,7 +1819,7 @@ namespace AMDiS {
DofContainer dofs; DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
DofContainer& tmp = recvDofs[it.getRank()][feSpace]; DofContainer& tmp = recvDofs.getDofCont(it.getRank(), feSpace);
tmp.insert(tmp.end(), dofs.begin(), dofs.end()); tmp.insert(tmp.end(), dofs.begin(), dofs.end());
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS)) if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
...@@ -1847,41 +1830,18 @@ namespace AMDiS { ...@@ -1847,41 +1830,18 @@ namespace AMDiS {
} else { } else {
for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it) for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it)
it->rankObj.el->getAllDofs(feSpace, it->rankObj, it->rankObj.el->getAllDofs(feSpace, it->rankObj,
sendDofs[it.getRank()][feSpace]); sendDofs.getDofCont(it.getRank(), feSpace));
for (InteriorBoundary::iterator it(otherIntBoundary); !it.end(); ++it) for (InteriorBoundary::iterator it(otherIntBoundary); !it.end(); ++it)
it->rankObj.el->getAllDofs(feSpace, it->rankObj, it->rankObj.el->getAllDofs(feSpace, it->rankObj,
recvDofs[it.getRank()][feSpace]); recvDofs.getDofCont(it.getRank(), feSpace));
} }
// === Delete all empty DOF send and recv positions === // === Delete all empty DOF send and recv positions ===
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type; sendDofs.removeEmpty();
for (it_type sendIt = sendDofs.begin(); sendIt != sendDofs.end(); ++sendIt) { recvDofs.removeEmpty();
map<const FiniteElemSpace*, DofContainer>::iterator it =
sendIt->second.begin();
while (it != sendIt->second.end()) {
if (it->second.size() == 0) {
const FiniteElemSpace* fe = it->first;
++it;
sendIt->second.erase(fe);
} else
++it;
}
}
for (it_type recvIt = recvDofs.begin(); recvIt != recvDofs.end(); ++recvIt) {
map<const FiniteElemSpace*, DofContainer>::iterator it =
recvIt->second.begin();
while (it != recvIt->second.end()) {
if (it->second.size() == 0) {
const FiniteElemSpace* fe = it->first;
++it;
recvIt->second.erase(fe);
} else
++it;
}
}
} }
...@@ -1964,19 +1924,15 @@ namespace AMDiS { ...@@ -1964,19 +1924,15 @@ namespace AMDiS {
// All DOFs that must be received are DOFs not owned by rank and have // All DOFs that must be received are DOFs not owned by rank and have
// therefore to be removed from the set 'rankDofs'. // therefore to be removed from the set 'rankDofs'.
for (map<int, map<const FiniteElemSpace*, DofContainer> >::iterator recvIt = recvDofs.begin(); for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank()) {
recvIt != recvDofs.end(); ++recvIt) { for (; !it.endDofIter(); it.nextDof()) {
DofContainer &rDofs = recvIt->second[feSpace];
for (DofContainer::iterator dofIt = rDofs.begin();
dofIt != rDofs.end(); ++dofIt) {
DofContainer::iterator eraseIt = DofContainer::iterator eraseIt =
find(rankDofs.begin(), rankDofs.end(), *dofIt); find(rankDofs.begin(), rankDofs.end(), it.getDof());
if (eraseIt != rankDofs.end()) if (eraseIt != rankDofs.end())
rankDofs.erase(eraseIt); rankDofs.erase(eraseIt);
} }
} }
// Get displacment for global rank DOF ordering and global DOF number. // Get displacment for global rank DOF ordering and global DOF number.
dofFeData[feSpace].nRankDofs = rankDofs.size(); dofFeData[feSpace].nRankDofs = rankDofs.size();
mpi::getDofNumbering(mpiComm, mpi::getDofNumbering(mpiComm,
...@@ -1994,26 +1950,25 @@ namespace AMDiS { ...@@ -1994,26 +1950,25 @@ namespace AMDiS {
// === Send and receive new DOF indices. === // === Send and receive new DOF indices. ===
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this, sendDofs, recvDofs); ParallelDebug::testDofContainerCommunication(*this,
sendDofs.getData(),
recvDofs.getData());
#endif #endif
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm); StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (map<int, map<const FiniteElemSpace*, DofContainer> >::iterator sendIt = sendDofs.begin(); for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank()) {
sendIt != sendDofs.end(); ++sendIt) { stdMpi.getSendData(it.getRank()).resize(0);
DofContainer &sDofs = sendIt->second[feSpace]; stdMpi.getSendData(it.getRank()).reserve(it.getDofs().size());
stdMpi.getSendData(sendIt->first).resize(0); for (; !it.endDofIter(); it.nextDof())
stdMpi.getSendData(sendIt->first).reserve(sDofs.size()); stdMpi.getSendData(it.getRank()).
for (DofContainer::iterator dofIt = sDofs.begin(); push_back(rankDofsNewGlobalIndex[it.getDof()]);
dofIt != sDofs.end(); ++dofIt)
stdMpi.getSendData(sendIt->first).push_back(rankDofsNewGlobalIndex[*dofIt]);
} }
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
for (map<int, map<const FiniteElemSpace*, DofContainer> >::iterator recvIt = recvDofs.begin(); for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank())
recvIt != recvDofs.end(); ++recvIt) stdMpi.recv(it.getRank());
stdMpi.recv(recvIt->first);
stdMpi.startCommunication(); stdMpi.startCommunication();
...@@ -2024,15 +1979,11 @@ namespace AMDiS { ...@@ -2024,15 +1979,11 @@ namespace AMDiS {
for (int i = 0; i < nRankAllDofs; i++) for (int i = 0; i < nRankAllDofs; i++)
dofFeData[feSpace].isRankDof[i] = true; dofFeData[feSpace].isRankDof[i] = true;
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank()) {
for (map<int, map<const FiniteElemSpace*, DofContainer> >::iterator recvIt = recvDofs.begin(); for (; !it.endDofIter(); it.nextDof()) {
recvIt != recvDofs.end(); ++recvIt) { rankDofsNewGlobalIndex[it.getDof()] =
DofContainer &rDofs = recvIt->second[feSpace]; stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
int i = 0; dofFeData[feSpace].isRankDof[it.getDofIndex()] = false;
for (DofContainer::iterator dofIt = rDofs.begin();
dofIt != rDofs.end(); ++dofIt) {
rankDofsNewGlobalIndex[*dofIt] = stdMpi.getRecvData(recvIt->first)[i++];
dofFeData[feSpace].isRankDof[**dofIt] = false;
} }
} }
...@@ -2296,8 +2247,8 @@ namespace AMDiS { ...@@ -2296,8 +2247,8 @@ namespace AMDiS {
otherIntBoundary.serialize(out); otherIntBoundary.serialize(out);
periodicBoundary.serialize(out); periodicBoundary.serialize(out);
serialize(out, sendDofs); serialize(out, sendDofs.getData());
serialize(out, recvDofs); serialize(out, recvDofs.getData());
// === Serialieze FE space dependent data === // === Serialieze FE space dependent data ===
...@@ -2368,8 +2319,8 @@ namespace AMDiS { ...@@ -2368,8 +2319,8 @@ namespace AMDiS {
otherIntBoundary.deserialize(in, elIndexMap); otherIntBoundary.deserialize(in, elIndexMap);
periodicBoundary.deserialize(in, elIndexMap); periodicBoundary.deserialize(in, elIndexMap);