Commit f5e012b5 authored by Thomas Witkowski's avatar Thomas Witkowski

Small changes and a new test for parallel interior boundaries.

parent b1357ae4
......@@ -142,6 +142,7 @@
#if HAVE_PARALLEL_DOMAIN_AMDIS
#include "parallel/InteriorBoundary.h"
#include "parallel/MpiHelper.h"
#include "parallel/ParallelDebug.h"
#include "parallel/StdMpi.h"
#if HAVE_PARALLEL_MTL4
......
......@@ -135,4 +135,11 @@ namespace AMDiS {
type == other.type);
}
bool AtomicBoundary::operator!=(const AtomicBoundary& other) const
{
return (rankObj != other.rankObj ||
neighObj != other.neighObj ||
type != other.type);
}
}
......@@ -110,6 +110,8 @@ namespace AMDiS {
bool operator==(const AtomicBoundary& other) const;
bool operator!=(const AtomicBoundary& other) const;
/// The rank's part of the boundary.
BoundaryObject rankObj;
......
......@@ -1498,4 +1498,16 @@ namespace AMDiS {
Element::deletedDOFs.clear();
}
void Mesh::getElementIndexMap(map<int, Element*> &elIndexMap)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
Element *el = elInfo->getElement();
elIndexMap[el->getIndex()] = el;
elInfo = stack.traverseNext(elInfo);
}
}
}
......@@ -629,6 +629,10 @@ namespace AMDiS {
}
#endif
/// Creates a map for all elements in mesh that maps from element indices
/// to the corresponding pointers.
void getElementIndexMap(map<int, Element*> &elIndexMap);
public:
///
static const Flag FILL_NOTHING;
......
......@@ -37,6 +37,8 @@
namespace AMDiS {
using namespace std;
template<typename ProblemType>
class Serializer : public FileWriterInterface
{
......@@ -64,7 +66,7 @@ namespace AMDiS {
}
Serializer(ProblemType *prob, std::string filename, int writeEveryIth)
Serializer(ProblemType *prob, string filename, int writeEveryIth)
: name(filename),
problem(prob),
tsModulo(writeEveryIth),
......@@ -119,10 +121,10 @@ namespace AMDiS {
}
#if HAVE_PARALLEL_DOMAIN_AMDIS
filename += ".p" + boost::lexical_cast<std::string>(MPI::COMM_WORLD.Get_rank());
filename += ".p" + boost::lexical_cast<string>(MPI::COMM_WORLD.Get_rank());
#endif
std::ofstream out(filename.c_str());
ofstream out(filename.c_str());
TEST_EXIT(out.is_open())("Cannot open serialization file!\n");
out.write(reinterpret_cast<const char*>(&amdisRevisionNumber), sizeof(int));
problem->serialize(out);
......@@ -134,7 +136,7 @@ namespace AMDiS {
protected:
/// Name of file to which the problem is serialized.
std::string name;
string name;
/// Pointer to the problem.
ProblemType *problem;
......@@ -159,40 +161,40 @@ namespace AMDiS {
namespace SerUtil {
template<typename T>
void serialize(std::ostream& out, T& data)
void serialize(ostream& out, T& data)
{
out.write(reinterpret_cast<const char*>(&data), sizeof(T));
}
template<typename T>
void deserialize(std::istream& in, T& data)
void deserialize(istream& in, T& data)
{
in.read(reinterpret_cast<char*>(&data), sizeof(T));
}
void serialize(std::ostream& out, DofEdge& data);
void serialize(ostream& out, DofEdge& data);
void deserialize(std::istream& in, DofEdge& data);
void deserialize(istream& in, DofEdge& data);
void serialize(std::ostream& out, DofFace& data);
void serialize(ostream& out, DofFace& data);
void deserialize(std::istream& in, DofFace& data);
void deserialize(istream& in, DofFace& data);
template<typename T, typename U>
void serialize(std::ostream& out, std::pair<T, U>& data)
void serialize(ostream& out, pair<T, U>& data)
{
serialize(out, data.first);
serialize(out, data.second);
}
template<typename T, typename U>
void deserialize(std::istream& in, std::pair<T, U>& data)
void deserialize(istream& in, pair<T, U>& data)
{
deserialize(in, data.first);
deserialize(in, data.second);
......@@ -201,11 +203,11 @@ namespace AMDiS {
template<typename T>
void serialize(std::ostream& out, std::vector<T>& data)
void serialize(ostream& out, vector<T>& data)
{
int vecSize = data.size();
serialize(out, vecSize);
for (typename std::vector<T>::iterator it = data.begin();
for (typename vector<T>::iterator it = data.begin();
it != data.end(); ++it) {
T v = *it;
serialize(out, v);
......@@ -213,7 +215,7 @@ namespace AMDiS {
}
template<typename T>
void deserialize(std::istream& in, std::vector<T>& data)
void deserialize(istream& in, vector<T>& data)
{
data.clear();
......@@ -231,7 +233,7 @@ namespace AMDiS {
template<typename T>
void serialize(std::ostream& out, std::set<T>& data)
void serialize(ostream& out, std::set<T>& data)
{
int setSize = data.size();
serialize(out, setSize);
......@@ -243,7 +245,7 @@ namespace AMDiS {
}
template<typename T>
void deserialize(std::istream& in, std::set<T>& data)
void deserialize(istream& in, std::set<T>& data)
{
data.clear();
......@@ -260,12 +262,12 @@ namespace AMDiS {
template<typename T1, typename T2>
void serialize(std::ostream& out, std::map<T1, T2>& data)
void serialize(ostream& out, map<T1, T2>& data)
{
int mapSize = data.size();
serialize(out, mapSize);
for (typename std::map<T1,T2>::iterator it = data.begin();
for (typename map<T1,T2>::iterator it = data.begin();
it != data.end(); ++it) {
T1 v1 = it->first;
T2 v2 = it->second;
......@@ -275,7 +277,7 @@ namespace AMDiS {
}
template<typename T1, typename T2>
void deserialize(std::istream& in, std::map<T1, T2>& data)
void deserialize(istream& in, map<T1, T2>& data)
{
data.clear();
......
......@@ -174,10 +174,10 @@ namespace AMDiS {
template<typename T>
void FileWriterTemplated<T>::writeFiles(AdaptInfo *adaptInfo,
bool force,
int level,
Flag flag,
bool (*writeElem)(ElInfo*))
bool force,
int level,
Flag flag,
bool (*writeElem)(ElInfo*))
{
FUNCNAME("FileWriterTemplated<T>::writeFiles()");
......@@ -186,19 +186,19 @@ namespace AMDiS {
// Containers, which store the data to be written;
std::vector<DataCollector<T>*> dataCollectors(solutionVecs.size());
if (writeElem) {
for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++)
dataCollectors[i] = new DataCollector<T>(feSpace, solutionVecs[i],
level, flag, writeElem);
level, flag, writeElem);
} else {
for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++)
dataCollectors[i] = new DataCollector<T>(feSpace, solutionVecs[i],
traverseLevel,
flag | traverseFlag,
writeElement);
traverseLevel,
flag | traverseFlag,
writeElement);
}
std::string fn = filename;
#if HAVE_PARALLEL_DOMAIN_AMDIS
......
......@@ -18,6 +18,30 @@ namespace AMDiS {
using namespace std;
void DofComm::init(int level,
MeshLevelData &ld,
vector<const FiniteElemSpace*> &fe)
{
FUNCNAME("DofComm::init()");
meshLevel = level;
levelData = &ld;
feSpaces = fe;
nLevel = levelData->getLevelNumber() - meshLevel;
TEST_EXIT_DBG(nLevel >= 1)("Should not happen!\n");
sendDofs.clear();
recvDofs.clear();
periodicDofs.clear();
sendDofs.resize(nLevel);
recvDofs.resize(nLevel);
periodicDofs.resize(nLevel);
}
void DofComm::create(InteriorBoundary &boundary)
{
createContainer(boundary.getOwn(), sendDofs);
......@@ -28,6 +52,8 @@ namespace AMDiS {
void DofComm::createContainer(RankToBoundMap &boundary,
LevelDataType &data)
{
FUNCNAME("DofComm::createContainer()");
// === Fill data. ===
for (unsigned int i = 0; i < feSpaces.size(); i++)
......
......@@ -38,7 +38,10 @@ namespace AMDiS {
DofComm()
: recvDofs(1),
sendDofs(1),
periodicDofs(0)
periodicDofs(0),
meshLevel(-1),
nLevel(0),
levelData(NULL)
{}
typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
......@@ -48,23 +51,9 @@ namespace AMDiS {
// meshLevel: map[rank -> map[feSpace -> DofContainer]]
typedef vector<DataType> LevelDataType;
void init(int n, vector<const FiniteElemSpace*> &fe)
{
FUNCNAME("DofComm::init()");
TEST_EXIT_DBG(n >= 1)("Should not happen!\n");
nLevel = n;
feSpaces = fe;
sendDofs.clear();
recvDofs.clear();
periodicDofs.clear();
sendDofs.resize(nLevel);
recvDofs.resize(nLevel);
periodicDofs.resize(nLevel);
}
void init(int level,
MeshLevelData &levelData,
vector<const FiniteElemSpace*> &fe);
void create(InteriorBoundary &boundary);
......@@ -119,8 +108,12 @@ namespace AMDiS {
/// considered here.
LevelDataType periodicDofs;
int meshLevel;
int nLevel;
MeshLevelData *levelData;
vector<const FiniteElemSpace*> feSpaces;
friend class Iterator;
......
......@@ -579,35 +579,23 @@ namespace AMDiS {
TEST_EXIT_DBG(macroElementRankMap)("Should not happen!\n");
int owner = -1;
switch (iterGeoPos) {
case VERTEX:
{
vector<ElementObjectData>& vertexData = vertexElements[vertexIter->first];
for (vector<ElementObjectData>::iterator it = vertexData.begin();
it != vertexData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case EDGE:
{
vector<ElementObjectData>& edgeData = edgeElements[edgeIter->first];
for (vector<ElementObjectData>::iterator it = edgeData.begin();
it != edgeData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
case FACE:
{
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
}
break;
default:
ERROR_EXIT("Should not happen!\n");
}
vector<ElementObjectData> *objData;
switch (iterGeoPos) {
case VERTEX:
objData = &(vertexElements[vertexIter->first]);
break;
case EDGE:
objData = &(edgeElements[edgeIter->first]);
break;
case FACE:
objData = &(faceElements[faceIter->first]);
break;
}
for (vector<ElementObjectData>::iterator it = objData->begin();
it != objData->end(); ++it)
owner = std::max(owner, (*macroElementRankMap)[it->elIndex]);
return owner;
}
......@@ -641,10 +629,12 @@ namespace AMDiS {
}
break;
case FACE:
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
{
vector<ElementObjectData>& faceData = faceElements[faceIter->first];
for (vector<ElementObjectData>::iterator it = faceData.begin();
it != faceData.end(); ++it)
ranksInLevel[0].insert((*macroElementRankMap)[it->elIndex]);
}
break;
default:
ERROR_EXIT("Should not happen!\n");
......
......@@ -19,6 +19,9 @@
namespace AMDiS {
using namespace std;
void InteriorBoundary::create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb)
{
......@@ -345,13 +348,19 @@ namespace AMDiS {
}
void InteriorBoundary::serialize(std::ostream &out)
void InteriorBoundary::serialize(ostream &out)
{
FUNCNAME("InteriorBoundary::serialize()");
serialize(out, own);
serialize(out, other);
serialize(out, periodic);
}
ERROR_EXIT("REWRITE TO MULTILEVEL STRUCTURE!\n");
#if 0
void InteriorBoundary::serialize(ostream &out,
RankToBoundMap& boundary)
{
FUNCNAME("InteriorBoundary::serialize()");
int mSize = boundary.size();
SerUtil::serialize(out, mSize);
for (RankToBoundMap::iterator it = boundary.begin();
......@@ -380,18 +389,26 @@ namespace AMDiS {
SerUtil::serialize(out, bound.type);
}
}
#endif
}
void InteriorBoundary::deserialize(std::istream &in,
std::map<int, Element*> &elIndexMap)
void InteriorBoundary::deserialize(istream &in, Mesh *mesh)
{
FUNCNAME("InteriorBoundary::deserialize()");
map<int, Element*> elIndexMap;
mesh->getElementIndexMap(elIndexMap);
deserialize(in, own, elIndexMap);
deserialize(in, other, elIndexMap);
deserialize(in, periodic, elIndexMap);
}
ERROR_EXIT("REWRITE TO MULTILEVEL STRUCTURE!\n");
#if 0
void InteriorBoundary::deserialize(istream &in,
RankToBoundMap& boundary,
map<int, Element*> &elIndexMap)
{
FUNCNAME("InteriorBoundary::deserialize()");
int mSize = 0;
SerUtil::deserialize(in, mSize);
for (int i = 0; i < mSize; i++) {
......@@ -431,14 +448,13 @@ namespace AMDiS {
// For the case of periodic interior boundaries, a rank may have an
// boundary with itself. In this case, also the pointer to the neighbour
// object must be set correctly.
// object must be set correctly.
if (elIndexMap.count(bound.neighObj.elIndex))
bound.neighObj.el = elIndexMap[bound.neighObj.elIndex];
else
bound.neighObj.el = NULL;
}
}
#endif
}
......@@ -466,7 +482,7 @@ namespace AMDiS {
}
void InteriorBoundary::serializeExcludeList(std::ostream &out,
void InteriorBoundary::serializeExcludeList(ostream &out,
ExcludeList &list)
{
int size = list.size();
......@@ -478,7 +494,7 @@ namespace AMDiS {
}
void InteriorBoundary::deserializeExcludeList(std::istream &in,
void InteriorBoundary::deserializeExcludeList(istream &in,
ExcludeList &list)
{
int size = 0;
......@@ -492,7 +508,7 @@ namespace AMDiS {
SerUtil::deserialize(in, a);
SerUtil::deserialize(in, b);
list.push_back(std::make_pair(a, b));
list.push_back(make_pair(a, b));
}
}
......
......@@ -44,12 +44,6 @@ namespace AMDiS {
void create(MPI::Intracomm &mpiComm,
ElementObjectDatabase &elObjDb);
/// Writes this object to a file.
void serialize(ostream &out);
/// Reads the state of an interior boundary from a file.
void deserialize(istream &in, map<int, Element*> &elIndexMap);
RankToBoundMap& getOwn()
{
return own;
......@@ -70,6 +64,12 @@ namespace AMDiS {
return static_cast<bool>(periodic.size());
}
/// Writes this object to a file.
void serialize(ostream &out);
/// Reads the state of an interior boundary from a file.
void deserialize(istream &in, Mesh *mesh);
private:
AtomicBoundary& getNewOwn(int rank);
......@@ -77,6 +77,12 @@ namespace AMDiS {
AtomicBoundary& getNewPeriodic(int rank);
void serialize(ostream &out, RankToBoundMap& boundary);
void deserialize(istream &in,
RankToBoundMap& boundary,
map<int, Element*> &elIndexMap);
void serializeExcludeList(ostream &out, ExcludeList &list);
void deserializeExcludeList(istream &in, ExcludeList &list);
......
......@@ -228,12 +228,8 @@ namespace AMDiS {
createMeshLevelStructure();
// Create interior boundary information.
createInteriorBoundaryInfo();
createInteriorBoundary(true);
#if (DEBUG != 0)
ParallelDebug::printBoundaryInfo(*this);
#endif
// === Remove neighbourhood relations due to periodic bounday conditions. ===
for (deque<MacroElement*>::iterator it = mesh->firstMacroElement();
......@@ -1438,7 +1434,7 @@ namespace AMDiS {
mesh->dofCompress();
partitioner->createPartitionMap(partitionMap);
updateInteriorBoundaryInfo();
createInteriorBoundary(false);
updateLocalGlobalNumbering();
......@@ -1509,19 +1505,12 @@ namespace AMDiS {
}
void MeshDistributor::createInteriorBoundaryInfo()
void MeshDistributor::createInteriorBoundary(bool firstCall)
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
elObjDb.create(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
}
FUNCNAME("MeshDistributor::createInteriorBoundary()");
void MeshDistributor::updateInteriorBoundaryInfo()
{
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
if (firstCall)
elObjDb.create(partitionMap, levelData);
elObjDb.updateRankData();
intBoundary.create(mpiComm, elObjDb);
......@@ -1536,17 +1525,18 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createBoundaryDofs()");
dofComm.init(levelData.getLevelNumber(), feSpaces);
dofComm.create(intBoundary);
createBoundaryDofInfo();
}
// === Create DOF communicator. ===
dofComm.init(0, levelData, feSpaces);
dofComm.create(intBoundary);
void MeshDistributor::createBoundaryDofInfo()
{
FUNCNAME("MeshDistributor::createBoundaryDofInfo()");
if (levelData.getLevelNumber() > 1) {
dofCommSd.init(1, levelData, feSpaces);
dofCommSd.create(intBoundary);
}
// === If requested, create more information on communication DOFs. ===