Commit 78a18179 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on multi level methods, NEVER, NEVER, NEVER USE THIS RELEASE OTHERWISE YOU WILL BURN IN HELL.

parent 6758eb3d
...@@ -172,13 +172,13 @@ namespace AMDiS { ...@@ -172,13 +172,13 @@ namespace AMDiS {
mesh->getDofIndexCoords(coords); mesh->getDofIndexCoords(coords);
InteriorBoundary &intBoundary = InteriorBoundary &intBoundary =
MeshDistributor::globalMeshDistributor->getIntBoundary(); MeshDistributor::globalMeshDistributor->getIntBoundary(0);
ElInfo *elInfo = mesh->createNewElInfo(); ElInfo *elInfo = mesh->createNewElInfo();
elInfo->setFillFlag(Mesh::FILL_COORDS); elInfo->setFillFlag(Mesh::FILL_COORDS);
StdMpi<vector<double> > stdMpiDet(MeshDistributor::globalMeshDistributor->getMpiComm()); StdMpi<vector<double> > stdMpiDet(MeshDistributor::globalMeshDistributor->getMpiComm(0));
StdMpi<vector<vector<WorldVector<double> > > > stdMpiGrdUh(MeshDistributor::globalMeshDistributor->getMpiComm()); StdMpi<vector<vector<WorldVector<double> > > > stdMpiGrdUh(MeshDistributor::globalMeshDistributor->getMpiComm(0));
RankToBoundMap allBounds = intBoundary.getOther(); RankToBoundMap allBounds = intBoundary.getOther();
allBounds.insert(intBoundary.getOwn().begin(), intBoundary.getOwn().end()); allBounds.insert(intBoundary.getOwn().begin(), intBoundary.getOwn().end());
......
...@@ -16,32 +16,22 @@ ...@@ -16,32 +16,22 @@
#include "FiniteElemSpace.h" #include "FiniteElemSpace.h"
#include "Debug.h" #include "Debug.h"
#include "ElementDofIterator.h" #include "ElementDofIterator.h"
#include "DOFVector.h"
namespace AMDiS { namespace AMDiS {
using namespace std; using namespace std;
void DofComm::init(int level, void DofComm::init(vector<const FiniteElemSpace*> &fe)
MeshLevelData &ld,
vector<const FiniteElemSpace*> &fe)
{ {
FUNCNAME("DofComm::init()"); FUNCNAME("DofComm::init()");
meshLevel = level;
levelData = &ld;
feSpaces = fe; feSpaces = fe;
nLevel = levelData->getLevelNumber() - meshLevel;
TEST_EXIT_DBG(nLevel >= 1)("Should not happen!\n");
sendDofs.clear(); sendDofs.clear();
recvDofs.clear(); recvDofs.clear();
periodicDofs.clear(); periodicDofs.clear();
sendDofs.resize(nLevel);
recvDofs.resize(nLevel);
periodicDofs.resize(nLevel);
} }
...@@ -55,25 +45,22 @@ namespace AMDiS { ...@@ -55,25 +45,22 @@ namespace AMDiS {
void DofComm::createContainer(RankToBoundMap &boundary, void DofComm::createContainer(RankToBoundMap &boundary,
LevelDataType &data) DataType &data)
{ {
FUNCNAME("DofComm::createContainer()"); FUNCNAME("DofComm::createContainer()");
// === Fill data. === // === Fill data. ===
for (unsigned int i = 0; i < feSpaces.size(); i++) for (unsigned int i = 0; i < feSpaces.size(); i++)
for (int level = 0; level < nLevel; level++) for (InteriorBoundary::iterator it(boundary); !it.end(); ++it)
for (InteriorBoundary::iterator it(boundary, level); !it.end(); ++it){
it->rankObj.el->getAllDofs(feSpaces[i], it->rankObj, it->rankObj.el->getAllDofs(feSpaces[i], it->rankObj,
data[level][it.getRank()][feSpaces[i]]); data[it.getRank()][feSpaces[i]]);
}
// === Remove empty data containers. === // === Remove empty data containers. ===
for (unsigned int i = 0; i < data.size(); i++) { DataIter dit = data.begin();
DataIter dit = data[i].begin(); while (dit != data.end()) {
while (dit != data[i].end()) {
FeMapIter it = dit->second.begin(); FeMapIter it = dit->second.begin();
while (it != dit->second.end()) { while (it != dit->second.end()) {
if (it->second.size() == 0) { if (it->second.size() == 0) {
...@@ -85,12 +72,11 @@ namespace AMDiS { ...@@ -85,12 +72,11 @@ namespace AMDiS {
} }
if (dit->second.size() == 0) if (dit->second.size() == 0)
data[i].erase(dit++); data.erase(dit++);
else else
++dit; ++dit;
} }
} }
}
void DofComm::serialize(ostream &out) void DofComm::serialize(ostream &out)
...@@ -101,20 +87,16 @@ namespace AMDiS { ...@@ -101,20 +87,16 @@ namespace AMDiS {
} }
int DofComm::getNumberDofs(LevelDataType &data, int DofComm::getNumberDofs(DataType &data,
int level,
const FiniteElemSpace *feSpace, const FiniteElemSpace *feSpace,
bool countDouble) bool countDouble)
{ {
FUNCNAME("DofComm::getNumberDofs()"); FUNCNAME("DofComm::getNumberDofs()");
TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofSet; DofContainerSet dofSet;
DofContainer dofVec; DofContainer dofVec;
for (DataIter rankIt = data[level].begin(); for (DataIter rankIt = data.begin(); rankIt != data.end(); ++rankIt)
rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin(); for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt) feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace) if (feIt->first == feSpace)
...@@ -133,7 +115,7 @@ namespace AMDiS { ...@@ -133,7 +115,7 @@ namespace AMDiS {
{ {
FUNCNAME("DofComm::Iterator::setNextFeMap()"); FUNCNAME("DofComm::Iterator::setNextFeMap()");
if (dataIter != data[traverseLevel].end()) { if (dataIter != data.end()) {
TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n"); TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
feMapIter = dataIter->second.begin(); feMapIter = dataIter->second.begin();
...@@ -158,4 +140,25 @@ namespace AMDiS { ...@@ -158,4 +140,25 @@ namespace AMDiS {
return true; return true;
} }
void MultiLevelDofComm::init(MeshLevelData &levelData,
vector<const FiniteElemSpace*> &fe)
{
FUNCNAME("MultiLevelDofComm::init()");
int nLevel = levelData.getNumberOfLevels();
for (int level = 0; level < nLevel; level++)
levelDofComm[level].init(fe);
}
void MultiLevelDofComm::create(MultiLevelInteriorBoundary &boundary)
{
FUNCNAME("MultiLevelDofComm::create()");
for (map<int, DofComm>::iterator it = levelDofComm.begin();
it != levelDofComm.end(); ++it)
it->second.create(boundary[it->first]);
}
} }
...@@ -37,38 +37,28 @@ namespace AMDiS { ...@@ -37,38 +37,28 @@ namespace AMDiS {
{ {
public: public:
DofComm() DofComm()
: recvDofs(1),
sendDofs(1),
periodicDofs(0),
meshLevel(-1),
nLevel(0),
levelData(NULL)
{} {}
typedef map<const FiniteElemSpace*, DofContainer> FeMapType; typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
typedef FeMapType::iterator FeMapIter; typedef FeMapType::iterator FeMapIter;
typedef map<int, FeMapType> DataType; typedef map<int, FeMapType> DataType;
typedef DataType::iterator DataIter; typedef DataType::iterator DataIter;
// meshLevel: map[rank -> map[feSpace -> DofContainer]]
typedef vector<DataType> LevelDataType;
void init(int level, void init(vector<const FiniteElemSpace*> &fe);
MeshLevelData &levelData,
vector<const FiniteElemSpace*> &fe);
void create(InteriorBoundary &boundary); void create(InteriorBoundary &boundary);
LevelDataType& getSendDofs() DataType& getSendDofs()
{ {
return sendDofs; return sendDofs;
} }
LevelDataType& getRecvDofs() DataType& getRecvDofs()
{ {
return recvDofs; return recvDofs;
} }
LevelDataType& getPeriodicDofs() DataType& getPeriodicDofs()
{ {
return periodicDofs; return periodicDofs;
} }
...@@ -76,35 +66,29 @@ namespace AMDiS { ...@@ -76,35 +66,29 @@ namespace AMDiS {
// Writes all data of this object to an output stream. // Writes all data of this object to an output stream.
void serialize(ostream &out); void serialize(ostream &out);
int getNumberDofs(LevelDataType &data, int getNumberDofs(DataType &data,
int level,
const FiniteElemSpace *feSpace, const FiniteElemSpace *feSpace,
bool countDouble = false); bool countDouble = false);
protected: protected:
void createContainer(RankToBoundMap &boundary, LevelDataType &data); void createContainer(RankToBoundMap &boundary,
DataType &data);
protected: protected:
/// This map contains for each rank the list of DOFs the current rank must /// This map contains for each rank the list of DOFs the current rank must
/// end to exchange solution DOFs at the interior boundaries. /// end to exchange solution DOFs at the interior boundaries.
LevelDataType sendDofs; DataType sendDofs;
/// This map contains on each rank the list of DOFs from which the current /// This map contains on each rank the list of DOFs from which the current
/// rank will receive DOF values (i.e., this are all DOFs at an interior /// rank will receive DOF values (i.e., this are all DOFs at an interior
/// boundary). The DOF indices are given in rank's local numbering. /// boundary). The DOF indices are given in rank's local numbering.
LevelDataType recvDofs; DataType recvDofs;
/// This map contains on each rank a list of DOFs along the interior bound- /// This map contains on each rank a list of DOFs along the interior bound-
/// aries to communicate with other ranks. The DOF indices are given in rank's /// aries to communicate with other ranks. The DOF indices are given in rank's
/// local numbering. Periodic boundaries within one subdomain are not /// local numbering. Periodic boundaries within one subdomain are not
/// considered here. /// considered here.
LevelDataType periodicDofs; DataType periodicDofs;
int meshLevel;
int nLevel;
MeshLevelData *levelData;
vector<const FiniteElemSpace*> feSpaces; vector<const FiniteElemSpace*> feSpaces;
...@@ -114,24 +98,11 @@ namespace AMDiS { ...@@ -114,24 +98,11 @@ namespace AMDiS {
class Iterator class Iterator
{ {
public: public:
Iterator(LevelDataType &d, Iterator(DataType &d,
const FiniteElemSpace *fe = NULL) const FiniteElemSpace *fe = NULL)
: data(d), : data(d),
dofCounter(-1), dofCounter(-1),
traverseFeSpace(fe), traverseFeSpace(fe),
traverseLevel(0),
removedDof(false)
{
goFirst();
}
Iterator(LevelDataType &d,
int level,
const FiniteElemSpace *fe = NULL)
: data(d),
dofCounter(-1),
traverseFeSpace(fe),
traverseLevel(level),
removedDof(false) removedDof(false)
{ {
goFirst(); goFirst();
...@@ -139,7 +110,7 @@ namespace AMDiS { ...@@ -139,7 +110,7 @@ namespace AMDiS {
inline bool end() inline bool end()
{ {
return (dataIter == data[traverseLevel].end()); return (dataIter == data.end());
} }
inline void nextRank() inline void nextRank()
...@@ -242,7 +213,7 @@ namespace AMDiS { ...@@ -242,7 +213,7 @@ namespace AMDiS {
protected: protected:
void goFirst() void goFirst()
{ {
dataIter = data[traverseLevel].begin(); dataIter = data.begin();
while (setNextFeMap() == false) while (setNextFeMap() == false)
++dataIter; ++dataIter;
...@@ -251,7 +222,7 @@ namespace AMDiS { ...@@ -251,7 +222,7 @@ namespace AMDiS {
bool setNextFeMap(); bool setNextFeMap();
protected: protected:
LevelDataType &data; DataType &data;
DofComm::DataIter dataIter; DofComm::DataIter dataIter;
...@@ -263,12 +234,26 @@ namespace AMDiS { ...@@ -263,12 +234,26 @@ namespace AMDiS {
const FiniteElemSpace *traverseFeSpace; const FiniteElemSpace *traverseFeSpace;
int traverseLevel;
bool removedDof; bool removedDof;
}; };
};
class MultiLevelDofComm {
public:
void init(MeshLevelData &levelData,
vector<const FiniteElemSpace*> &fe);
void create(MultiLevelInteriorBoundary &boundary);
inline DofComm& operator[](int level)
{
return levelDofComm[level];
}
private:
map<int, DofComm> levelDofComm;
}; };
} }
......
...@@ -722,10 +722,12 @@ namespace AMDiS { ...@@ -722,10 +722,12 @@ namespace AMDiS {
int ElementObjectDatabase::getOwner(vector<ElementObjectData>& objData, int ElementObjectDatabase::getOwner(vector<ElementObjectData>& objData,
int level) int level)
{ {
FUNCNAME("ElementObjectDatabase::getOwner()");
int owner = -1; int owner = -1;
std::set<int> &levelRanks = levelData->getLevelRanks(level); std::set<int> &levelRanks = levelData->getLevelRanks(level);
bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1); bool allRanks = (level == 0);
for (vector<ElementObjectData>::iterator it = objData.begin(); for (vector<ElementObjectData>::iterator it = objData.begin();
it != objData.end(); ++it) { it != objData.end(); ++it) {
...@@ -744,7 +746,7 @@ namespace AMDiS { ...@@ -744,7 +746,7 @@ namespace AMDiS {
{ {
FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()"); FUNCNAME("ElementObjectDatabase::getIterateMaxLevel()");
int nLevel = levelData->getLevelNumber(); int nLevel = levelData->getNumberOfLevels();
TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n"); TEST_EXIT_DBG(nLevel > 0)("Should not happen!\n");
vector<std::set<int> > ranksInLevel; vector<std::set<int> > ranksInLevel;
......
...@@ -143,6 +143,12 @@ namespace AMDiS { ...@@ -143,6 +143,12 @@ namespace AMDiS {
/// All data from the database is dropped. /// All data from the database is dropped.
void clear(); void clear();
/// Resets iteration;
bool resetIterator()
{
iterGeoPos = CENTER;
}
/** \brief /** \brief
* Iterates over all elements for one geometrical index, i.e., over all * Iterates over all elements for one geometrical index, i.e., over all
* vertices, edges or faces in the mesh. The function returns true, if the * vertices, edges or faces in the mesh. The function returns true, if the
...@@ -151,7 +157,7 @@ namespace AMDiS { ...@@ -151,7 +157,7 @@ namespace AMDiS {
* \param[in] pos Must be either VERTEX, EDGE or FACE and defines the * \param[in] pos Must be either VERTEX, EDGE or FACE and defines the
* elements that should be traversed. * elements that should be traversed.
*/ */
bool iterate(GeoIndex pos) inline bool iterate(GeoIndex pos)
{ {
// CENTER marks the variable "iterGeoPos" to be in an undefined state. I.e., // CENTER marks the variable "iterGeoPos" to be in an undefined state. I.e.,
// there is no iteration that is actually running. // there is no iteration that is actually running.
......
...@@ -35,19 +35,24 @@ namespace AMDiS { ...@@ -35,19 +35,24 @@ namespace AMDiS {
Mesh *mesh = elObjDb.getMesh(); Mesh *mesh = elObjDb.getMesh();
TEST_EXIT_DBG(mesh)("Should not happen!\n"); TEST_EXIT_DBG(mesh)("Should not happen!\n");
TEST_EXIT_DBG(level < levelData.getLevelNumber())("Should not happen!\n"); TEST_EXIT_DBG(level < levelData.getNumberOfLevels())
("Should not happen!\n");
MPI::Intracomm mpiComm = levelData.getMpiComm(level); MPI::Intracomm mpiComm = levelData.getMpiComm(level);
if (mpiComm == MPI::COMM_SELF)
return;
int levelMpiRank = mpiComm.Get_rank(); int levelMpiRank = mpiComm.Get_rank();
int globalMpiRank = MPI::COMM_WORLD.Get_rank(); int globalMpiRank = MPI::COMM_WORLD.Get_rank();
std::set<int> levelRanks = levelData.getLevelRanks(level); std::set<int> levelRanks = levelData.getLevelRanks(level);
bool allRanks = (levelRanks.size() == 1 && *(levelRanks.begin()) == -1);
// === Create interior boundary data structure. === // === Create interior boundary data structure. ===
for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) { for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) {
GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim()); GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim());
elObjDb.resetIterator();
while (elObjDb.iterate(geoIndex)) { while (elObjDb.iterate(geoIndex)) {
flat_map<int, ElementObjectData>& objData = elObjDb.getIterateData(); flat_map<int, ElementObjectData>& objData = elObjDb.getIterateData();
...@@ -58,17 +63,22 @@ namespace AMDiS { ...@@ -58,17 +63,22 @@ namespace AMDiS {
// Test, if the boundary object defines an interior boundary within the // Test, if the boundary object defines an interior boundary within the
// ranks of the MPI group. If not, go to next element. // ranks of the MPI group. If not, go to next element.
bool boundaryWithinMpiGroup = false; bool boundaryWithinMpiGroup = false;
if (allRanks) { if (levelData.getNumberOfLevels() == 1) {
boundaryWithinMpiGroup = true; boundaryWithinMpiGroup = true;
} else { } else {
TEST_EXIT_DBG(level + 1 < levelData.getNumberOfLevels())
("Level = %d but number of levels = %d\n",
level, levelData.getNumberOfLevels());
for (flat_map<int, ElementObjectData>::iterator it = objData.begin(); for (flat_map<int, ElementObjectData>::iterator it = objData.begin();
it != objData.end(); ++it) { it != objData.end(); ++it) {
if (it->first != globalMpiRank && levelRanks.count(it->first)) { if (!levelData.rankInLevel(it->first, level + 1)) {
boundaryWithinMpiGroup = true; boundaryWithinMpiGroup = true;
break; break;
} }
} }
} }
if (!boundaryWithinMpiGroup) if (!boundaryWithinMpiGroup)
continue; continue;
...@@ -100,7 +110,7 @@ namespace AMDiS { ...@@ -100,7 +110,7 @@ namespace AMDiS {
if (it2->first == globalMpiRank) if (it2->first == globalMpiRank)
continue; continue;
if (!allRanks && levelRanks.count(it2->first) == 0) if (!levelData.rankInLevel(it2->first, level))
continue; continue;
bound.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex); bound.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex);
...@@ -111,7 +121,8 @@ namespace AMDiS { ...@@ -111,7 +121,8 @@ namespace AMDiS {
bound.type = INTERIOR; bound.type = INTERIOR;
AtomicBoundary& b = getNewOwn(it2->first); int rankInLevel = levelData.mapRank(it2->first, 0, level);