Commit 2c96156e authored by Thomas Witkowski's avatar Thomas Witkowski

Blub

parent a2e41205
...@@ -42,6 +42,11 @@ namespace AMDiS { ...@@ -42,6 +42,11 @@ namespace AMDiS {
elCounter, 6 * static_cast<int>(pow(mpiSize, 1.5)), mpiSize); elCounter, 6 * static_cast<int>(pow(mpiSize, 1.5)), mpiSize);
} }
if (multilevel) {
TEST_EXIT(MPI::COMM_WORLD.Get_size() == 16)
("Multilevel partitioning is implemented for 16 nodes only!\n");
}
int dim = mesh->getDim(); int dim = mesh->getDim();
TraverseStack stack; TraverseStack stack;
...@@ -54,7 +59,13 @@ namespace AMDiS { ...@@ -54,7 +59,13 @@ namespace AMDiS {
switch (mode) { switch (mode) {
case 0: case 0:
elInRank = boxIndex; if (!multilevel) {
elInRank = boxIndex;
} else {
TEST_EXIT_DBG(boxIndex >= 0 && boxIndex <= 15)("Wrong box index!\n");
int rs[16] = {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
elInRank = rs[boxIndex];
}
break; break;
case 1: case 1:
......
...@@ -38,21 +38,26 @@ namespace AMDiS { ...@@ -38,21 +38,26 @@ namespace AMDiS {
CheckerPartitioner(MPI::Intracomm *comm) CheckerPartitioner(MPI::Intracomm *comm)
: MeshPartitioner(comm), : MeshPartitioner(comm),
mpiRank(mpiComm->Get_rank()), mpiRank(mpiComm->Get_rank()),
mpiSize(mpiComm->Get_size()) mpiSize(mpiComm->Get_size()),
mode(0),
multilevel(false)
{ {
string modestr = ""; string modestr = "";
Parameters::get("parallel->partitioner->mode", modestr); Parameters::get("parallel->partitioner->mode", modestr);
if (modestr == "") if (modestr == "x-stripes")
mode = 0;
else if (modestr == "x-stripes")
mode = 1; mode = 1;
else if (modestr == "y-stripes") else if (modestr == "y-stripes")
mode = 2; mode = 2;
else if (modestr == "z-stripes") else if (modestr == "z-stripes")
mode = 3; mode = 3;
else else if (modestr == "multilevel")
ERROR_EXIT("No partitioner mode \"%s\"!\n", modestr.c_str()); multilevel = true;
else {
if (modestr != "") {
ERROR_EXIT("No partitioner mode \"%s\"!\n", modestr.c_str());
}
}
} }
~CheckerPartitioner() {} ~CheckerPartitioner() {}
...@@ -77,6 +82,8 @@ namespace AMDiS { ...@@ -77,6 +82,8 @@ namespace AMDiS {
/// 1: x-stripes: each node gets one x-stripe of boxes /// 1: x-stripes: each node gets one x-stripe of boxes
/// 2: y-stripes: each node gets one y-stripe of boxes /// 2: y-stripes: each node gets one y-stripe of boxes
int mode; int mode;
bool multilevel;
}; };
} }
......
...@@ -91,22 +91,29 @@ namespace AMDiS { ...@@ -91,22 +91,29 @@ namespace AMDiS {
int DofComm::getNumberDofs(LevelDataType &data, int DofComm::getNumberDofs(LevelDataType &data,
int level, int level,
const FiniteElemSpace *feSpace) const FiniteElemSpace *feSpace,
bool countDouble)
{ {
FUNCNAME("DofComm::getNumberDofs()"); FUNCNAME("DofComm::getNumberDofs()");
TEST_EXIT_DBG(level < data.size())("Should not happen!\n"); TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
DofContainerSet dofs; DofContainerSet dofSet;
DofContainer dofVec;
for (DataIter rankIt = data[level].begin(); for (DataIter rankIt = data[level].begin();
rankIt != data[level].end(); ++rankIt) rankIt != data[level].end(); ++rankIt)
for (FeMapIter feIt = rankIt->second.begin(); for (FeMapIter feIt = rankIt->second.begin();
feIt != rankIt->second.end(); ++feIt) feIt != rankIt->second.end(); ++feIt)
if (feIt->first == feSpace) if (feIt->first == feSpace)
dofs.insert(feIt->second.begin(), feIt->second.end()); if (countDouble)
dofVec.insert(dofVec.end(), feIt->second.begin(), feIt->second.end());
return static_cast<int>(dofs.size()); else
dofSet.insert(feIt->second.begin(), feIt->second.end());
if (countDouble)
return static_cast<int>(dofVec.size());
return static_cast<int>(dofSet.size());
} }
......
...@@ -87,7 +87,8 @@ namespace AMDiS { ...@@ -87,7 +87,8 @@ namespace AMDiS {
int getNumberDofs(LevelDataType &data, int getNumberDofs(LevelDataType &data,
int level, int level,
const FiniteElemSpace *feSpace); const FiniteElemSpace *feSpace,
bool countDouble = false);
protected: protected:
void createContainer(RankToBoundMap &boundary, LevelDataType &data); void createContainer(RankToBoundMap &boundary, LevelDataType &data);
......
...@@ -176,7 +176,8 @@ namespace AMDiS { ...@@ -176,7 +176,8 @@ namespace AMDiS {
createBoundaryDofs(); createBoundaryDofs();
#if (DEBUG != 0) #if (DEBUG != 0)
ParallelDebug::writeDebugFile(*this, debugOutputDir + "mpi-dbg", "dat"); ParallelDebug::writeDebugFile(feSpaces[feSpaces.size() - 1], dofMap,
debugOutputDir + "mpi-dbg", "dat");
#endif #endif
initialized = true; initialized = true;
...@@ -501,13 +502,18 @@ namespace AMDiS { ...@@ -501,13 +502,18 @@ namespace AMDiS {
} }
void MeshDistributor::synchVector(SystemVector &vec) void MeshDistributor::synchVector(SystemVector &vec, int level)
{ {
FUNCNAME("MeshDistributor::synchVector()"); FUNCNAME("MeshDistributor::synchVector()");
StdMpi<vector<double> > stdMpi(mpiComm); TEST_EXIT_DBG(level >= 0 && level <= 1)("Wrong level number!\n");
for (DofComm::Iterator it(dofComm.getSendDofs()); MPI::Intracomm &levelComm = levelData.getMpiComm(level);
DofComm &dc = (level == 0 ? dofComm : dofCommSd);
StdMpi<vector<double> > stdMpi(levelComm);
for (DofComm::Iterator it(dc.getSendDofs());
!it.end(); it.nextRank()) { !it.end(); it.nextRank()) {
vector<double> dofs; vector<double> dofs;
...@@ -518,22 +524,33 @@ namespace AMDiS { ...@@ -518,22 +524,33 @@ namespace AMDiS {
dofs.push_back(dofVec[it.getDofIndex()]); dofs.push_back(dofVec[it.getDofIndex()]);
} }
stdMpi.send(it.getRank(), dofs); int rank = it.getRank();
if (level > 0)
rank = levelData.mapRank(rank, 0, level);
stdMpi.send(rank, dofs);
} }
for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank()) for (DofComm::Iterator it(dc.getRecvDofs()); !it.end(); it.nextRank()) {
stdMpi.recv(it.getRank()); int rank = it.getRank();
if (level > 0)
rank = levelData.mapRank(rank, 0, level);
stdMpi.recv(rank);
}
stdMpi.startCommunication(); stdMpi.startCommunication();
for (DofComm::Iterator it(dc.getRecvDofs()); !it.end(); it.nextRank()) {
int rank = it.getRank();
if (level > 0)
rank = levelData.mapRank(rank, 0, level);
for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank()) {
int counter = 0; int counter = 0;
for (int i = 0; i < vec.getSize(); i++) { for (int i = 0; i < vec.getSize(); i++) {
DOFVector<double> &dofVec = *(vec.getDOFVector(i)); DOFVector<double> &dofVec = *(vec.getDOFVector(i));
for (it.beginDofIter(vec.getFeSpace(i)); !it.endDofIter(); it.nextDof()) for (it.beginDofIter(vec.getFeSpace(i)); !it.endDofIter(); it.nextDof())
dofVec[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[counter++]; dofVec[it.getDofIndex()] = 0.0;//stdMpi.getRecvData(rank)[counter++];
} }
} }
} }
...@@ -777,60 +794,60 @@ namespace AMDiS { ...@@ -777,60 +794,60 @@ namespace AMDiS {
std::set<int> neighbours; std::set<int> neighbours;
switch (mpiRank) { switch (mpiRank) {
case 0: case 0:
neighbours.insert(1); neighbours.insert(4); neighbours.insert(5); neighbours.insert(1); neighbours.insert(2); neighbours.insert(3);
break; break;
case 1: case 1:
neighbours.insert(0); neighbours.insert(2); neighbours.insert(4); neighbours.insert(5); neighbours.insert(6); neighbours.insert(0); neighbours.insert(4); neighbours.insert(2); neighbours.insert(3); neighbours.insert(6);
break; break;
case 2: case 2:
neighbours.insert(1); neighbours.insert(3); neighbours.insert(5); neighbours.insert(6); neighbours.insert(7); neighbours.insert(0); neighbours.insert(1); neighbours.insert(3); neighbours.insert(8); neighbours.insert(9);
break; break;
case 3: case 3:
neighbours.insert(2); neighbours.insert(6); neighbours.insert(7); neighbours.insert(0); neighbours.insert(1); neighbours.insert(4);
neighbours.insert(2); neighbours.insert(6);
neighbours.insert(8); neighbours.insert(9); neighbours.insert(12);
break; break;
case 4: case 4:
neighbours.insert(0); neighbours.insert(1); neighbours.insert(5); neighbours.insert(8); neighbours.insert(9); neighbours.insert(1); neighbours.insert(3); neighbours.insert(6); neighbours.insert(7); neighbours.insert(5);
break; break;
case 5: case 5:
neighbours.insert(0); neighbours.insert(1); neighbours.insert(2); neighbours.insert(4); neighbours.insert(6); neighbours.insert(7);
neighbours.insert(4); neighbours.insert(6);
neighbours.insert(8); neighbours.insert(9); neighbours.insert(10);
break; break;
case 6: case 6:
neighbours.insert(1); neighbours.insert(2); neighbours.insert(3); neighbours.insert(1); neighbours.insert(4); neighbours.insert(5);
neighbours.insert(5); neighbours.insert(7); neighbours.insert(3); neighbours.insert(7);
neighbours.insert(9); neighbours.insert(10); neighbours.insert(11); neighbours.insert(9); neighbours.insert(12); neighbours.insert(13);
break; break;
case 7: case 7:
neighbours.insert(2); neighbours.insert(3); neighbours.insert(6); neighbours.insert(10); neighbours.insert(11); neighbours.insert(4); neighbours.insert(5); neighbours.insert(6); neighbours.insert(12); neighbours.insert(13);
break; break;
case 8: case 8:
neighbours.insert(4); neighbours.insert(5); neighbours.insert(9); neighbours.insert(12); neighbours.insert(13); neighbours.insert(2); neighbours.insert(3); neighbours.insert(9); neighbours.insert(10); neighbours.insert(11);
break; break;
case 9: case 9:
neighbours.insert(4); neighbours.insert(5); neighbours.insert(6); neighbours.insert(2); neighbours.insert(3); neighbours.insert(6);
neighbours.insert(8); neighbours.insert(10); neighbours.insert(8); neighbours.insert(12);
neighbours.insert(12); neighbours.insert(13); neighbours.insert(14); neighbours.insert(10); neighbours.insert(11); neighbours.insert(14);
break; break;
case 10: case 10:
neighbours.insert(5); neighbours.insert(6); neighbours.insert(7); neighbours.insert(8); neighbours.insert(9); neighbours.insert(11);
neighbours.insert(9); neighbours.insert(11);
neighbours.insert(13); neighbours.insert(14); neighbours.insert(15);
break; break;
case 11: case 11:
neighbours.insert(6); neighbours.insert(7); neighbours.insert(10); neighbours.insert(14); neighbours.insert(15); neighbours.insert(8); neighbours.insert(9); neighbours.insert(12); neighbours.insert(10); neighbours.insert(14);
break; break;
case 12: case 12:
neighbours.insert(8); neighbours.insert(9); neighbours.insert(13); neighbours.insert(3); neighbours.insert(6); neighbours.insert(7);
neighbours.insert(9); neighbours.insert(13);
neighbours.insert(11); neighbours.insert(14); neighbours.insert(15);
break; break;
case 13: case 13:
neighbours.insert(8); neighbours.insert(9); neighbours.insert(10); neighbours.insert(12); neighbours.insert(14); neighbours.insert(6); neighbours.insert(7); neighbours.insert(12); neighbours.insert(14); neighbours.insert(15);
break; break;
case 14: case 14:
neighbours.insert(9); neighbours.insert(10); neighbours.insert(11); neighbours.insert(13); neighbours.insert(15); neighbours.insert(9); neighbours.insert(12); neighbours.insert(13); neighbours.insert(11); neighbours.insert(15);
break; break;
case 15: case 15:
neighbours.insert(10); neighbours.insert(11); neighbours.insert(14); neighbours.insert(12); neighbours.insert(13); neighbours.insert(14);
break; break;
} }
...@@ -844,31 +861,31 @@ namespace AMDiS { ...@@ -844,31 +861,31 @@ namespace AMDiS {
Parameters::get("parallel->multi level test", multiLevelTest); Parameters::get("parallel->multi level test", multiLevelTest);
if (multiLevelTest) { if (multiLevelTest) {
switch (mpiRank) { switch (mpiRank) {
case 0: case 1: case 4: case 5: case 0: case 1: case 2: case 3:
{ {
std::set<int> m; std::set<int> m;
m.insert(0); m.insert(1); m.insert(4); m.insert(5); m.insert(0); m.insert(1); m.insert(2); m.insert(3);
levelData.addLevel(m, 0); levelData.addLevel(m, 0);
} }
break; break;
case 2: case 3: case 6: case 7: case 4: case 5: case 6: case 7:
{ {
std::set<int> m; std::set<int> m;
m.insert(2); m.insert(3); m.insert(6); m.insert(7); m.insert(4); m.insert(5); m.insert(6); m.insert(7);
levelData.addLevel(m, 1); levelData.addLevel(m, 1);
} }
break; break;
case 8: case 9: case 12: case 13: case 8: case 9: case 10: case 11:
{ {
std::set<int> m; std::set<int> m;
m.insert(8); m.insert(9); m.insert(12); m.insert(13); m.insert(8); m.insert(9); m.insert(10); m.insert(11);
levelData.addLevel(m, 2); levelData.addLevel(m, 2);
} }
break; break;
case 10: case 11: case 14: case 15: case 12: case 13: case 14: case 15:
{ {
std::set<int> m; std::set<int> m;
m.insert(10); m.insert(11); m.insert(14); m.insert(15); m.insert(12); m.insert(13); m.insert(14); m.insert(15);
levelData.addLevel(m, 3); levelData.addLevel(m, 3);
} }
break; break;
...@@ -1672,7 +1689,8 @@ namespace AMDiS { ...@@ -1672,7 +1689,8 @@ namespace AMDiS {
debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex-" + debug::writeElementIndexMesh(mesh, debugOutputDir + "elementIndex-" +
lexical_cast<string>(mpiRank) + ".vtu"); lexical_cast<string>(mpiRank) + ".vtu");
ParallelDebug::writeDebugFile(*this, debugOutputDir + "mpi-dbg", "dat"); ParallelDebug::writeDebugFile(feSpaces[feSpaces.size() - 1], dofMap,
debugOutputDir + "mpi-dbg", "dat");
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this); ParallelDebug::testGlobalIndexByCoords(*this);
...@@ -1680,7 +1698,8 @@ namespace AMDiS { ...@@ -1680,7 +1698,8 @@ namespace AMDiS {
int tmp = 0; int tmp = 0;
Parameters::get(name + "->write parallel debug file", tmp); Parameters::get(name + "->write parallel debug file", tmp);
if (tmp) if (tmp)
ParallelDebug::writeDebugFile(*this, debugOutputDir + "mpi-dbg", "dat"); ParallelDebug::writeDebugFile(feSpaces[feSpaces.size() - 1], dofMap,
debugOutputDir + "mpi-dbg", "dat");
#endif #endif
} }
......
...@@ -254,12 +254,10 @@ namespace AMDiS { ...@@ -254,12 +254,10 @@ namespace AMDiS {
stdMpi.getRecvData(it.getRank())[it.getDofCounter()]; stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
} }
/** \brief /// Works in the same way as the function above defined for DOFVectors. Due
* Works in the same way as the function above defined for DOFVectors. Due /// to performance, this function does not call \ref synchVector for each
* to performance, this function does not call \ref synchVector for each /// DOFVector, but instead sends all values of all DOFVectors all at once.
* DOFVector, but instead sends all values of all DOFVectors all at once. void synchVector(SystemVector &vec, int level = 0);
*/
void synchVector(SystemVector &vec);
void check3dValidMesh(); void check3dValidMesh();
......
...@@ -70,11 +70,13 @@ namespace AMDiS { ...@@ -70,11 +70,13 @@ namespace AMDiS {
void MeshLevelData::serialize(ostream &out) void MeshLevelData::serialize(ostream &out)
{ {
ERROR_EXIT("Not yet implemented!\n");
} }
void MeshLevelData::deserialize(istream &in) void MeshLevelData::deserialize(istream &in)
{ {
ERROR_EXIT("Not yet implemented!\n");
} }
......
...@@ -92,16 +92,16 @@ namespace AMDiS { ...@@ -92,16 +92,16 @@ namespace AMDiS {
TEST_EXIT_DBG(level == 1 && rank <= 15)("Should not happen!\n"); TEST_EXIT_DBG(level == 1 && rank <= 15)("Should not happen!\n");
TEST_EXIT_DBG(MPI::COMM_WORLD.Get_size() == 16)("Should not happen!\n"); TEST_EXIT_DBG(MPI::COMM_WORLD.Get_size() == 16)("Should not happen!\n");
if (rank == 0 || rank == 1 || rank == 4 || rank == 5) if (rank == 0 || rank == 1 || rank == 2 || rank == 3)
return 0; return 0;
if (rank == 2 || rank == 3 || rank == 6 || rank == 7) if (rank == 4 || rank == 5 || rank == 6 || rank == 7)
return 1; return 1;
if (rank == 8 || rank == 9 || rank == 12 || rank == 13) if (rank == 8 || rank == 9 || rank == 10 || rank == 11)
return 2; return 2;
if (rank == 10 || rank == 11 || rank == 14 || rank == 15) if (rank == 12 || rank == 13 || rank == 14 || rank == 15)
return 3; return 3;
ERROR_EXIT("Should not happen!\n"); ERROR_EXIT("Should not happen!\n");
......
...@@ -758,25 +758,27 @@ namespace AMDiS { ...@@ -758,25 +758,27 @@ namespace AMDiS {
} }
void ParallelDebug::writeDebugFile(MeshDistributor &pdb, void ParallelDebug::writeDebugFile(const FiniteElemSpace *feSpace,
string prefix, string postfix) ParallelDofMapping &dofMap,
string prefix,
string postfix)
{ {
FUNCNAME("ParallelDebug::writeCoordsFile()"); FUNCNAME("ParallelDebug::writeCoordsFile()");
const FiniteElemSpace *feSpace = pdb.feSpaces[pdb.feSpaces.size() - 1]; Mesh *mesh = feSpace->getMesh();
stringstream filename; stringstream filename;
filename << prefix << "-" << pdb.mpiRank << "." << postfix; filename << prefix << "-" << MPI::COMM_WORLD.Get_rank() << "." << postfix;
DOFVector<WorldVector<double> > coords(feSpace, "tmp"); DOFVector<WorldVector<double> > coords(feSpace, "tmp");
pdb.mesh->getDofIndexCoords(feSpace, coords);