Commit aff33faa authored by Thomas Witkowski's avatar Thomas Witkowski

Work on parallelization.

parent 808eac8e
...@@ -239,7 +239,8 @@ if(ENABLE_PARALLEL_DOMAIN) ...@@ -239,7 +239,8 @@ if(ENABLE_PARALLEL_DOMAIN)
find_package(PETSc REQUIRED) find_package(PETSc REQUIRED)
include_directories(${PETSC_DIR}/include ${PETSC_DIR}/${PETSC_ARCH}/include) include_directories(${PETSC_DIR}/include ${PETSC_DIR}/${PETSC_ARCH}/include)
list(APPEND AMDIS_INCLUDE_DIRS ${PETSC_DIR}/include ${PETSC_DIR}/${PETSC_ARCH}/include) list(APPEND AMDIS_INCLUDE_DIRS ${PETSC_DIR}/include ${PETSC_DIR}/${PETSC_ARCH}/include)
list(APPEND PARALLEL_DOMAIN_AMDIS_SRC ${SOURCE_DIR}/parallel/PetscSolver.cc list(APPEND PARALLEL_DOMAIN_AMDIS_SRC ${SOURCE_DIR}/parallel/PetscMultigridPrecon.cc
${SOURCE_DIR}/parallel/PetscSolver.cc
${SOURCE_DIR}/parallel/PetscProblemStat.cc ${SOURCE_DIR}/parallel/PetscProblemStat.cc
${SOURCE_DIR}/parallel/PetscSolverFeti.cc ${SOURCE_DIR}/parallel/PetscSolverFeti.cc
${SOURCE_DIR}/parallel/PetscSolverGlobalMatrix.cc ${SOURCE_DIR}/parallel/PetscSolverGlobalMatrix.cc
......
...@@ -63,7 +63,7 @@ namespace AMDiS { ...@@ -63,7 +63,7 @@ namespace AMDiS {
problemIteration->endIteration(adaptInfo); problemIteration->endIteration(adaptInfo);
adaptInfo->incSpaceIteration(); adaptInfo->incSpaceIteration();
} }
// adaption loop // adaption loop
while (!adaptInfo->spaceToleranceReached() && while (!adaptInfo->spaceToleranceReached() &&
(adaptInfo->getSpaceIteration() < adaptInfo->getMaxSpaceIteration() || (adaptInfo->getSpaceIteration() < adaptInfo->getMaxSpaceIteration() ||
...@@ -72,9 +72,17 @@ namespace AMDiS { ...@@ -72,9 +72,17 @@ namespace AMDiS {
problemIteration->beginIteration(adaptInfo); problemIteration->beginIteration(adaptInfo);
Flag adapted = problemIteration->oneIteration(adaptInfo, FULL_ITERATION); Flag adapted = problemIteration->oneIteration(adaptInfo, FULL_ITERATION);
problemIteration->endIteration(adaptInfo); problemIteration->endIteration(adaptInfo);
#if HAVE_PARALLEL_DOMAIN_AMDIS
int recvAllValues = 0;
int isAdapted = static_cast<bool>(adapted);
MPI::COMM_WORLD.Allreduce(&isAdapted, &recvAllValues, 1, MPI_INT, MPI_SUM);
if (recvAllValues == 0)
break;
#else
if (!adapted) if (!adapted)
break; break;
#endif
adaptInfo->incSpaceIteration(); adaptInfo->incSpaceIteration();
} }
......
...@@ -158,7 +158,7 @@ namespace AMDiS { ...@@ -158,7 +158,7 @@ namespace AMDiS {
inline Flag operator^(const Flag& f) const inline Flag operator^(const Flag& f) const
{ {
Flag r(flags); Flag r(flags);
r.flags ^=f.flags; r.flags ^= f.flags;
return r; return r;
} }
......
...@@ -363,10 +363,8 @@ namespace AMDiS { ...@@ -363,10 +363,8 @@ namespace AMDiS {
} }
void MeshStructure::print(bool resetCode) string MeshStructure::toStr(bool resetCode)
{ {
FUNCNAME("MeshStructure::print()");
std::stringstream oss; std::stringstream oss;
if (empty()) { if (empty()) {
...@@ -385,14 +383,24 @@ namespace AMDiS { ...@@ -385,14 +383,24 @@ namespace AMDiS {
cont = nextElement(); cont = nextElement();
} }
} }
return oss.str();
}
void MeshStructure::print(bool resetCode)
{
FUNCNAME("MeshStructure::print()");
string str = toStr(resetCode);
if (oss.str().length() < 255) { if (str.length() < 255) {
MSG("Mesh structure code: %s\n", oss.str().c_str()); MSG("Mesh structure code: %s\n", str.c_str());
} else { } else {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
std::cout << "[" << MPI::COMM_WORLD.Get_rank() << "] Mesh structure code: " << oss.str() << "\n"; std::cout << "[" << MPI::COMM_WORLD.Get_rank() << "] Mesh structure code: " << str << "\n";
#else #else
std::cout << " Mesh structure code: " << oss.str() << "\n"; std::cout << " Mesh structure code: " << str << "\n";
#endif #endif
} }
} }
......
...@@ -126,6 +126,9 @@ namespace AMDiS { ...@@ -126,6 +126,9 @@ namespace AMDiS {
int macroElIndex = -1, int macroElIndex = -1,
bool ignoreFinerMesh = false); bool ignoreFinerMesh = false);
/// Converts the mesh structure code to a string (for debugging).
string toStr(bool resetCode = true);
/// Prints the mesh structure code. /// Prints the mesh structure code.
void print(bool resetCode = true); void print(bool resetCode = true);
......
...@@ -45,10 +45,10 @@ namespace AMDiS { ...@@ -45,10 +45,10 @@ namespace AMDiS {
: name(""), : name(""),
problem(prob), problem(prob),
tsModulo(1), tsModulo(1),
timestepNumber(-1),
appendIndex(0), appendIndex(0),
indexLength(5), indexLength(5),
indexDecimals(3) indexDecimals(3),
timestepNumber(-1)
{ {
FUNCNAME("Serializer::Serializer()"); FUNCNAME("Serializer::Serializer()");
...@@ -68,10 +68,10 @@ namespace AMDiS { ...@@ -68,10 +68,10 @@ namespace AMDiS {
: name(filename), : name(filename),
problem(prob), problem(prob),
tsModulo(writeEveryIth), tsModulo(writeEveryIth),
timestepNumber(-1),
appendIndex(0), appendIndex(0),
indexLength(5), indexLength(5),
indexDecimals(3) indexDecimals(3),
timestepNumber(-1)
{ {
FUNCNAME("Serializer::Serializer()"); FUNCNAME("Serializer::Serializer()");
......
...@@ -15,27 +15,105 @@ ...@@ -15,27 +15,105 @@
namespace AMDiS { namespace AMDiS {
void CheckerPartitioner::createInitialPartitioning() void CheckerPartitioner::createInitialPartitioning()
{ {
FUNCNAME("CheckerPartitioner::createInitialPartitioning()"); FUNCNAME("CheckerPartitioner::createInitialPartitioning()");
int mpiRank = mpiComm->Get_rank(); // In one of the stripes mode, we have to check if the number of macro
int mpiSize = mpiComm->Get_size(); // elements with together with the number of nodes.
if (mode == 1 || mode == 2 || mode == 3) {
int elCounter = 0;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
elCounter++;
elInfo = stack.traverseNext(elInfo);
}
if (mesh->getDim() == 2)
TEST_EXIT(elCounter == 2 * mpiSize * mpiSize)
("The number of macro elements is %d, but must be %d for %d number of nodes!",
elCounter, 2 * mpiSize * mpiSize, mpiSize);
int nElementsPerBlock = (mesh->getDim() == 2 ? 2 : 6); if (mesh->getDim() == 3)
TEST_EXIT(elCounter == 6 * static_cast<int>(pow(mpiSize, 1.5)))
("The number of macro elements is %d, but must be %d for %d number of nodes!",
elCounter, 6 * static_cast<int>(pow(mpiSize, 1.5)), mpiSize);
}
int dim = mesh->getDim();
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL); ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) { while (elInfo) {
Element *el = elInfo->getElement(); Element *el = elInfo->getElement();
int elIndex = el->getIndex(); int elIndex = el->getIndex();
int elInRank = elIndex / nElementsPerBlock; int boxIndex = elIndex / (dim == 2 ? 2 : 6);
TEST_EXIT_DBG(elInRank < mpiSize)("Should not happen!\n"); int elInRank = -1;
switch (mode) {
case 0:
elInRank = boxIndex;
break;
case 1:
// x-slices
{
if (dim == 2)
elInRank = elIndex / (2 * mpiSize);
if (dim == 3) {
int boxSliceY =
(boxIndex % mpiSize) / static_cast<int>(sqrt(mpiSize));
int boxSliceZ = boxIndex / mpiSize;
elInRank = boxSliceY * static_cast<int>(sqrt(mpiSize)) + boxSliceZ;
}
}
break;
case 2:
// y-slices
{
if (dim == 2)
elInRank = (elIndex % (2 * mpiSize)) / 2;
if (dim == 3) {
int boxSliceX =
(boxIndex % mpiSize) % static_cast<int>(sqrt(mpiSize));
int boxSliceZ = boxIndex / mpiSize;
elInRank = boxSliceX * static_cast<int>(sqrt(mpiSize)) + boxSliceZ;
}
}
break;
case 3:
// z-slices
{
int boxSliceX = (boxIndex % mpiSize) % static_cast<int>(sqrt(mpiSize));
int boxSliceY = (boxIndex % mpiSize) / static_cast<int>(sqrt(mpiSize));
elInRank = boxSliceX * static_cast<int>(sqrt(mpiSize)) + boxSliceY;
}
break;
default:
ERROR_EXIT("Mode %d does not exists for checker based mesh partitioning!\n",
mode);
}
TEST_EXIT_DBG(elInRank >= 0)("Should not happen!\n");
TEST_EXIT_DBG(elInRank < mpiSize)("Should not happen!\n");
elementInRank[elIndex] = (elInRank == mpiRank); elementInRank[elIndex] = (elInRank == mpiRank);
partitionMap[elIndex] = elInRank; partitionMap[elIndex] = elInRank;
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
} }
......
...@@ -25,16 +25,35 @@ ...@@ -25,16 +25,35 @@
#include "AMDiS_fwd.h" #include "AMDiS_fwd.h"
#include "Global.h" #include "Global.h"
#include "Initfile.h"
#include "parallel/MeshPartitioner.h" #include "parallel/MeshPartitioner.h"
namespace AMDiS { namespace AMDiS {
using namespace std;
class CheckerPartitioner : public MeshPartitioner class CheckerPartitioner : public MeshPartitioner
{ {
public: public:
CheckerPartitioner(MPI::Intracomm *comm) CheckerPartitioner(MPI::Intracomm *comm)
: MeshPartitioner(comm) : MeshPartitioner(comm),
{} mpiRank(mpiComm->Get_rank()),
mpiSize(mpiComm->Get_size())
{
string modestr = "";
Parameters::get("parallel->partitioner->mode", modestr);
if (modestr == "")
mode = 0;
else if (modestr == "x-stripes")
mode = 1;
else if (modestr == "y-stripes")
mode = 2;
else if (modestr == "z-stripes")
mode = 3;
else
ERROR_EXIT("No partitioner mode \"%s\"!\n", modestr.c_str());
}
~CheckerPartitioner() {} ~CheckerPartitioner() {}
...@@ -50,6 +69,14 @@ namespace AMDiS { ...@@ -50,6 +69,14 @@ namespace AMDiS {
{ {
pMap = partitionMap; pMap = partitionMap;
} }
protected:
int mpiRank, mpiSize;
/// 0: standard mode: each node gets one box
/// 1: x-stripes: each node gets one x-stripe of boxes
/// 2: y-stripes: each node gets one y-stripe of boxes
int mode;
}; };
} }
......
...@@ -77,8 +77,8 @@ namespace AMDiS { ...@@ -77,8 +77,8 @@ namespace AMDiS {
info(10), info(10),
partitioner(NULL), partitioner(NULL),
nRankDofs(0), nRankDofs(0),
nOverallDofs(0),
rStartDofs(0), rStartDofs(0),
nOverallDofs(0),
deserialized(false), deserialized(false),
writeSerializationFile(false), writeSerializationFile(false),
repartitioningAllowed(false), repartitioningAllowed(false),
...@@ -297,6 +297,7 @@ namespace AMDiS { ...@@ -297,6 +297,7 @@ namespace AMDiS {
ParallelDebug::testAllElements(*this); ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::followBoundary(*this);
debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh"); debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh");
...@@ -892,8 +893,8 @@ namespace AMDiS { ...@@ -892,8 +893,8 @@ namespace AMDiS {
map<int, MeshCodeVec> sendCodes; map<int, MeshCodeVec> sendCodes;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) { for (RankToBoundMap::iterator it = allBound.begin();
it != allBound.end(); ++it) {
for (vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
MeshStructure elCode; MeshStructure elCode;
...@@ -904,7 +905,8 @@ namespace AMDiS { ...@@ -904,7 +905,8 @@ namespace AMDiS {
StdMpi<MeshCodeVec> stdMpi(mpiComm, true); StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes); stdMpi.send(sendCodes);
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) for (RankToBoundMap::iterator it = allBound.begin();
it != allBound.end(); ++it)
stdMpi.recv(it->first); stdMpi.recv(it->first);
MSG("DA 1\n"); MSG("DA 1\n");
...@@ -917,7 +919,8 @@ namespace AMDiS { ...@@ -917,7 +919,8 @@ namespace AMDiS {
bool meshChanged = false; bool meshChanged = false;
for (RankToBoundMap::iterator it = allBound.begin(); it != allBound.end(); ++it) { for (RankToBoundMap::iterator it = allBound.begin();
it != allBound.end(); ++it) {
MeshCodeVec &recvCodes = stdMpi.getRecvData()[it->first]; MeshCodeVec &recvCodes = stdMpi.getRecvData()[it->first];
int i = 0; int i = 0;
...@@ -928,6 +931,10 @@ namespace AMDiS { ...@@ -928,6 +931,10 @@ namespace AMDiS {
MeshStructure elCode; MeshStructure elCode;
elCode.init(boundIt->rankObj); elCode.init(boundIt->rankObj);
#if (DEBUG != 0)
ParallelDebug::followBoundary(mesh, *boundIt, elCode);
#endif
if (elCode.getCode() != recvCodes[i].getCode()) { if (elCode.getCode() != recvCodes[i].getCode()) {
// MSG("MACRO EL %d %d %d DOES NOT FIT WITH MACRO EL %d %d %d ON RANK %d\n", // MSG("MACRO EL %d %d %d DOES NOT FIT WITH MACRO EL %d %d %d ON RANK %d\n",
// boundIt->rankObj.elIndex, boundIt->rankObj.subObj, boundIt->rankObj.ithObj, // boundIt->rankObj.elIndex, boundIt->rankObj.subObj, boundIt->rankObj.ithObj,
...@@ -1440,77 +1447,78 @@ namespace AMDiS { ...@@ -1440,77 +1447,78 @@ namespace AMDiS {
while (elObjects.iterate(geoIndex)) { while (elObjects.iterate(geoIndex)) {
map<int, ElementObjectData>& objData = elObjects.getIterateData(); map<int, ElementObjectData>& objData = elObjects.getIterateData();
if (objData.count(mpiRank) && objData.size() > 1) { if (!(objData.count(mpiRank) && objData.size() > 1))
int owner = elObjects.getIterateOwner(); continue;
ElementObjectData& rankBoundEl = objData[mpiRank];
TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])("Should not happen!\n");
AtomicBoundary bound;
bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject;
if (geoIndex == FACE) {
for (int edgeNo = 0; edgeNo < 3; edgeNo++) {
int edgeOfFace =
bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo);
bound.rankObj.excludedSubstructures.push_back(make_pair(EDGE, edgeOfFace)); int owner = elObjects.getIterateOwner();
} ElementObjectData& rankBoundEl = objData[mpiRank];
}
TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])("Should not happen!\n");
if (owner == mpiRank) { AtomicBoundary bound;
for (map<int, ElementObjectData>::iterator it2 = objData.begin(); bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
it2 != objData.end(); ++it2) { bound.rankObj.elIndex = rankBoundEl.elIndex;
if (it2->first == mpiRank) bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex];
continue; bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject;
bound.neighObj.el = macroElIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex; if (geoIndex == FACE) {
bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex]; for (int edgeNo = 0; edgeNo < 3; edgeNo++) {
bound.neighObj.subObj = geoIndex; int edgeOfFace =
bound.neighObj.ithObj = it2->second.ithObject; bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo);
bound.type = INTERIOR;
AtomicBoundary& b = myIntBoundary.getNewAtomic(it2->first);
b = bound;
if (geoIndex == EDGE)
b.neighObj.reverseMode = elObjects.getEdgeReverseMode(rankBoundEl, it2->second);
if (geoIndex == FACE)
b.neighObj.reverseMode = elObjects.getFaceReverseMode(rankBoundEl, it2->second);
}
} else {
TEST_EXIT_DBG(objData.count(owner) == 1)
("Should not happen!\n");
ElementObjectData& ownerBoundEl = objData[owner]; bound.rankObj.excludedSubstructures.push_back(make_pair(EDGE, edgeOfFace));
}
}
if (owner == mpiRank) {
for (map<int, ElementObjectData>::iterator it2 = objData.begin();
it2 != objData.end(); ++it2) {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex]; bound.neighObj.el = macroElIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex; bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = -1; bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex];
bound.neighObj.subObj = geoIndex; bound.neighObj.subObj = geoIndex;
bound.neighObj.ithObj = ownerBoundEl.ithObject; bound.neighObj.ithObj = it2->second.ithObject;
bound.type = INTERIOR; bound.type = INTERIOR;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner); AtomicBoundary& b = myIntBoundary.getNewAtomic(it2->first);
b = bound; b = bound;
if (geoIndex == EDGE) if (geoIndex == EDGE)
b.rankObj.reverseMode = elObjects.getEdgeReverseMode(rankBoundEl, ownerBoundEl); b.neighObj.reverseMode = elObjects.getEdgeReverseMode(rankBoundEl, it2->second);
if (geoIndex == FACE) if (geoIndex == FACE)
b.rankObj.reverseMode = elObjects.getFaceReverseMode(rankBoundEl, ownerBoundEl); b.neighObj.reverseMode = elObjects.getFaceReverseMode(rankBoundEl, it2->second);
} }
} else {
TEST_EXIT_DBG(objData.count(owner) == 1)
("Should not happen!\n");
ElementObjectData& ownerBoundEl = objData[owner];
bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = geoIndex;
bound.neighObj.ithObj = ownerBoundEl.ithObject;
bound.type = INTERIOR;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner);
b = bound;
if (geoIndex == EDGE)
b.rankObj.reverseMode = elObjects.getEdgeReverseMode(rankBoundEl, ownerBoundEl);
if (geoIndex == FACE)
b.rankObj.reverseMode = elObjects.getFaceReverseMode(rankBoundEl, ownerBoundEl);
} }
} }
} }
// === Create periodic boundary data structure. === // === Create periodic boundary data structure. ===
for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjects.getPeriodicVertices().begin(); for (PerBoundMap<DegreeOfFreedom>::iterator it = elObjects.getPeriodicVertices().begin();
......
...@@ -835,6 +835,7 @@ namespace AMDiS { ...@@ -835,6 +835,7 @@ namespace AMDiS {
ElementFileWriter::writeFile(vec, pdb.mesh, filename); ElementFileWriter::writeFile(vec, pdb.mesh, filename);
} }
void ParallelDebug::writePartitioningFile(string filename, void ParallelDebug::writePartitioningFile(string filename,
int counter, int counter,
FiniteElemSpace *feSpace) FiniteElemSpace *feSpace)
...@@ -851,4 +852,57 @@ namespace AMDiS { ...@@ -851,4 +852,57 @@ namespace AMDiS {