Commit abe07e75 authored by Thomas Witkowski's avatar Thomas Witkowski

Bugfix in parallel code.

parent 61282acc
...@@ -44,39 +44,25 @@ namespace AMDiS { ...@@ -44,39 +44,25 @@ namespace AMDiS {
public: public:
virtual ~ProblemTimeInterface() {}; virtual ~ProblemTimeInterface() {};
/** \brief /// Executes all needed operations when the simulation time changes.
* Executes all needed operations when the simulation time changes.
*/
virtual void setTime(AdaptInfo *adaptInfo) = 0; virtual void setTime(AdaptInfo *adaptInfo) = 0;
/** \brief /// Called at the beginning of each timestep
* Called at the beginning of each timestep
*/
virtual void initTimestep(AdaptInfo *adaptInfo) = 0; virtual void initTimestep(AdaptInfo *adaptInfo) = 0;
/** \brief /// Called at the end of each timestep.
* Called at the end of each timestep.
*/
virtual void closeTimestep(AdaptInfo *adaptInfo) = 0; virtual void closeTimestep(AdaptInfo *adaptInfo) = 0;
/** \brief /// Solves the initial problem.
* Solves the initial problem.
*/
virtual void solveInitialProblem(AdaptInfo *adaptInfo) = 0; virtual void solveInitialProblem(AdaptInfo *adaptInfo) = 0;
/** \brief /// Solves the initial problem.
* Solves the initial problem.
*/
virtual void transferInitialSolution(AdaptInfo *adaptInfo) = 0; virtual void transferInitialSolution(AdaptInfo *adaptInfo) = 0;
/** \brief /// Function that serializes the problem plus information about the iteration.
* Function that serializes the problem plus information about the iteration.
*/
virtual void serialize(std::ostream &out) = 0; virtual void serialize(std::ostream &out) = 0;
/** \brief /// Function that deserializes the problem plus information about the iteration.
* Function that deserializes the problem plus information about the iteration.
*/
virtual void deserialize(std::istream &in) = 0; virtual void deserialize(std::istream &in) = 0;
}; };
......
...@@ -177,6 +177,9 @@ namespace AMDiS { ...@@ -177,6 +177,9 @@ namespace AMDiS {
inline void nextNonempty() inline void nextNonempty()
{ {
if (mapIt == bound.boundary.end())
return;
while (mapIt->second.size() == 0) { while (mapIt->second.size() == 0) {
++mapIt; ++mapIt;
if (mapIt == bound.boundary.end()) if (mapIt == bound.boundary.end())
......
#include <algorithm> #include <algorithm>
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
#include <boost/lexical_cast.hpp>
#include "parallel/ParallelDomainBase.h" #include "parallel/ParallelDomainBase.h"
#include "parallel/StdMpi.h" #include "parallel/StdMpi.h"
#include "ParMetisPartitioner.h" #include "ParMetisPartitioner.h"
...@@ -21,12 +21,16 @@ ...@@ -21,12 +21,16 @@
#include "ElementFileWriter.h" #include "ElementFileWriter.h"
#include "VertexVector.h" #include "VertexVector.h"
#include "MeshStructure.h" #include "MeshStructure.h"
#include "ProblemVec.h"
#include "ProblemInstat.h"
#include "Debug.h" #include "Debug.h"
#include "petscksp.h" #include "petscksp.h"
namespace AMDiS { namespace AMDiS {
using boost::lexical_cast;
PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *) PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
{ {
if (iter % 10 == 0 && MPI::COMM_WORLD.Get_rank() == 0) if (iter % 10 == 0 && MPI::COMM_WORLD.Get_rank() == 0)
...@@ -40,22 +44,22 @@ namespace AMDiS { ...@@ -40,22 +44,22 @@ namespace AMDiS {
return (*dof1 < *dof2); return (*dof1 < *dof2);
} }
ParallelDomainBase::ParallelDomainBase(ProblemIterationInterface *iIF, ParallelDomainBase::ParallelDomainBase(ProblemVec *problemStat,
ProblemTimeInterface *tIF, ProblemInstatVec *problemInstat)
FiniteElemSpace *fe, : iterationIF(problemStat),
RefinementManager *refinementManager) timeIF(problemInstat),
: iterationIF(iIF), probStat(problemStat),
timeIF(tIF), name(problemStat->getName()),
name(iIF->getName()), feSpace(problemStat->getFESpace(0)),
feSpace(fe), mesh(feSpace->getMesh()),
mesh(fe->getMesh()), refineManager(problemStat->getRefinementManager(0)),
refineManager(refinementManager), info(problemStat->getInfo()),
initialPartitionMesh(true), initialPartitionMesh(true),
d_nnz(NULL), d_nnz(NULL),
o_nnz(NULL), o_nnz(NULL),
nRankDofs(0), nRankDofs(0),
rstart(0), rstart(0),
nComponents(1), nComponents(problemStat->getNumComponents()),
deserialized(false), deserialized(false),
lastMeshChangeIndex(0) lastMeshChangeIndex(0)
{ {
...@@ -70,8 +74,34 @@ namespace AMDiS { ...@@ -70,8 +74,34 @@ namespace AMDiS {
mpiSize = MPI::COMM_WORLD.Get_size(); mpiSize = MPI::COMM_WORLD.Get_size();
mpiComm = MPI::COMM_WORLD; mpiComm = MPI::COMM_WORLD;
partitioner = new ParMetisPartitioner(mesh, &mpiComm); partitioner = new ParMetisPartitioner(mesh, &mpiComm);
// Test if all fe spaces are equal. Yet, different fe spaces are not supported for
// domain parallelization.
const FiniteElemSpace *fe = probStat->getFESpace(0);
for (int i = 0; i < nComponents; i++)
TEST_EXIT(fe == probStat->getFESpace(i))
("Parallelization does not supported different FE spaces!\n");
// Create parallel serialization file writer, if needed.
int writeSerialization = 0;
GET_PARAMETER(0, name + "->output->write serialization", "%d", &writeSerialization);
if (writeSerialization)
probStat->getFileWriterList().push_back(new Serializer<ParallelDomainBase>(this));
int readSerialization = 0;
GET_PARAMETER(0, name + "->input->read serialization", "%d", &readSerialization);
if (readSerialization) {
std::string filename = "";
GET_PARAMETER(0, name + "->input->serialization filename", &filename);
filename += ".p" + lexical_cast<std::string>(mpiRank);
MSG("Start serialization with %s\n", filename.c_str());
std::ifstream in(filename.c_str());
deserialize(in);
in.close();
}
} }
void ParallelDomainBase::initParallelization(AdaptInfo *adaptInfo) void ParallelDomainBase::initParallelization(AdaptInfo *adaptInfo)
{ {
FUNCNAME("ParallelDomainBase::initParallelization()"); FUNCNAME("ParallelDomainBase::initParallelization()");
...@@ -157,6 +187,46 @@ namespace AMDiS { ...@@ -157,6 +187,46 @@ namespace AMDiS {
createPeriodicMap(); createPeriodicMap();
} }
/// === Set DOF rank information to all matrices and vectors. ===
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++)
if (probStat->getSystemMatrix(i, j))
probStat->getSystemMatrix(i, j)->setRankDofs(isRankDof);
TEST_EXIT_DBG(probStat->getRHS()->getDOFVector(i))("No rhs vector!\n");
TEST_EXIT_DBG(probStat->getSolution()->getDOFVector(i))("No solution vector!\n");
probStat->getRHS()->getDOFVector(i)->setRankDofs(isRankDof);
probStat->getSolution()->getDOFVector(i)->setRankDofs(isRankDof);
}
// === Remove periodic boundary conditions in sequential problem definition. ===
// Remove periodic boundaries in boundary manager on matrices and vectors.
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++) {
DOFMatrix* mat = probStat->getSystemMatrix(i, j);
if (mat && mat->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(mat->getBoundaryManager()->getBoundaryConditionMap()));
}
if (probStat->getSolution()->getDOFVector(i)->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(probStat->getSolution()->getDOFVector(i)->getBoundaryManager()->getBoundaryConditionMap()));
if (probStat->getRHS()->getDOFVector(i)->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(probStat->getRHS()->getDOFVector(i)->getBoundaryManager()->getBoundaryConditionMap()));
}
// Remove periodic boundaries on elements in mesh.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
elInfo->getElement()->deleteElementData(PERIODIC);
elInfo = stack.traverseNext(elInfo);
}
} }
...@@ -726,6 +796,18 @@ namespace AMDiS { ...@@ -726,6 +796,18 @@ namespace AMDiS {
} }
void ParallelDomainBase::removeBoundaryCondition(BoundaryIndexMap& boundaryMap)
{
BoundaryIndexMap::iterator it = boundaryMap.begin();
while (it != boundaryMap.end()) {
if (it->second->isPeriodic())
boundaryMap.erase(it++);
else
++it;
}
}
void ParallelDomainBase::checkMeshChange() void ParallelDomainBase::checkMeshChange()
{ {
FUNCNAME("ParallelDomainBase::checkMeshChange()"); FUNCNAME("ParallelDomainBase::checkMeshChange()");
...@@ -929,7 +1011,7 @@ namespace AMDiS { ...@@ -929,7 +1011,7 @@ namespace AMDiS {
void ParallelDomainBase::serialize(std::ostream &out, DofContainer &data) void ParallelDomainBase::serialize(std::ostream &out, DofContainer &data)
{ {
int vecSize = data.size(); int vecSize = data.size();
SerUtil::serialize(out, vecSize); SerUtil::serialize(out, vecSize);
for (int i = 0; i < vecSize; i++) { for (int i = 0; i < vecSize; i++) {
...@@ -1058,6 +1140,15 @@ namespace AMDiS { ...@@ -1058,6 +1140,15 @@ namespace AMDiS {
} }
void ParallelDomainBase::solveInitialProblem(AdaptInfo *adaptInfo)
{
if (timeIF)
timeIF->solveInitialProblem(adaptInfo);
synchVector(*(probStat->getSolution()));
}
void ParallelDomainBase::createInteriorBoundaryInfo() void ParallelDomainBase::createInteriorBoundaryInfo()
{ {
FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()"); FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()");
...@@ -1346,7 +1437,8 @@ namespace AMDiS { ...@@ -1346,7 +1437,8 @@ namespace AMDiS {
// Stores all vertices at rank's interior boundaries. // Stores all vertices at rank's interior boundaries.
std::set<DegreeOfFreedom> rankBoundaryDofs; std::set<DegreeOfFreedom> rankBoundaryDofs;
// First, traverse the rank owned elements af the interior boundaries.
// First, traverse the rank owned elements at the interior boundaries.
for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it) { for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it) {
Element *el = it.getBound().rankObj.el; Element *el = it.getBound().rankObj.el;
basFcts->getLocalIndices(el, feSpace->getAdmin(), localIndices); basFcts->getLocalIndices(el, feSpace->getAdmin(), localIndices);
...@@ -2262,6 +2354,29 @@ namespace AMDiS { ...@@ -2262,6 +2354,29 @@ namespace AMDiS {
} }
void ParallelDomainBase::solve()
{
FUNCNAME("ParallelDomainBase::solve()");
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();
fillPetscMatrix(probStat->getSystemMatrix(), probStat->getRHS());
solvePetscMatrix(*(probStat->getSolution()));
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()),
omp_get_wtime() - wtime);
#else
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
}
void ParallelDomainBase::serialize(std::ostream &out) void ParallelDomainBase::serialize(std::ostream &out)
{ {
SerUtil::serialize(out, elemWeights); SerUtil::serialize(out, elemWeights);
...@@ -2286,6 +2401,8 @@ namespace AMDiS { ...@@ -2286,6 +2401,8 @@ namespace AMDiS {
SerUtil::serialize(out, rstart); SerUtil::serialize(out, rstart);
SerUtil::serialize(out, nRankRows); SerUtil::serialize(out, nRankRows);
SerUtil::serialize(out, nOverallRows); SerUtil::serialize(out, nOverallRows);
probStat->serialize(out);
} }
...@@ -2332,6 +2449,8 @@ namespace AMDiS { ...@@ -2332,6 +2449,8 @@ namespace AMDiS {
SerUtil::deserialize(in, nOverallRows); SerUtil::deserialize(in, nOverallRows);
deserialized = true; deserialized = true;
probStat->deserialize(in);
} }
void ParallelDomainBase::serialize(std::ostream &out, PeriodicDofMap &data) void ParallelDomainBase::serialize(std::ostream &out, PeriodicDofMap &data)
......
...@@ -26,21 +26,21 @@ ...@@ -26,21 +26,21 @@
#include <map> #include <map>
#include <set> #include <set>
#include <vector> #include <vector>
#include <mpi.h>
#include "Global.h"
#include "ProblemTimeInterface.h" #include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h" #include "ProblemIterationInterface.h"
#include "FiniteElemSpace.h" #include "FiniteElemSpace.h"
#include "AdaptInfo.h" #include "AdaptInfo.h"
#include "InteriorBoundary.h" #include "parallel/InteriorBoundary.h"
#include "Serializer.h" #include "Serializer.h"
#include "BoundaryManager.h"
#include "AMDiS_fwd.h" #include "AMDiS_fwd.h"
#include "petsc.h" #include "petsc.h"
#include "petscsys.h" #include "petscsys.h"
#include "petscao.h" #include "petscao.h"
#include "mpi.h"
#include "Global.h"
namespace AMDiS { namespace AMDiS {
...@@ -90,11 +90,9 @@ namespace AMDiS { ...@@ -90,11 +90,9 @@ namespace AMDiS {
typedef std::vector<MeshStructure> MeshCodeVec; typedef std::vector<MeshStructure> MeshCodeVec;
public: public:
ParallelDomainBase(ProblemIterationInterface *iterationIF, ParallelDomainBase(ProblemVec *problemStat,
ProblemTimeInterface *timeIF, ProblemInstatVec *problemInstat);
FiniteElemSpace *feSpace,
RefinementManager *refineManager);
virtual ~ParallelDomainBase() {} virtual ~ParallelDomainBase() {}
virtual void initParallelization(AdaptInfo *adaptInfo); virtual void initParallelization(AdaptInfo *adaptInfo);
...@@ -143,11 +141,7 @@ namespace AMDiS { ...@@ -143,11 +141,7 @@ namespace AMDiS {
timeIF->closeTimestep(adaptInfo); timeIF->closeTimestep(adaptInfo);
} }
virtual void solveInitialProblem(AdaptInfo *adaptInfo) virtual void solveInitialProblem(AdaptInfo *adaptInfo);
{
if (timeIF)
timeIF->solveInitialProblem(adaptInfo);
}
virtual void transferInitialSolution(AdaptInfo *adaptInfo) virtual void transferInitialSolution(AdaptInfo *adaptInfo)
{ {
...@@ -169,7 +163,7 @@ namespace AMDiS { ...@@ -169,7 +163,7 @@ namespace AMDiS {
iterationIF->endIteration(adaptInfo); iterationIF->endIteration(adaptInfo);
} }
virtual void solve() {} virtual void solve();
virtual int getNumProblems() virtual int getNumProblems()
{ {
...@@ -378,6 +372,9 @@ namespace AMDiS { ...@@ -378,6 +372,9 @@ namespace AMDiS {
*/ */
void synchVector(SystemVector &vec); void synchVector(SystemVector &vec);
// Removes all periodic boundaries from a given boundary map.
void removeBoundaryCondition(BoundaryIndexMap& boundaryMap);
bool fitElementToMeshCode(MeshStructure &code, bool fitElementToMeshCode(MeshStructure &code,
Element *el, Element *el,
int ithSide, int ithSide,
...@@ -489,6 +486,9 @@ namespace AMDiS { ...@@ -489,6 +486,9 @@ namespace AMDiS {
/// ///
ProblemTimeInterface *timeIF; ProblemTimeInterface *timeIF;
///
ProblemVec *probStat;
/// The rank of the current process. /// The rank of the current process.
int mpiRank; int mpiRank;
......
...@@ -12,132 +12,9 @@ namespace AMDiS { ...@@ -12,132 +12,9 @@ namespace AMDiS {
ParallelDomainVec::ParallelDomainVec(ProblemVec *problem, ParallelDomainVec::ParallelDomainVec(ProblemVec *problem,
ProblemInstatVec *problemInstat) ProblemInstatVec *problemInstat)
: ParallelDomainBase(problem, : ParallelDomainBase(problem, problemInstat)
problemInstat,
problem->getFESpace(0),
problem->getRefinementManager(0)),
probVec(problem)
{ {
FUNCNAME("ParallelDomainVec::ParallelDomainVec()"); FUNCNAME("ParallelDomainVec::ParallelDomainVec()");
info = probVec->getInfo();
nComponents = probVec->getNumComponents();
// Test if all fe spaces are equal. Yet, different fe spaces are not supported for
// domain parallelization.
const FiniteElemSpace *fe = probVec->getFESpace(0);
for (int i = 0; i < nComponents; i++)
TEST_EXIT(fe == probVec->getFESpace(i))
("Parallelization does not supported different FE spaces!\n");
// Create parallel serialization file writer, if needed.
int writeSerialization = 0;
GET_PARAMETER(0, name + "->output->write serialization", "%d", &writeSerialization);
if (writeSerialization)
problem->getFileWriterList().push_back(new Serializer<ParallelDomainVec>(this));
int readSerialization = 0;
GET_PARAMETER(0, name + "->input->read serialization", "%d", &readSerialization);
if (readSerialization) {
std::string filename = "";
GET_PARAMETER(0, name + "->input->serialization filename", &filename);
filename += ".p" + lexical_cast<std::string>(mpiRank);
MSG("Start serialization with %s\n", filename.c_str());
std::ifstream in(filename.c_str());
deserialize(in);
in.close();
}
}
void ParallelDomainVec::initParallelization(AdaptInfo *adaptInfo)
{
FUNCNAME("ParallelDomainVec::initParallelization()");
ParallelDomainBase::initParallelization(adaptInfo);
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++)
if (probVec->getSystemMatrix(i, j))
probVec->getSystemMatrix(i, j)->setRankDofs(isRankDof);
TEST_EXIT_DBG(probVec->getRHS()->getDOFVector(i))("No rhs vector!\n");
TEST_EXIT_DBG(probVec->getSolution()->getDOFVector(i))("No solution vector!\n");
probVec->getRHS()->getDOFVector(i)->setRankDofs(isRankDof);
probVec->getSolution()->getDOFVector(i)->setRankDofs(isRankDof);
}
// === Remove periodic boundary conditions in sequential problem definition. ===
// Remove periodic boundaries in boundary manager on matrices and vectors.
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++) {
DOFMatrix* mat = probVec->getSystemMatrix(i, j);
if (mat && mat->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(mat->getBoundaryManager()->getBoundaryConditionMap()));
}
if (probVec->getSolution()->getDOFVector(i)->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(probVec->getSolution()->getDOFVector(i)->getBoundaryManager()->getBoundaryConditionMap()));
if (probVec->getRHS()->getDOFVector(i)->getBoundaryManager())
removeBoundaryCondition(const_cast<BoundaryIndexMap&>(probVec->getRHS()->getDOFVector(i)->getBoundaryManager()->getBoundaryConditionMap()));
}
// Remove periodic boundaries on elements in mesh.
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
elInfo->getElement()->deleteElementData(PERIODIC);
elInfo = stack.traverseNext(elInfo);
}
} }
void ParallelDomainVec::exitParallelization(AdaptInfo *adaptInfo)
{
ParallelDomainBase::exitParallelization(adaptInfo);
}
void ParallelDomainVec::solveInitialProblem(AdaptInfo *adaptInfo)
{
ParallelDomainBase::solveInitialProblem(adaptInfo);
synchVector(*(probVec->getSolution()));
}
void ParallelDomainVec::solve()
{
FUNCNAME("ParallelDomainVec::solve()");
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();