Commit 1de9db4b authored by Thomas Witkowski's avatar Thomas Witkowski

Global MeshDistributor added.

parent 91edc9fa
...@@ -332,4 +332,39 @@ namespace AMDiS { ...@@ -332,4 +332,39 @@ namespace AMDiS {
while (clock() < endwait) {} while (clock() < endwait) {}
} }
void processMemUsage(double& vm_usage, double& resident_set)
{
using std::ios_base;
using std::ifstream;
using std::string;
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;
// in case x86-64 is configured to use 2MB pages
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
}
} }
...@@ -109,6 +109,8 @@ namespace AMDiS { ...@@ -109,6 +109,8 @@ namespace AMDiS {
void waitSec(int seconds); void waitSec(int seconds);
void processMemUsage(double& vm_usage, double& resident_set);
/// Content comparision of two pointers. Used e.g. for find_if /// Content comparision of two pointers. Used e.g. for find_if
template<typename T> template<typename T>
struct comparePtrContents : public binary_function<T*, T*, bool> struct comparePtrContents : public binary_function<T*, T*, bool>
......
...@@ -514,21 +514,12 @@ namespace AMDiS { ...@@ -514,21 +514,12 @@ namespace AMDiS {
return; return;
} }
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock(); clock_t first = clock();
solver->solveSystem(solverMatrix, *solution, *rhs); solver->solveSystem(solverMatrix, *solution, *rhs);
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("solution of discrete system needed %.5f seconds\n", INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
adaptInfo->setSolverIterations(solver->getIterations()); adaptInfo->setSolverIterations(solver->getIterations());
adaptInfo->setMaxSolverIterations(solver->getMaxIterations()); adaptInfo->setMaxSolverIterations(solver->getMaxIterations());
...@@ -543,12 +534,8 @@ namespace AMDiS { ...@@ -543,12 +534,8 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime(); double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else #else
clock_t first = clock(); clock_t first = clock();
#endif
#endif #endif
if (computeExactError) { if (computeExactError) {
...@@ -577,14 +564,9 @@ namespace AMDiS { ...@@ -577,14 +564,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
INFO(info, 8)("estimation of the error needed %.5f seconds\n", INFO(info, 8)("estimation of the error needed %.5f seconds\n",
MPI::Wtime() - first); MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
omp_get_wtime() - first);
#else #else
INFO(info, 8)("estimation of the error needed %.5f seconds\n", INFO(info, 8)("estimation of the error needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
#endif #endif
} }
...@@ -682,13 +664,9 @@ namespace AMDiS { ...@@ -682,13 +664,9 @@ namespace AMDiS {
MPI::Wtime() - first); MPI::Wtime() - first);
first = MPI::Wtime(); first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else #else
clock_t first = clock(); clock_t first = clock();
#endif #endif
#endif
Flag assembleFlag = Flag assembleFlag =
...@@ -812,14 +790,9 @@ namespace AMDiS { ...@@ -812,14 +790,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
MPI::Wtime() - first); MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
omp_get_wtime() - first);
#else #else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
#endif #endif
} }
...@@ -829,9 +802,6 @@ namespace AMDiS { ...@@ -829,9 +802,6 @@ namespace AMDiS {
FUNCNAME("ProblemStat::buildAfterCoarsen()"); FUNCNAME("ProblemStat::buildAfterCoarsen()");
clock_t first = clock(); clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
for (int i = 0; i < static_cast<int>(meshes.size()); i++) for (int i = 0; i < static_cast<int>(meshes.size()); i++)
meshes[i]->dofCompress(); meshes[i]->dofCompress();
...@@ -1005,14 +975,8 @@ namespace AMDiS { ...@@ -1005,14 +975,8 @@ namespace AMDiS {
createPrecon(); createPrecon();
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz); INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
} }
bool ProblemStatSeq::dualMeshTraverseRequired() bool ProblemStatSeq::dualMeshTraverseRequired()
...@@ -1037,10 +1001,6 @@ namespace AMDiS { ...@@ -1037,10 +1001,6 @@ namespace AMDiS {
clock_t first = clock(); clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
Flag assembleFlag = Flag assembleFlag =
flag | flag |
...@@ -1278,13 +1238,8 @@ namespace AMDiS { ...@@ -1278,13 +1238,8 @@ namespace AMDiS {
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz); INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
} }
...@@ -1312,34 +1267,20 @@ namespace AMDiS { ...@@ -1312,34 +1267,20 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime(); double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else #else
clock_t first = clock(); clock_t first = clock();
#endif
#endif #endif
int size = static_cast<int>(fileWriters.size()); for (int i = 0; i < static_cast<int>(fileWriters.size()); i++)
#ifdef _OPENMP
#pragma omp parallel for schedule(static, 1)
#endif
for (int i = 0; i < size; i++) {
fileWriters[i]->writeFiles(adaptInfo, force); fileWriters[i]->writeFiles(adaptInfo, force);
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
INFO(info, 8)("writeFiles needed %.5f seconds\n", INFO(info, 8)("writeFiles needed %.5f seconds\n",
MPI::Wtime() - first); MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("writeFiles needed %.5f seconds\n",
omp_get_wtime() - first);
#else #else
INFO(info, 8)("writeFiles needed %.5f seconds\n", INFO(info, 8)("writeFiles needed %.5f seconds\n",
TIME_USED(first, clock())); TIME_USED(first, clock()));
#endif
#endif #endif
} }
......
...@@ -110,7 +110,6 @@ namespace AMDiS { ...@@ -110,7 +110,6 @@ namespace AMDiS {
ProblemStatSeq *adoptProblem = NULL, ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING); Flag adoptFlag = INIT_NOTHING);
/// Used in \ref initialize(). /// Used in \ref initialize().
virtual void createMesh(); virtual void createMesh();
......
...@@ -55,6 +55,8 @@ namespace AMDiS { ...@@ -55,6 +55,8 @@ namespace AMDiS {
using namespace boost::filesystem; using namespace boost::filesystem;
using namespace std; using namespace std;
MeshDistributor* MeshDistributor::globalMeshDistributor = NULL;
const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED = 0X01L; const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED = 0X01L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS = 0X02L; const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS = 0X02L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS = 0X04L; const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS = 0X04L;
...@@ -436,6 +438,17 @@ namespace AMDiS { ...@@ -436,6 +438,17 @@ namespace AMDiS {
} }
void MeshDistributor::addProblemStatGlobal(ProblemStatSeq *probStat)
{
FUNCNAME("MeshDistributor::addProblemStatGlobal()");
if (globalMeshDistributor == NULL)
globalMeshDistributor = new MeshDistributor();
globalMeshDistributor->addProblemStat(probStat);
}
void MeshDistributor::exitParallelization() void MeshDistributor::exitParallelization()
{} {}
......
...@@ -51,17 +51,16 @@ namespace AMDiS { ...@@ -51,17 +51,16 @@ namespace AMDiS {
class MeshDistributor class MeshDistributor
{ {
public: private:
MeshDistributor(); MeshDistributor();
virtual ~MeshDistributor() {} virtual ~MeshDistributor() {}
public:
void initParallelization(); void initParallelization();
void exitParallelization(); void exitParallelization();
void addProblemStat(ProblemStatSeq *probStat);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned. /// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector<double> *vec) void addInterchangeVector(DOFVector<double> *vec)
...@@ -279,7 +278,15 @@ namespace AMDiS { ...@@ -279,7 +278,15 @@ namespace AMDiS {
void getAllBoundaryDofs(DofContainer& dofs); void getAllBoundaryDofs(DofContainer& dofs);
public:
/// Adds a stationary problem to the global mesh distributor objects.
static void addProblemStatGlobal(ProblemStatSeq *probStat);
protected: protected:
void addProblemStat(ProblemStatSeq *probStat);
/** \brief /** \brief
* Determines the interior boundaries, i.e. boundaries between ranks, and stores * Determines the interior boundaries, i.e. boundaries between ranks, and stores
* all information about them in \ref interiorBoundary. * all information about them in \ref interiorBoundary.
...@@ -590,6 +597,7 @@ namespace AMDiS { ...@@ -590,6 +597,7 @@ namespace AMDiS {
Flag createBoundaryDofFlag; Flag createBoundaryDofFlag;
BoundaryDofInfo boundaryDofInfo; BoundaryDofInfo boundaryDofInfo;
public: public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all /// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// face DOFs, edge DOFs and to the last vertex DOFs will be set to
...@@ -606,6 +614,8 @@ namespace AMDiS { ...@@ -606,6 +614,8 @@ namespace AMDiS {
/// that are owned by another rank). /// that are owned by another rank).
static const Flag BOUNDARY_FILL_INFO_RECV_DOFS; static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
static MeshDistributor *globalMeshDistributor;
friend class ParallelDebug; friend class ParallelDebug;
}; };
} }
......
...@@ -43,23 +43,6 @@ namespace AMDiS { ...@@ -43,23 +43,6 @@ namespace AMDiS {
CreatorMap< OEMSolver >::addCreator("pminres", creator); CreatorMap< OEMSolver >::addCreator("pminres", creator);
} }
/* void Mtl4Solver::addPMTLPrecons()
{
ParallelPreconditionCreator *creator;
creator = new DiagonalPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pdiag", creator);
creator = new ILUPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pilu", creator);
creator = new ICPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pic", creator);
creator = new IdentityPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pno", creator);
}*/
void Mtl4Solver::createSolver() void Mtl4Solver::createSolver()
{ {
...@@ -77,57 +60,39 @@ namespace AMDiS { ...@@ -77,57 +60,39 @@ namespace AMDiS {
solver->initParameters(); solver->initParameters();
} }
//TODO: replace the name in the map
void Mtl4Solver::createPrecon()
{
/*std::string preconType("no");
GET_PARAMETER(0, name + "->solver->left precon", &preconType);
preconType = "p" + preconType ;
CreatorInterface<BasePreconditioner> *preconCreator =
CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setLeftPrecon(preconCreator->create(solverMatrix.getMatrix())); void Mtl4Solver::createPrecon()
solver->setLeftPrecon(preconCreator); {}
preconType= "no";
GET_PARAMETER(0, name + "->solver->right precon", &preconType);
preconType = "p" + preconType;
preconCreator = CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setRightPrecon(preconCreator->create(solverMatrix.getMatrix()));
solver->setRightPrecon(preconCreator);*/
}
void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix) void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix)
{ {
/* FUNCNAME("Mtl4Solver::solve()"); FUNCNAME("Mtl4Solver::solve()");
std::string solverName("");
GET_PARAMETER(0, name+"->solver", &solverName); clock_t first = clock();
TEST_EXIT(solverName != "")("need solver name");
OEMSolver* solver = getPMTLSolver(solverName);
delete solver;
*/
ParallelMapper pmapper(*meshDistributor, nComponents); ParallelMapper pmapper(*meshDistributor, nComponents);
solver->solveSystem(solverMatrix, *solution, *rhs, pmapper); solver->solveSystem(solverMatrix, *solution, *rhs, pmapper);
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
} }
} }
template< > template< >
void CreatorMap< Parallel::ParallelPreconditioner >::addDefaultCreators() { void CreatorMap< Parallel::ParallelPreconditioner >::addDefaultCreators() {
Parallel::ParallelPreconditionCreator *creator; Parallel::ParallelPreconditionCreator *creator;
creator = new Parallel::DiagonalPreconditioner::Creator; creator = new Parallel::DiagonalPreconditioner::Creator;
addCreator("diag", creator); addCreator("diag", creator);
creator = new Parallel::ILUPreconditioner::Creator; creator = new Parallel::ILUPreconditioner::Creator;
addCreator("ilu", creator); addCreator("ilu", creator);
creator = new Parallel::ICPreconditioner::Creator; creator = new Parallel::ICPreconditioner::Creator;
addCreator("ic", creator); addCreator("ic", creator);
creator = new Parallel::IdentityPreconditioner::Creator; creator = new Parallel::IdentityPreconditioner::Creator;
addCreator("no", creator); addCreator("no", creator);
} }
} }
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include "parallel/ParallelProblemStatBase.h" #include "parallel/ParallelProblemStatBase.h"
#include "parallel/MeshDistributor.h" #include "parallel/MeshDistributor.h"
#include "parallel/MpiHelper.h" #include "parallel/MpiHelper.h"
#include "Global.h"
namespace AMDiS { namespace AMDiS {
...@@ -22,7 +23,9 @@ namespace AMDiS { ...@@ -22,7 +23,9 @@ namespace AMDiS {
{ {
FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()"); FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
meshDistributor->checkMeshChange(); TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
("Should not happen!\n");
MeshDistributor::globalMeshDistributor->checkMeshChange();
ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag, ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag,
assembleMatrix, assembleMatrix,
assembleVector); assembleVector);
...@@ -39,47 +42,17 @@ namespace AMDiS { ...@@ -39,47 +42,17 @@ namespace AMDiS {
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss); MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
} }
void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
{
meshDistributor = &m;
m.addProblemStat(this);
}
void ParallelProblemStatBase::processMemUsage(double& vm_usage, void ParallelProblemStatBase::initialize(Flag initFlag,
double& resident_set) ProblemStatSeq *adoptProblem,
Flag adoptFlag)
{ {
using std::ios_base; ProblemStatSeq::initialize(initFlag, adoptProblem, adoptFlag);
using std::ifstream;
using std::string; MeshDistributor::addProblemStatGlobal(this);
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;