Commit 1de9db4b authored by Thomas Witkowski's avatar Thomas Witkowski

Global MeshDistributor added.

parent 91edc9fa
......@@ -332,4 +332,39 @@ namespace AMDiS {
while (clock() < endwait) {}
}
void processMemUsage(double& vm_usage, double& resident_set)
{
using std::ios_base;
using std::ifstream;
using std::string;
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;
// in case x86-64 is configured to use 2MB pages
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
}
}
......@@ -109,6 +109,8 @@ namespace AMDiS {
void waitSec(int seconds);
void processMemUsage(double& vm_usage, double& resident_set);
/// Content comparision of two pointers. Used e.g. for find_if
template<typename T>
struct comparePtrContents : public binary_function<T*, T*, bool>
......
......@@ -514,21 +514,12 @@ namespace AMDiS {
return;
}
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();
solver->solveSystem(solverMatrix, *solution, *rhs);
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
adaptInfo->setSolverIterations(solver->getIterations());
adaptInfo->setMaxSolverIterations(solver->getMaxIterations());
......@@ -543,12 +534,8 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
if (computeExactError) {
......@@ -577,14 +564,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......@@ -682,13 +664,9 @@ namespace AMDiS {
MPI::Wtime() - first);
first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
Flag assembleFlag =
......@@ -812,14 +790,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
TIME_USED(first, clock()));
#endif
}
......@@ -829,9 +802,6 @@ namespace AMDiS {
FUNCNAME("ProblemStat::buildAfterCoarsen()");
clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
for (int i = 0; i < static_cast<int>(meshes.size()); i++)
meshes[i]->dofCompress();
......@@ -1005,14 +975,8 @@ namespace AMDiS {
createPrecon();
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
}
bool ProblemStatSeq::dualMeshTraverseRequired()
......@@ -1037,10 +1001,6 @@ namespace AMDiS {
clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
Flag assembleFlag =
flag |
......@@ -1278,13 +1238,8 @@ namespace AMDiS {
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
}
......@@ -1312,34 +1267,20 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
int size = static_cast<int>(fileWriters.size());
#ifdef _OPENMP
#pragma omp parallel for schedule(static, 1)
#endif
for (int i = 0; i < size; i++) {
for (int i = 0; i < static_cast<int>(fileWriters.size()); i++)
fileWriters[i]->writeFiles(adaptInfo, force);
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("writeFiles needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("writeFiles needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("writeFiles needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......
......@@ -110,7 +110,6 @@ namespace AMDiS {
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
/// Used in \ref initialize().
virtual void createMesh();
......
......@@ -55,6 +55,8 @@ namespace AMDiS {
using namespace boost::filesystem;
using namespace std;
MeshDistributor* MeshDistributor::globalMeshDistributor = NULL;
const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED = 0X01L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS = 0X02L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS = 0X04L;
......@@ -436,6 +438,17 @@ namespace AMDiS {
}
void MeshDistributor::addProblemStatGlobal(ProblemStatSeq *probStat)
{
FUNCNAME("MeshDistributor::addProblemStatGlobal()");
if (globalMeshDistributor == NULL)
globalMeshDistributor = new MeshDistributor();
globalMeshDistributor->addProblemStat(probStat);
}
void MeshDistributor::exitParallelization()
{}
......
......@@ -51,17 +51,16 @@ namespace AMDiS {
class MeshDistributor
{
public:
private:
MeshDistributor();
virtual ~MeshDistributor() {}
public:
void initParallelization();
void exitParallelization();
void addProblemStat(ProblemStatSeq *probStat);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector<double> *vec)
......@@ -279,7 +278,15 @@ namespace AMDiS {
void getAllBoundaryDofs(DofContainer& dofs);
public:
/// Adds a stationary problem to the global mesh distributor objects.
static void addProblemStatGlobal(ProblemStatSeq *probStat);
protected:
void addProblemStat(ProblemStatSeq *probStat);
/** \brief
* Determines the interior boundaries, i.e. boundaries between ranks, and stores
* all information about them in \ref interiorBoundary.
......@@ -590,6 +597,7 @@ namespace AMDiS {
Flag createBoundaryDofFlag;
BoundaryDofInfo boundaryDofInfo;
public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
......@@ -606,6 +614,8 @@ namespace AMDiS {
/// that are owned by another rank).
static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
static MeshDistributor *globalMeshDistributor;
friend class ParallelDebug;
};
}
......
......@@ -43,23 +43,6 @@ namespace AMDiS {
CreatorMap< OEMSolver >::addCreator("pminres", creator);
}
/* void Mtl4Solver::addPMTLPrecons()
{
ParallelPreconditionCreator *creator;
creator = new DiagonalPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pdiag", creator);
creator = new ILUPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pilu", creator);
creator = new ICPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pic", creator);
creator = new IdentityPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pno", creator);
}*/
void Mtl4Solver::createSolver()
{
......@@ -77,57 +60,39 @@ namespace AMDiS {
solver->initParameters();
}
//TODO: replace the name in the map
void Mtl4Solver::createPrecon()
{
/*std::string preconType("no");
GET_PARAMETER(0, name + "->solver->left precon", &preconType);
preconType = "p" + preconType ;
CreatorInterface<BasePreconditioner> *preconCreator =
CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setLeftPrecon(preconCreator->create(solverMatrix.getMatrix()));
solver->setLeftPrecon(preconCreator);
preconType= "no";
GET_PARAMETER(0, name + "->solver->right precon", &preconType);
preconType = "p" + preconType;
void Mtl4Solver::createPrecon()
{}
preconCreator = CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setRightPrecon(preconCreator->create(solverMatrix.getMatrix()));
solver->setRightPrecon(preconCreator);*/
}
void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix)
{
/* FUNCNAME("Mtl4Solver::solve()");
std::string solverName("");
GET_PARAMETER(0, name+"->solver", &solverName);
TEST_EXIT(solverName != "")("need solver name");
OEMSolver* solver = getPMTLSolver(solverName);
delete solver;
*/
FUNCNAME("Mtl4Solver::solve()");
clock_t first = clock();
ParallelMapper pmapper(*meshDistributor, nComponents);
solver->solveSystem(solverMatrix, *solution, *rhs, pmapper);
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
}
}
template< >
void CreatorMap< Parallel::ParallelPreconditioner >::addDefaultCreators() {
Parallel::ParallelPreconditionCreator *creator;
creator = new Parallel::DiagonalPreconditioner::Creator;
addCreator("diag", creator);
creator = new Parallel::ILUPreconditioner::Creator;
addCreator("ilu", creator);
creator = new Parallel::ICPreconditioner::Creator;
addCreator("ic", creator);
creator = new Parallel::IdentityPreconditioner::Creator;
addCreator("no", creator);
creator = new Parallel::DiagonalPreconditioner::Creator;
addCreator("diag", creator);
creator = new Parallel::ILUPreconditioner::Creator;
addCreator("ilu", creator);
creator = new Parallel::ICPreconditioner::Creator;
addCreator("ic", creator);
creator = new Parallel::IdentityPreconditioner::Creator;
addCreator("no", creator);
}
}
......@@ -13,6 +13,7 @@
#include "parallel/ParallelProblemStatBase.h"
#include "parallel/MeshDistributor.h"
#include "parallel/MpiHelper.h"
#include "Global.h"
namespace AMDiS {
......@@ -22,7 +23,9 @@ namespace AMDiS {
{
FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
meshDistributor->checkMeshChange();
TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
("Should not happen!\n");
MeshDistributor::globalMeshDistributor->checkMeshChange();
ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag,
assembleMatrix,
assembleVector);
......@@ -39,47 +42,17 @@ namespace AMDiS {
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
}
void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
{
meshDistributor = &m;
m.addProblemStat(this);
}
void ParallelProblemStatBase::processMemUsage(double& vm_usage,
double& resident_set)
void ParallelProblemStatBase::initialize(Flag initFlag,
ProblemStatSeq *adoptProblem,
Flag adoptFlag)
{
using std::ios_base;
using std::ifstream;
using std::string;
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;
ProblemStatSeq::initialize(initFlag, adoptProblem, adoptFlag);
MeshDistributor::addProblemStatGlobal(this);
// in case x86-64 is configured to use 2MB pages
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
meshDistributor = MeshDistributor::globalMeshDistributor;
}
......
......@@ -43,13 +43,12 @@ namespace AMDiS {
bool assembleMatrix,
bool assembleVector);
virtual void addToMeshDistributor(MeshDistributor& m);
void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
protected:
void processMemUsage(double& vm_usage, double& resident_set);
protected:
MeshDistributor *meshDistributor;
MeshDistributor *meshDistributor;
};
}
......
......@@ -15,12 +15,52 @@
#include "parallel/PetscProblemStat.h"
#include "parallel/PetscSolver.h"
#include "parallel/MpiHelper.h"
namespace AMDiS {
using namespace std;
PetscProblemStat::PetscProblemStat(string nameStr,
ProblemIterationInterface *problemIteration)
: ParallelProblemStatBase(nameStr, problemIteration)
{
FUNCNAME("PetscProblemStat::PetscProblemStat()");
string name("");
Parameters::get("parallel->solver", name);
if (name == "petsc-schur") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverSchur();
#else
ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc-feti") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverFeti();
#else
ERROR_EXIT("PETSc FETI-DP solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc" || name == "") {
petscSolver = new PetscSolverGlobalMatrix();
} else {
ERROR_EXIT("No parallel solver %s available!\n", name.c_str());
}
}
void PetscProblemStat::initialize(Flag initFlag,
ProblemStatSeq* adoptProblem,
Flag adoptFlag)
{
ParallelProblemStatBase::initialize(initFlag, adoptProblem, adoptFlag);
meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
}
void PetscProblemStat::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
{
FUNCNAME("PetscProblemStat::solve()");
......@@ -29,10 +69,49 @@ namespace AMDiS {
double wtime = MPI::Wtime();
double vm, rss;
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 1\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
petscSolver->setMeshDistributor(meshDistributor);
petscSolver->fillPetscMatrix(systemMatrix, rhs);
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 2\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
petscSolver->solvePetscMatrix(*solution, adaptInfo);
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 3\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
MPI::Wtime() - wtime);
}
......
......@@ -36,43 +36,17 @@ namespace AMDiS {
class PetscProblemStat : public ParallelProblemStatBase
{
public:
PetscProblemStat(std::string nameStr,
ProblemIterationInterface *problemIteration = NULL)
: ParallelProblemStatBase(nameStr, problemIteration)
{
string name("");
Parameters::get("parallel->solver", name);
if (name == "petsc-schur") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverSchur();
#else
ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc-feti") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverFeti();
#else
ERROR_EXIT("PETSc FETI-DP solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc" || name == "") {
petscSolver = new PetscSolverGlobalMatrix();
} else {
ERROR_EXIT("No parallel solver %s available!\n", name.c_str());
}
}
PetscProblemStat(std::string nameStr,
ProblemIterationInterface *problemIteration = NULL);
~PetscProblemStat()
{
delete petscSolver;
}
void addToMeshDistributor(MeshDistributor& m)
{
ParallelProblemStatBase::addToMeshDistributor(m);
meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
}
void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
void solve(AdaptInfo *adaptInfo, bool fixedMatrix = false);