Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit 1de9db4b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Global MeshDistributor added.

parent 91edc9fa
......@@ -332,4 +332,39 @@ namespace AMDiS {
while (clock() < endwait) {}
}
void processMemUsage(double& vm_usage, double& resident_set)
{
using std::ios_base;
using std::ifstream;
using std::string;
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;
// in case x86-64 is configured to use 2MB pages
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
}
}
......@@ -109,6 +109,8 @@ namespace AMDiS {
void waitSec(int seconds);
void processMemUsage(double& vm_usage, double& resident_set);
/// Content comparision of two pointers. Used e.g. for find_if
template<typename T>
struct comparePtrContents : public binary_function<T*, T*, bool>
......
......@@ -514,21 +514,12 @@ namespace AMDiS {
return;
}
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();
solver->solveSystem(solverMatrix, *solution, *rhs);
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
adaptInfo->setSolverIterations(solver->getIterations());
adaptInfo->setMaxSolverIterations(solver->getMaxIterations());
......@@ -543,12 +534,8 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
if (computeExactError) {
......@@ -577,14 +564,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......@@ -682,13 +664,9 @@ namespace AMDiS {
MPI::Wtime() - first);
first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
Flag assembleFlag =
......@@ -812,14 +790,9 @@ namespace AMDiS {
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
TIME_USED(first, clock()));
#endif
}
......@@ -829,9 +802,6 @@ namespace AMDiS {
FUNCNAME("ProblemStat::buildAfterCoarsen()");
clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
for (int i = 0; i < static_cast<int>(meshes.size()); i++)
meshes[i]->dofCompress();
......@@ -1005,14 +975,8 @@ namespace AMDiS {
createPrecon();
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
}
bool ProblemStatSeq::dualMeshTraverseRequired()
......@@ -1037,10 +1001,6 @@ namespace AMDiS {
clock_t first = clock();
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
Flag assembleFlag =
flag |
......@@ -1278,13 +1238,8 @@ namespace AMDiS {
INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
#ifdef _OPENMP
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
#else
INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
}
......@@ -1312,34 +1267,20 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
int size = static_cast<int>(fileWriters.size());
#ifdef _OPENMP
#pragma omp parallel for schedule(static, 1)
#endif
for (int i = 0; i < size; i++) {
for (int i = 0; i < static_cast<int>(fileWriters.size()); i++)
fileWriters[i]->writeFiles(adaptInfo, force);
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
MPI::COMM_WORLD.Barrier();
INFO(info, 8)("writeFiles needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("writeFiles needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("writeFiles needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......
......@@ -110,7 +110,6 @@ namespace AMDiS {
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
/// Used in \ref initialize().
virtual void createMesh();
......
......@@ -55,6 +55,8 @@ namespace AMDiS {
using namespace boost::filesystem;
using namespace std;
MeshDistributor* MeshDistributor::globalMeshDistributor = NULL;
const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED = 0X01L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS = 0X02L;
const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS = 0X04L;
......@@ -436,6 +438,17 @@ namespace AMDiS {
}
void MeshDistributor::addProblemStatGlobal(ProblemStatSeq *probStat)
{
FUNCNAME("MeshDistributor::addProblemStatGlobal()");
if (globalMeshDistributor == NULL)
globalMeshDistributor = new MeshDistributor();
globalMeshDistributor->addProblemStat(probStat);
}
void MeshDistributor::exitParallelization()
{}
......
......@@ -51,17 +51,16 @@ namespace AMDiS {
class MeshDistributor
{
public:
private:
MeshDistributor();
virtual ~MeshDistributor() {}
public:
void initParallelization();
void exitParallelization();
void addProblemStat(ProblemStatSeq *probStat);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
/// be automatically interchanged between ranks when mesh is repartitioned.
void addInterchangeVector(DOFVector<double> *vec)
......@@ -279,7 +278,15 @@ namespace AMDiS {
void getAllBoundaryDofs(DofContainer& dofs);
public:
/// Adds a stationary problem to the global mesh distributor objects.
static void addProblemStatGlobal(ProblemStatSeq *probStat);
protected:
void addProblemStat(ProblemStatSeq *probStat);
/** \brief
* Determines the interior boundaries, i.e. boundaries between ranks, and stores
* all information about them in \ref interiorBoundary.
......@@ -590,6 +597,7 @@ namespace AMDiS {
Flag createBoundaryDofFlag;
BoundaryDofInfo boundaryDofInfo;
public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
......@@ -606,6 +614,8 @@ namespace AMDiS {
/// that are owned by another rank).
static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
static MeshDistributor *globalMeshDistributor;
friend class ParallelDebug;
};
}
......
......@@ -43,23 +43,6 @@ namespace AMDiS {
CreatorMap< OEMSolver >::addCreator("pminres", creator);
}
/* void Mtl4Solver::addPMTLPrecons()
{
ParallelPreconditionCreator *creator;
creator = new DiagonalPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pdiag", creator);
creator = new ILUPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pilu", creator);
creator = new ICPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pic", creator);
creator = new IdentityPreconditioner::Creator;
CreatorMap<ParallelPreconditioner >::addCreator("pno", creator);
}*/
void Mtl4Solver::createSolver()
{
......@@ -77,57 +60,39 @@ namespace AMDiS {
solver->initParameters();
}
//TODO: replace the name in the map
void Mtl4Solver::createPrecon()
{
/*std::string preconType("no");
GET_PARAMETER(0, name + "->solver->left precon", &preconType);
preconType = "p" + preconType ;
CreatorInterface<BasePreconditioner> *preconCreator =
CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setLeftPrecon(preconCreator->create(solverMatrix.getMatrix()));
solver->setLeftPrecon(preconCreator);
preconType= "no";
GET_PARAMETER(0, name + "->solver->right precon", &preconType);
preconType = "p" + preconType;
void Mtl4Solver::createPrecon()
{}
preconCreator = CreatorMap<BasePreconditioner>::getCreator(preconType);
// solver->setRightPrecon(preconCreator->create(solverMatrix.getMatrix()));
solver->setRightPrecon(preconCreator);*/
}
void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix)
{
/* FUNCNAME("Mtl4Solver::solve()");
std::string solverName("");
GET_PARAMETER(0, name+"->solver", &solverName);
TEST_EXIT(solverName != "")("need solver name");
OEMSolver* solver = getPMTLSolver(solverName);
delete solver;
*/
FUNCNAME("Mtl4Solver::solve()");
clock_t first = clock();
ParallelMapper pmapper(*meshDistributor, nComponents);
solver->solveSystem(solverMatrix, *solution, *rhs, pmapper);
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
TIME_USED(first, clock()));
}
}
template< >
void CreatorMap< Parallel::ParallelPreconditioner >::addDefaultCreators() {
Parallel::ParallelPreconditionCreator *creator;
creator = new Parallel::DiagonalPreconditioner::Creator;
addCreator("diag", creator);
creator = new Parallel::ILUPreconditioner::Creator;
addCreator("ilu", creator);
creator = new Parallel::ICPreconditioner::Creator;
addCreator("ic", creator);
creator = new Parallel::IdentityPreconditioner::Creator;
addCreator("no", creator);
creator = new Parallel::DiagonalPreconditioner::Creator;
addCreator("diag", creator);
creator = new Parallel::ILUPreconditioner::Creator;
addCreator("ilu", creator);
creator = new Parallel::ICPreconditioner::Creator;
addCreator("ic", creator);
creator = new Parallel::IdentityPreconditioner::Creator;
addCreator("no", creator);
}
}
......@@ -13,6 +13,7 @@
#include "parallel/ParallelProblemStatBase.h"
#include "parallel/MeshDistributor.h"
#include "parallel/MpiHelper.h"
#include "Global.h"
namespace AMDiS {
......@@ -22,7 +23,9 @@ namespace AMDiS {
{
FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
meshDistributor->checkMeshChange();
TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
("Should not happen!\n");
MeshDistributor::globalMeshDistributor->checkMeshChange();
ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag,
assembleMatrix,
assembleVector);
......@@ -39,47 +42,17 @@ namespace AMDiS {
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
}
void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
{
meshDistributor = &m;
m.addProblemStat(this);
}
void ParallelProblemStatBase::processMemUsage(double& vm_usage,
double& resident_set)
void ParallelProblemStatBase::initialize(Flag initFlag,
ProblemStatSeq *adoptProblem,
Flag adoptFlag)
{
using std::ios_base;
using std::ifstream;
using std::string;
vm_usage = 0.0;
resident_set = 0.0;
// 'file' stat seems to give the most reliable results
ifstream stat_stream("/proc/self/stat",ios_base::in);
// dummy vars for leading entries in stat that we don't care about
string pid, comm, state, ppid, pgrp, session, tty_nr;
string tpgid, flags, minflt, cminflt, majflt, cmajflt;
string utime, stime, cutime, cstime, priority, nice;
string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss;
ProblemStatSeq::initialize(initFlag, adoptProblem, adoptFlag);
MeshDistributor::addProblemStatGlobal(this);
// in case x86-64 is configured to use 2MB pages
long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
vm_usage = vsize / 1024.0;
resident_set = rss * page_size_kb;
meshDistributor = MeshDistributor::globalMeshDistributor;
}
......
......@@ -43,13 +43,12 @@ namespace AMDiS {
bool assembleMatrix,
bool assembleVector);
virtual void addToMeshDistributor(MeshDistributor& m);
void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
protected:
void processMemUsage(double& vm_usage, double& resident_set);
protected:
MeshDistributor *meshDistributor;
MeshDistributor *meshDistributor;
};
}
......
......@@ -15,12 +15,52 @@
#include "parallel/PetscProblemStat.h"
#include "parallel/PetscSolver.h"
#include "parallel/MpiHelper.h"
namespace AMDiS {
using namespace std;
PetscProblemStat::PetscProblemStat(string nameStr,
ProblemIterationInterface *problemIteration)
: ParallelProblemStatBase(nameStr, problemIteration)
{
FUNCNAME("PetscProblemStat::PetscProblemStat()");
string name("");
Parameters::get("parallel->solver", name);
if (name == "petsc-schur") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverSchur();
#else
ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc-feti") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverFeti();
#else
ERROR_EXIT("PETSc FETI-DP solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc" || name == "") {
petscSolver = new PetscSolverGlobalMatrix();
} else {
ERROR_EXIT("No parallel solver %s available!\n", name.c_str());
}
}
void PetscProblemStat::initialize(Flag initFlag,
ProblemStatSeq* adoptProblem,
Flag adoptFlag)
{
ParallelProblemStatBase::initialize(initFlag, adoptProblem, adoptFlag);
meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
}
void PetscProblemStat::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
{
FUNCNAME("PetscProblemStat::solve()");
......@@ -29,10 +69,49 @@ namespace AMDiS {
double wtime = MPI::Wtime();
double vm, rss;
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 1\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
petscSolver->setMeshDistributor(meshDistributor);
petscSolver->fillPetscMatrix(systemMatrix, rhs);
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 2\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
petscSolver->solvePetscMatrix(*solution, adaptInfo);
processMemUsage(vm, rss);
vm /= 1024.0;
rss /= 1024.0;
MSG("STAGE 3\n");
MSG("My memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
mpi::globalAdd(vm);
mpi::globalAdd(rss);
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
MPI::Wtime() - wtime);
}
......
......@@ -36,43 +36,17 @@ namespace AMDiS {
class PetscProblemStat : public ParallelProblemStatBase
{
public:
PetscProblemStat(std::string nameStr,
ProblemIterationInterface *problemIteration = NULL)
: ParallelProblemStatBase(nameStr, problemIteration)
{
string name("");
Parameters::get("parallel->solver", name);
if (name == "petsc-schur") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverSchur();
#else
ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
#endif
} else if (name == "petsc-feti") {
#ifdef HAVE_PETSC_DEV
petscSolver = new PetscSolverFeti();
#else