Global MeshDistributor added.

AMDiS/src/Global.cc

@@ -332,4 +332,39 @@ namespace AMDiS {
     while (clock() < endwait) {}
   }
 
+
+  void processMemUsage(double& vm_usage, double& resident_set)
+  {
+    using std::ios_base;
+    using std::ifstream;
+    using std::string;
+    
+    vm_usage     = 0.0;
+    resident_set = 0.0;
+    
+    // 'file' stat seems to give the most reliable results    
+    ifstream stat_stream("/proc/self/stat",ios_base::in);
+    
+    // dummy vars for leading entries in stat that we don't care about
+    string pid, comm, state, ppid, pgrp, session, tty_nr;
+    string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+    string utime, stime, cutime, cstime, priority, nice;
+    string O, itrealvalue, starttime;
+    
+    // the two fields we want    
+    unsigned long vsize;
+    long rss;
+    
+    stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
+		>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
+		>> utime >> stime >> cutime >> cstime >> priority >> nice
+		>> O >> itrealvalue >> starttime >> vsize >> rss;
+
+    // in case x86-64 is configured to use 2MB pages    
+    long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+    vm_usage     = vsize / 1024.0;
+    resident_set = rss * page_size_kb;
+  }
+
+
 }

AMDiS/src/Global.h

@@ -109,6 +109,8 @@ namespace AMDiS {
 
   void waitSec(int seconds);
 
+  void processMemUsage(double& vm_usage, double& resident_set);
+
   /// Content comparision of two pointers. Used e.g. for find_if
   template<typename T>
   struct comparePtrContents : public binary_function<T*, T*, bool>

AMDiS/src/ProblemStat.cc

@@ -514,21 +514,12 @@ namespace AMDiS {
       return;
     }
 
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
-
     clock_t first = clock();
     
     solver->solveSystem(solverMatrix, *solution, *rhs);
 
-#ifdef _OPENMP
-    INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("solution of discrete system needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));
-#endif
 
     adaptInfo->setSolverIterations(solver->getIterations());
     adaptInfo->setMaxSolverIterations(solver->getMaxIterations());
@@ -543,12 +534,8 @@ namespace AMDiS {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     double first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
-#endif
 #endif
 
     if (computeExactError) {
@@ -577,14 +564,9 @@ namespace AMDiS {
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("estimation of the error needed %.5f seconds\n", 
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("estimation of the error needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("estimation of the error needed %.5f seconds\n",
 		  TIME_USED(first, clock()));
-#endif
 #endif
   }
 
@@ -682,13 +664,9 @@ namespace AMDiS {
 		  MPI::Wtime() - first);
 
     first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
 #endif
-#endif
 
 
     Flag assembleFlag = 
@@ -812,14 +790,9 @@ namespace AMDiS {
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
-		  TIME_USED(first, clock()));    
-#endif
+		  TIME_USED(first, clock()));
 #endif
   }
 
@@ -829,9 +802,6 @@ namespace AMDiS {
     FUNCNAME("ProblemStat::buildAfterCoarsen()");
 
     clock_t first = clock();
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
 
     for (int i = 0; i < static_cast<int>(meshes.size()); i++)
       meshes[i]->dofCompress();
@@ -1005,14 +975,8 @@ namespace AMDiS {
     createPrecon();
 
     INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
-
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));
-#endif     
   }
 
   bool ProblemStatSeq::dualMeshTraverseRequired()
@@ -1037,10 +1001,6 @@ namespace AMDiS {
 
 
     clock_t first = clock();
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
-
     
     Flag assembleFlag = 
       flag | 
@@ -1278,13 +1238,8 @@ namespace AMDiS {
 
     INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
 
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));    
-#endif    
   }
 
 
@@ -1312,34 +1267,20 @@ namespace AMDiS {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     double first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
-#endif
 #endif
 
-    int size = static_cast<int>(fileWriters.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(static, 1)
-#endif
-    for (int i = 0; i < size; i++) {
+    for (int i = 0; i < static_cast<int>(fileWriters.size()); i++)
       fileWriters[i]->writeFiles(adaptInfo, force);
-    }
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("writeFiles needed %.5f seconds\n",
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("writeFiles needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("writeFiles needed %.5f seconds\n",
 		  TIME_USED(first, clock()));
-#endif
 #endif
   }
 

AMDiS/src/ProblemStat.h

@@ -110,7 +110,6 @@ namespace AMDiS {
 			    ProblemStatSeq *adoptProblem = NULL,
 			    Flag adoptFlag = INIT_NOTHING);
 
-
     /// Used in \ref initialize().
     virtual void createMesh();
 

AMDiS/src/parallel/MeshDistributor.cc

@@ -55,6 +55,8 @@ namespace AMDiS {
   using namespace boost::filesystem;
   using namespace std;
 
+  MeshDistributor* MeshDistributor::globalMeshDistributor = NULL;
+
   const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED              = 0X01L;
   const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS        = 0X02L;
   const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS        = 0X04L;
@@ -436,6 +438,17 @@ namespace AMDiS {
   }
 
 
+  void MeshDistributor::addProblemStatGlobal(ProblemStatSeq *probStat)
+  {
+    FUNCNAME("MeshDistributor::addProblemStatGlobal()");
+
+    if (globalMeshDistributor == NULL)
+      globalMeshDistributor = new MeshDistributor();
+
+    globalMeshDistributor->addProblemStat(probStat);
+  }
+
+
   void MeshDistributor::exitParallelization()
   {}
 

AMDiS/src/parallel/MeshDistributor.h

@@ -51,17 +51,16 @@ namespace AMDiS {
 
   class MeshDistributor
   {
-  public:
+  private:
     MeshDistributor();
-		          
+	          
     virtual ~MeshDistributor() {}
 
+  public:
     void initParallelization();
 
     void exitParallelization();
 
-    void addProblemStat(ProblemStatSeq *probStat);
-
     /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
     /// be automatically interchanged between ranks when mesh is repartitioned.
     void addInterchangeVector(DOFVector<double> *vec)
@@ -279,7 +278,15 @@ namespace AMDiS {
 
     void getAllBoundaryDofs(DofContainer& dofs);
 
+
+  public:
+    /// Adds a stationary problem to the global mesh distributor objects.
+    static void addProblemStatGlobal(ProblemStatSeq *probStat);
+
+    
   protected:
+    void addProblemStat(ProblemStatSeq *probStat);
+
     /** \brief
      * Determines the interior boundaries, i.e. boundaries between ranks, and stores
      * all information about them in \ref interiorBoundary.
@@ -590,6 +597,7 @@ namespace AMDiS {
     Flag createBoundaryDofFlag;
 
     BoundaryDofInfo boundaryDofInfo;
+
   public:
     /// The boundary DOFs are sorted by subobject entities, i.e., first all
     /// face DOFs, edge DOFs and to the last vertex DOFs will be set to
@@ -606,6 +614,8 @@ namespace AMDiS {
     /// that are owned by another rank).
     static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
 
+    static MeshDistributor *globalMeshDistributor;
+
     friend class ParallelDebug;
   };
 }

AMDiS/src/parallel/Mtl4Solver.cc

@@ -43,23 +43,6 @@ namespace AMDiS {
       CreatorMap< OEMSolver >::addCreator("pminres", creator);
     }
 
-/*    void Mtl4Solver::addPMTLPrecons() 
-    {
-       ParallelPreconditionCreator *creator;
-
-       creator =  new DiagonalPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pdiag", creator);
-
-       creator = new ILUPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pilu", creator);
-
-       creator = new ICPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pic", creator);
-
-       creator =  new IdentityPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pno", creator);
-
-    }*/
   
     void Mtl4Solver::createSolver()
     {
@@ -77,57 +60,39 @@ namespace AMDiS {
       solver->initParameters();
     }
 
-    //TODO: replace the name in the map
-    void Mtl4Solver::createPrecon()
-    {
-      /*std::string preconType("no");
-      GET_PARAMETER(0, name + "->solver->left precon", &preconType);
-      preconType = "p" + preconType ;
-
-      CreatorInterface<BasePreconditioner> *preconCreator = 
-	CreatorMap<BasePreconditioner>::getCreator(preconType);
 
-  //    solver->setLeftPrecon(preconCreator->create(solverMatrix.getMatrix()));
-      solver->setLeftPrecon(preconCreator);
-
-      preconType= "no";
-      GET_PARAMETER(0, name + "->solver->right precon", &preconType);
-      preconType = "p" + preconType;
+    void Mtl4Solver::createPrecon()
+    {}
 
-      preconCreator = CreatorMap<BasePreconditioner>::getCreator(preconType);
-  //    solver->setRightPrecon(preconCreator->create(solverMatrix.getMatrix()));
-      solver->setRightPrecon(preconCreator);*/
-    }
     
     void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix)
     {
-/*      FUNCNAME("Mtl4Solver::solve()");
-      std::string solverName("");
-      GET_PARAMETER(0, name+"->solver", &solverName);
-      TEST_EXIT(solverName != "")("need solver name");
-      OEMSolver* solver = getPMTLSolver(solverName);
-
-      delete solver;
-      */
+      FUNCNAME("Mtl4Solver::solve()");
+
+      clock_t first = clock();
+
       ParallelMapper pmapper(*meshDistributor, nComponents);
       solver->solveSystem(solverMatrix, *solution, *rhs, pmapper);
+
+      INFO(info, 8)("solution of discrete system needed %.5f seconds\n", 
+		    TIME_USED(first, clock()));    
     }
   }
 
   template< > 
   void CreatorMap< Parallel::ParallelPreconditioner >::addDefaultCreators() {
     Parallel::ParallelPreconditionCreator *creator;
-
-     creator =  new Parallel::DiagonalPreconditioner::Creator;
-     addCreator("diag", creator);
-
-     creator = new Parallel::ILUPreconditioner::Creator;
-     addCreator("ilu", creator);
-
-     creator = new Parallel::ICPreconditioner::Creator;
-     addCreator("ic", creator);
-
-     creator =  new Parallel::IdentityPreconditioner::Creator;
-     addCreator("no", creator);
+    
+    creator =  new Parallel::DiagonalPreconditioner::Creator;
+    addCreator("diag", creator);
+    
+    creator = new Parallel::ILUPreconditioner::Creator;
+    addCreator("ilu", creator);
+    
+    creator = new Parallel::ICPreconditioner::Creator;
+    addCreator("ic", creator);
+    
+    creator =  new Parallel::IdentityPreconditioner::Creator;
+    addCreator("no", creator);
   }
 }

AMDiS/src/parallel/ParallelProblemStatBase.cc

@@ -13,6 +13,7 @@
 #include "parallel/ParallelProblemStatBase.h"
 #include "parallel/MeshDistributor.h"
 #include "parallel/MpiHelper.h"
+#include "Global.h"
 
 namespace AMDiS {
 
@@ -22,7 +23,9 @@ namespace AMDiS {
   {
     FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
 
-    meshDistributor->checkMeshChange();
+    TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
+      ("Should not happen!\n");
+    MeshDistributor::globalMeshDistributor->checkMeshChange();
     ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag, 
 				      assembleMatrix, 
 				      assembleVector);
@@ -39,47 +42,17 @@ namespace AMDiS {
 
     MSG("Overall memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
   }
-  
-  
-  void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
-  {
-    meshDistributor = &m;
-    m.addProblemStat(this);
-  }
 
 
-  void ParallelProblemStatBase::processMemUsage(double& vm_usage, 
-						double& resident_set)
+  void ParallelProblemStatBase::initialize(Flag initFlag,
+					   ProblemStatSeq *adoptProblem,
+					   Flag adoptFlag)
   {
-    using std::ios_base;
-    using std::ifstream;
-    using std::string;
-    
-    vm_usage     = 0.0;
-    resident_set = 0.0;
-    
-    // 'file' stat seems to give the most reliable results    
-    ifstream stat_stream("/proc/self/stat",ios_base::in);
-    
-    // dummy vars for leading entries in stat that we don't care about
-    string pid, comm, state, ppid, pgrp, session, tty_nr;
-    string tpgid, flags, minflt, cminflt, majflt, cmajflt;
-    string utime, stime, cutime, cstime, priority, nice;
-    string O, itrealvalue, starttime;
-    
-    // the two fields we want    
-    unsigned long vsize;
-    long rss;
-    
-    stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
-		>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
-		>> utime >> stime >> cutime >> cstime >> priority >> nice
-		>> O >> itrealvalue >> starttime >> vsize >> rss;
+    ProblemStatSeq::initialize(initFlag, adoptProblem, adoptFlag);
+
+    MeshDistributor::addProblemStatGlobal(this);
 
-    // in case x86-64 is configured to use 2MB pages    
-    long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
-    vm_usage     = vsize / 1024.0;
-    resident_set = rss * page_size_kb;
+    meshDistributor = MeshDistributor::globalMeshDistributor;
   }
 
   

AMDiS/src/parallel/ParallelProblemStatBase.h

@@ -43,13 +43,12 @@ namespace AMDiS {
 			   bool assembleMatrix,
 			   bool assembleVector);
 
-    virtual void addToMeshDistributor(MeshDistributor& m);
+    void initialize(Flag initFlag,
+		    ProblemStatSeq *adoptProblem = NULL,
+		    Flag adoptFlag = INIT_NOTHING);
 
   protected:
-    void processMemUsage(double& vm_usage, double& resident_set);
-
-  protected:
-    MeshDistributor *meshDistributor;     
+    MeshDistributor *meshDistributor;
   };
 
 }

AMDiS/src/parallel/PetscProblemStat.cc

@@ -15,12 +15,52 @@
 
 #include "parallel/PetscProblemStat.h"
 #include "parallel/PetscSolver.h"
+#include "parallel/MpiHelper.h"
 
 namespace AMDiS {
 
   using namespace std;
 
 
+  PetscProblemStat::PetscProblemStat(string nameStr,
+				     ProblemIterationInterface *problemIteration)
+    : ParallelProblemStatBase(nameStr, problemIteration)
+  {
+    FUNCNAME("PetscProblemStat::PetscProblemStat()");
+
+    string name("");
+    Parameters::get("parallel->solver", name);
+    
+    if (name == "petsc-schur") {
+#ifdef HAVE_PETSC_DEV
+      petscSolver = new PetscSolverSchur();
+#else
+      ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
+#endif
+    } else if (name == "petsc-feti") {
+#ifdef HAVE_PETSC_DEV
+      petscSolver = new PetscSolverFeti();
+#else
+      ERROR_EXIT("PETSc FETI-DP solver is only supported when petsc-dev is used!\n");
+#endif
+    } else if (name == "petsc" || name == "") {
+      petscSolver = new PetscSolverGlobalMatrix();
+    } else {
+      ERROR_EXIT("No parallel solver %s available!\n", name.c_str());
+    }
+  }
+
+
+  void PetscProblemStat::initialize(Flag initFlag, 
+				    ProblemStatSeq* adoptProblem,
+				    Flag adoptFlag)
+  {
+      ParallelProblemStatBase::initialize(initFlag, adoptProblem, adoptFlag);
+
+      meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
+  }
+
+
   void PetscProblemStat::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
   {
     FUNCNAME("PetscProblemStat::solve()");
@@ -29,10 +69,49 @@ namespace AMDiS {
 
     double wtime = MPI::Wtime();
 
+    double vm, rss;
+    processMemUsage(vm, rss);   
+    vm /= 1024.0;
+    rss /= 1024.0;
+    
+    MSG("STAGE 1\n");
+    MSG("My memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+    
+    mpi::globalAdd(vm);
+    mpi::globalAdd(rss);
+
+    MSG("Overall memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+
     petscSolver->setMeshDistributor(meshDistributor);
     petscSolver->fillPetscMatrix(systemMatrix, rhs);
+
+
+    processMemUsage(vm, rss);   
+    vm /= 1024.0;
+    rss /= 1024.0;
+    MSG("STAGE 2\n");
+    MSG("My memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+    
+    mpi::globalAdd(vm);
+    mpi::globalAdd(rss);
+
+    MSG("Overall memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+
+
     petscSolver->solvePetscMatrix(*solution, adaptInfo);   
 
+    processMemUsage(vm, rss);   
+    vm /= 1024.0;
+    rss /= 1024.0;
+    MSG("STAGE 3\n");
+    MSG("My memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+    
+    mpi::globalAdd(vm);
+    mpi::globalAdd(rss);
+
+    MSG("Overall memory usage is VM = %.1f MB    RSS = %.1f MB\n", vm, rss);
+
+
     INFO(info, 8)("solution of discrete system needed %.5f seconds\n", 
 		  MPI::Wtime() - wtime);
   }

AMDiS/src/parallel/PetscProblemStat.h

@@ -36,43 +36,17 @@ namespace AMDiS {
   class PetscProblemStat : public ParallelProblemStatBase
   {
   public:
-    PetscProblemStat(std::string nameStr,
-		     ProblemIterationInterface *problemIteration = NULL)
-      : ParallelProblemStatBase(nameStr, problemIteration)
-    {
-      string name("");
-      Parameters::get("parallel->solver", name);
-      
-      if (name == "petsc-schur") {
-#ifdef HAVE_PETSC_DEV
-	petscSolver = new PetscSolverSchur();
-#else
-	ERROR_EXIT("PETSc schur complement solver is only supported when petsc-dev is used!\n");
-#endif
-      } else if (name == "petsc-feti") {
-#ifdef HAVE_PETSC_DEV
-	petscSolver = new PetscSolverFeti();