Global MeshDistributor added.

AMDiS/src/Global.cc

@@ -332,4 +332,39 @@ namespace AMDiS {
     while (clock() < endwait) {}
   }
 
+
+  void processMemUsage(double& vm_usage, double& resident_set)
+  {
+    using std::ios_base;
+    using std::ifstream;
+    using std::string;
+    
+    vm_usage     = 0.0;
+    resident_set = 0.0;
+    
+    // 'file' stat seems to give the most reliable results    
+    ifstream stat_stream("/proc/self/stat",ios_base::in);
+    
+    // dummy vars for leading entries in stat that we don't care about
+    string pid, comm, state, ppid, pgrp, session, tty_nr;
+    string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+    string utime, stime, cutime, cstime, priority, nice;
+    string O, itrealvalue, starttime;
+    
+    // the two fields we want    
+    unsigned long vsize;
+    long rss;
+    
+    stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
+		>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
+		>> utime >> stime >> cutime >> cstime >> priority >> nice
+		>> O >> itrealvalue >> starttime >> vsize >> rss;
+
+    // in case x86-64 is configured to use 2MB pages    
+    long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+    vm_usage     = vsize / 1024.0;
+    resident_set = rss * page_size_kb;
+  }
+
+
 }

AMDiS/src/Global.h

@@ -109,6 +109,8 @@ namespace AMDiS {
 
   void waitSec(int seconds);
 
+  void processMemUsage(double& vm_usage, double& resident_set);
+
   /// Content comparision of two pointers. Used e.g. for find_if
   template<typename T>
   struct comparePtrContents : public binary_function<T*, T*, bool>

AMDiS/src/ProblemStat.cc

@@ -514,21 +514,12 @@ namespace AMDiS {
       return;
     }
 
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
-
     clock_t first = clock();
     
     solver->solveSystem(solverMatrix, *solution, *rhs);
 
-#ifdef _OPENMP
-    INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("solution of discrete system needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));
-#endif
 
     adaptInfo->setSolverIterations(solver->getIterations());
     adaptInfo->setMaxSolverIterations(solver->getMaxIterations());
@@ -543,12 +534,8 @@ namespace AMDiS {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     double first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
-#endif
 #endif
 
     if (computeExactError) {
@@ -577,14 +564,9 @@ namespace AMDiS {
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("estimation of the error needed %.5f seconds\n", 
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("estimation of the error needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("estimation of the error needed %.5f seconds\n",
 		  TIME_USED(first, clock()));
-#endif
 #endif
   }
 
@@ -682,13 +664,9 @@ namespace AMDiS {
 		  MPI::Wtime() - first);
 
     first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
 #endif
-#endif
 
 
     Flag assembleFlag = 
@@ -812,14 +790,9 @@ namespace AMDiS {
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));
-#endif
 #endif
   }
 
@@ -829,9 +802,6 @@ namespace AMDiS {
     FUNCNAME("ProblemStat::buildAfterCoarsen()");
 
     clock_t first = clock();
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
 
     for (int i = 0; i < static_cast<int>(meshes.size()); i++)
       meshes[i]->dofCompress();
@@ -1005,14 +975,8 @@ namespace AMDiS {
     createPrecon();
 
     INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
-
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));
-#endif     
   }
 
   bool ProblemStatSeq::dualMeshTraverseRequired()
@@ -1037,10 +1001,6 @@ namespace AMDiS {
 
 
     clock_t first = clock();
-#ifdef _OPENMP
-    double wtime = omp_get_wtime();
-#endif
-
     
     Flag assembleFlag = 
       flag | 
@@ -1278,13 +1238,8 @@ namespace AMDiS {
 
     INFO(info, 8)("fillin of assembled matrix: %d\n", nnz);
 
-#ifdef _OPENMP
-    INFO(info, 8)("buildAfterCoarsen needed %.5f seconds system time / %.5f seconds wallclock time\n",
-		  TIME_USED(first, clock()), omp_get_wtime() - wtime);
-#else
     INFO(info, 8)("buildAfterCoarsen needed %.5f seconds\n", 
 		  TIME_USED(first, clock()));    
-#endif    
   }
 
 
@@ -1312,34 +1267,20 @@ namespace AMDiS {
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     double first = MPI::Wtime();
-#else
-#ifdef _OPENMP
-    double first = omp_get_wtime();
 #else
     clock_t first = clock();
-#endif
 #endif
 
-    int size = static_cast<int>(fileWriters.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(static, 1)
-#endif
-    for (int i = 0; i < size; i++) {
+    for (int i = 0; i < static_cast<int>(fileWriters.size()); i++)
       fileWriters[i]->writeFiles(adaptInfo, force);
-    }
 
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     MPI::COMM_WORLD.Barrier();
     INFO(info, 8)("writeFiles needed %.5f seconds\n",
 		  MPI::Wtime() - first);
-#else
-#ifdef _OPENMP
-    INFO(info, 8)("writeFiles needed %.5f seconds\n",
-		  omp_get_wtime() - first);
 #else
     INFO(info, 8)("writeFiles needed %.5f seconds\n",
 		  TIME_USED(first, clock()));
-#endif
 #endif
   }
 

AMDiS/src/ProblemStat.h

@@ -110,7 +110,6 @@ namespace AMDiS {
 			    ProblemStatSeq *adoptProblem = NULL,
 			    Flag adoptFlag = INIT_NOTHING);
 
-
     /// Used in \ref initialize().
     virtual void createMesh();
 

AMDiS/src/parallel/MeshDistributor.cc

@@ -55,6 +55,8 @@ namespace AMDiS {
   using namespace boost::filesystem;
   using namespace std;
 
+  MeshDistributor* MeshDistributor::globalMeshDistributor = NULL;
+
   const Flag MeshDistributor::BOUNDARY_SUBOBJ_SORTED              = 0X01L;
   const Flag MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS        = 0X02L;
   const Flag MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS        = 0X04L;
@@ -436,6 +438,17 @@ namespace AMDiS {
   }
 
 
+  void MeshDistributor::addProblemStatGlobal(ProblemStatSeq *probStat)
+  {
+    FUNCNAME("MeshDistributor::addProblemStatGlobal()");
+
+    if (globalMeshDistributor == NULL)
+      globalMeshDistributor = new MeshDistributor();
+
+    globalMeshDistributor->addProblemStat(probStat);
+  }
+
+
   void MeshDistributor::exitParallelization()
   {}
 

AMDiS/src/parallel/MeshDistributor.h

@@ -51,17 +51,16 @@ namespace AMDiS {
 
   class MeshDistributor
   {
-  public:
+  private:
     MeshDistributor();
 	          
     virtual ~MeshDistributor() {}
 
+  public:
     void initParallelization();
 
     void exitParallelization();
 
-    void addProblemStat(ProblemStatSeq *probStat);
-
     /// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector will
     /// be automatically interchanged between ranks when mesh is repartitioned.
     void addInterchangeVector(DOFVector<double> *vec)
@@ -279,7 +278,15 @@ namespace AMDiS {
 
     void getAllBoundaryDofs(DofContainer& dofs);
 
+
+  public:
+    /// Adds a stationary problem to the global mesh distributor objects.
+    static void addProblemStatGlobal(ProblemStatSeq *probStat);
+
+    
   protected:
+    void addProblemStat(ProblemStatSeq *probStat);
+
     /** \brief
      * Determines the interior boundaries, i.e. boundaries between ranks, and stores
      * all information about them in \ref interiorBoundary.
@@ -590,6 +597,7 @@ namespace AMDiS {
     Flag createBoundaryDofFlag;
 
     BoundaryDofInfo boundaryDofInfo;
+
   public:
     /// The boundary DOFs are sorted by subobject entities, i.e., first all
     /// face DOFs, edge DOFs and to the last vertex DOFs will be set to
@@ -606,6 +614,8 @@ namespace AMDiS {
     /// that are owned by another rank).
     static const Flag BOUNDARY_FILL_INFO_RECV_DOFS;
 
+    static MeshDistributor *globalMeshDistributor;
+
     friend class ParallelDebug;
   };
 }

AMDiS/src/parallel/Mtl4Solver.cc

@@ -43,23 +43,6 @@ namespace AMDiS {
       CreatorMap< OEMSolver >::addCreator("pminres", creator);
     }
 
-/*    void Mtl4Solver::addPMTLPrecons() 
-    {
-       ParallelPreconditionCreator *creator;
-
-       creator =  new DiagonalPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pdiag", creator);
-
-       creator = new ILUPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pilu", creator);
-
-       creator = new ICPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pic", creator);
-
-       creator =  new IdentityPreconditioner::Creator;
-       CreatorMap<ParallelPreconditioner >::addCreator("pno", creator);
-
-    }*/
   
     void Mtl4Solver::createSolver()
     {
@@ -77,40 +60,22 @@ namespace AMDiS {
       solver->initParameters();
     }
 
-    //TODO: replace the name in the map
-    void Mtl4Solver::createPrecon()
-    {
-      /*std::string preconType("no");
-      GET_PARAMETER(0, name + "->solver->left precon", &preconType);
-      preconType = "p" + preconType ;
 
-      CreatorInterface<BasePreconditioner> *preconCreator = 
-	CreatorMap<BasePreconditioner>::getCreator(preconType);
-
-  //    solver->setLeftPrecon(preconCreator->create(solverMatrix.getMatrix()));
-      solver->setLeftPrecon(preconCreator);
-
-      preconType= "no";
-      GET_PARAMETER(0, name + "->solver->right precon", &preconType);
-      preconType = "p" + preconType;
+    void Mtl4Solver::createPrecon()
+    {}
 
-      preconCreator = CreatorMap<BasePreconditioner>::getCreator(preconType);
-  //    solver->setRightPrecon(preconCreator->create(solverMatrix.getMatrix()));
-      solver->setRightPrecon(preconCreator);*/
-    }
     
     void Mtl4Solver::solve(AdaptInfo* adaptInfo, bool fixedMatrix)
     {
-/*      FUNCNAME("Mtl4Solver::solve()");
-      std::string solverName("");
-      GET_PARAMETER(0, name+"->solver", &solverName);
-      TEST_EXIT(solverName != "")("need solver name");
-      OEMSolver* solver = getPMTLSolver(solverName);
-
-      delete solver;
-      */
+      FUNCNAME("Mtl4Solver::solve()");
+
+      clock_t first = clock();
+
       ParallelMapper pmapper(*meshDistributor, nComponents);
       solver->solveSystem(solverMatrix, *solution, *rhs, pmapper);
+
+      INFO(info, 8)("solution of discrete system needed %.5f seconds\n", 
+		    TIME_USED(first, clock()));    
     }
   }
 

AMDiS/src/parallel/ParallelProblemStatBase.cc

@@ -13,6 +13,7 @@
 #include "parallel/ParallelProblemStatBase.h"
 #include "parallel/MeshDistributor.h"
 #include "parallel/MpiHelper.h"
+#include "Global.h"
 
 namespace AMDiS {
 
@@ -22,7 +23,9 @@ namespace AMDiS {
   {
     FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
 
-    meshDistributor->checkMeshChange();
+    TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
+      ("Should not happen!\n");
+    MeshDistributor::globalMeshDistributor->checkMeshChange();
     ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag, 
 				      assembleMatrix, 
 				      assembleVector);
@@ -41,45 +44,15 @@ namespace AMDiS {
   }
 
 
-  void ParallelProblemStatBase::addToMeshDistributor(MeshDistributor& m)
+  void ParallelProblemStatBase::initialize(Flag initFlag,
+					   ProblemStatSeq *adoptProblem,
+					   Flag adoptFlag)
   {
-    meshDistributor = &m;
-    m.addProblemStat(this);
-  }
-
-
-  void ParallelProblemStatBase::processMemUsage(double& vm_usage, 
-						double& resident_set)
-  {
-    using std::ios_base;
-    using std::ifstream;
-    using std::string;
-    
-    vm_usage     = 0.0;
-    resident_set = 0.0;
-    
-    // 'file' stat seems to give the most reliable results    
-    ifstream stat_stream("/proc/self/stat",ios_base::in);
-    
-    // dummy vars for leading entries in stat that we don't care about
-    string pid, comm, state, ppid, pgrp, session, tty_nr;
-    string tpgid, flags, minflt, cminflt, majflt, cmajflt;
-    string utime, stime, cutime, cstime, priority, nice;
-    string O, itrealvalue, starttime;
-    
-    // the two fields we want    
-    unsigned long vsize;
-    long rss;
+    ProblemStatSeq::initialize(initFlag, adoptProblem, adoptFlag);
 
-    stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
-		>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
-		>> utime >> stime >> cutime >> cstime >> priority >> nice
-		>> O >> itrealvalue >> starttime >> vsize >> rss;
+    MeshDistributor::addProblemStatGlobal(this);
 
-    // in case x86-64 is configured to use 2MB pages    
-    long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
-    vm_usage     = vsize / 1024.0;
-    resident_set = rss * page_size_kb;
+    meshDistributor = MeshDistributor::globalMeshDistributor;
   }
 
   

AMDiS/src/parallel/ParallelProblemStatBase.h

@@ -43,10 +43,9 @@ namespace AMDiS {
 			   bool assembleMatrix,
 			   bool assembleVector);
 
-    virtual void addToMeshDistributor(MeshDistributor& m);
-
-  protected:
-    void processMemUsage(double& vm_usage, double& resident_set);
+    void initialize(Flag initFlag,
+		    ProblemStatSeq *adoptProblem = NULL,
+		    Flag adoptFlag = INIT_NOTHING);
 
   protected:
     MeshDistributor *meshDistributor;

AMDiS/src/parallel/PetscProblemStat.cc

@@ -15,12 +15,52 @@
 
 #include "parallel/PetscProblemStat.h"