* Parallel file i/o

AMDiS/src/AdaptInstationary.cc

@@ -204,7 +204,21 @@ namespace AMDiS {
       iterationTimestamp_ = time(NULL);
 
       problemTime_->initTimestep(adaptInfo_);
+
+#ifdef _OPENMP
+#pragma omp parallel sections
+      {
+#pragma omp section
+	problemTime_->startDelayedTimestepCalculation();
+
+#pragma omp section
 	oneTimestep();
+      }
+#else
+      problemTime_->startDelayedTimestepCalculation();
+      oneTimestep();
+#endif
+
       problemTime_->closeTimestep(adaptInfo_);
 
       if (breakWhenStable && (adaptInfo_->getSolverIterations() == 0)) {
@@ -220,6 +234,8 @@ namespace AMDiS {
       }
     }
 
+    problemTime_->startDelayedTimestepCalculation();
+
     return errorCode;
   }
 

AMDiS/src/DataCollector.cc

@@ -56,6 +56,19 @@ namespace AMDiS {
     DELETE dofCoords_;
   }
 
+  void DataCollector::fillAllData()
+  {
+    if (!elementDataCollected_) {
+      startCollectingElementData();
+    }
+    if (!periodicDataCollected_) {
+      startCollectingPeriodicData();
+    }
+    if (!valueDataCollected_) {
+      startCollectingValueData();
+    }
+  }
+
   int DataCollector::startCollectingElementData()
   {
     FUNCNAME("DataCollector::startCollectingElementData()");
@@ -427,7 +440,7 @@ namespace AMDiS {
 
   ::std::list<PeriodicInfo>* DataCollector::getPeriodicInfos()
   {
-    if(!periodicDataCollected_) {
+    if (!periodicDataCollected_) {
       startCollectingPeriodicData();
     }
 
@@ -463,7 +476,7 @@ namespace AMDiS {
   
   int DataCollector::getNumberConnections()
   {
-    if(!periodicDataCollected_) {
+    if (!periodicDataCollected_) {
       startCollectingPeriodicData();
     }
 

AMDiS/src/DataCollector.h

@@ -61,6 +61,11 @@ namespace AMDiS {
 
       ~DataCollector();
 
+      /** \brief
+       * Fills the DataCollector with all possible datas.
+       */
+      void fillAllData();
+
       /** \brief
        * Returns list with element information.
        */

AMDiS/src/DuneSolver.cc

@@ -71,6 +71,13 @@ namespace AMDiS {
   {
     FUNCNAME("DuneSolver::solveSystem()");
 
+    // Calculate residual reduction which should be achieved by the dune solver.
+    *p = *x;
+    *p *= -1.0;
+    mv->matVec(NoTranspose, *p, *r);
+    *r += *b;
+    double reduction = this->tolerance / norm(r);
+
     typedef Dune::Matrix<DuneMatrix> SystemDuneMatrix;
     typedef Dune::BlockVector<DuneVector> SystemDuneVector;
 
@@ -117,7 +124,7 @@ namespace AMDiS {
     Dune::InverseOperator<SystemDuneVector, SystemDuneVector> *duneSolver =
       getSolver<SystemDuneMatrix, SystemDuneVector>(&op, dunePreconditioner);
 
-    duneSolver->apply(duneVecX, duneVecB, opRes);
+    duneSolver->apply(duneVecX, duneVecB, reduction, opRes);
 
     for (int i = 0; i < nComponents; i++) {
       DuneVector::Iterator duneVecIt;    
@@ -129,6 +136,16 @@ namespace AMDiS {
       }
     }
 
+    // Calculate and print the residual.
+    *p = *x;
+    *p *= -1.0;
+    mv->matVec(NoTranspose, *p, *r);
+    *r += *b;
+    
+    this->residual = norm(r);
+
+    MSG("Residual: %e\n", this->residual);
+
     delete dunePreconditioner;
     delete duneSolver;
    

AMDiS/src/DuneSolver.h

@@ -123,12 +123,20 @@ namespace AMDiS {
     /** \brief
      * Implements OEMSolver<VectorType>::init().
      */
-    void init() {};
+    void init() 
+    {
+      p = this->vectorCreator->create();
+      r = this->vectorCreator->create();
+    };
     
     /** \brief
      * Implements OEMSolver<VectorType>::exit().
      */
-    void exit() {};
+    void exit() 
+    {
+      this->vectorCreator->free(p);
+      this->vectorCreator->free(r);
+    };
 
     /** \brief
      * Creates a dune preconditioner using the informations in the used init file. 
@@ -168,6 +176,12 @@ namespace AMDiS {
      * to be allocated before.
      */
     void mapDOFVector(DOFVector<double> *dofVector, DuneVector *duneVector);
+
+  private:
+    /** \brief
+     * These vectors are justed to calculate the final residual of the solution.
+     */
+    VectorType *r, *p;
   };
 }
 

AMDiS/src/ElementFileWriter.h

@@ -30,6 +30,8 @@ namespace AMDiS {
 		    Flag traverseFlag = Mesh::CALL_LEAF_EL,
 		    bool (*writeElem)(ElInfo*) = NULL);
 
+    void writeDelayedFiles() {};
+
   protected:
     /**
      * Writes element data in tecplot format.

AMDiS/src/FileWriter.cc

@@ -5,7 +5,6 @@
 #include "ValueWriter.h"
 #include "MacroWriter.h"
 #include "VtkWriter.h"
-#include "DataCollector.h"
 #include "FiniteElemSpace.h"
 #include "AdaptInfo.h"
 #include "Flag.h"
@@ -28,6 +27,8 @@ namespace AMDiS {
 
     solutionVecs_.resize(1);
     solutionVecs_[0] = vec;
+
+    dataCollectors_.resize(1);
   }
 
 
@@ -48,6 +49,8 @@ namespace AMDiS {
 
     feSpace = vecs[0]->getFESpace();
     solutionVecs_ = vecs;
+
+    dataCollectors_.resize(vecs.size());
   }
 
 
@@ -79,6 +82,8 @@ namespace AMDiS {
     }
 
     feSpace = vec->getFESpace();
+
+    dataCollectors_.resize(nTmpSolutions_);
   }
 
 
@@ -91,6 +96,12 @@ namespace AMDiS {
         DELETE solutionVecs_[i]; 
       }
     }
+
+    for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+      if (dataCollectors_[i]) {
+	DELETE dataCollectors_[i];
+      }
+    }
   }
   
 
@@ -111,6 +122,9 @@ namespace AMDiS {
     indexDecimals = 3;
     tsModulo = 1;
     nTmpSolutions_ = 0;
+    delayWriting_ = 0;
+    writingIsDelayed_ = false;
+    delayedFilename_ = "";
     paraViewAnimationFrames_.resize(0);
 
     readParameters();
@@ -135,6 +149,7 @@ namespace AMDiS {
     GET_PARAMETER(0, name + "->index length", "%d", &indexLength);
     GET_PARAMETER(0, name + "->index decimals", "%d", &indexDecimals);
     GET_PARAMETER(0, name + "->write every i-th timestep", "%d", &tsModulo);
+    GET_PARAMETER(0, name + "->delay", "%d", &delayWriting_);
   }
 
   void FileWriter::writeFiles(AdaptInfo *adaptInfo,
@@ -148,6 +163,22 @@ namespace AMDiS {
     if ((adaptInfo->getTimestepNumber() % tsModulo != 0) && !force) 
       return;
 
+    if (writingIsDelayed_) {
+      ERROR_EXIT("This should not happen!\n");
+    }
+
+    if (writeElem) {
+      for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+	dataCollectors_[i] = NEW DataCollector(feSpace, solutionVecs_[i], 
+				  level, flag, writeElem);
+      }
+    } else {
+      for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+	dataCollectors_[i] = NEW DataCollector(feSpace, solutionVecs_[i], 
+				  traverseLevel, flag | traverseFlag, writeElement);
+      }
+    }
+
     ::std::string fn = filename;
 
     if (appendIndex) {
@@ -156,30 +187,28 @@ namespace AMDiS {
       TEST_EXIT(indexDecimals < indexLength)("index length <= index decimals\n");
     
       char formatStr[9];
-
-      sprintf(formatStr, "%%0%d.%df", indexLength, indexDecimals);
-
       char timeStr[20];
 
+      sprintf(formatStr, "%%0%d.%df", indexLength, indexDecimals);
       sprintf(timeStr, formatStr, adaptInfo ? adaptInfo->getTime() : 0.0);
 
       fn += timeStr;
     }
 
-    ::std::vector< DataCollector* > dc(solutionVecs_.size());
 
-    if (writeElem) {
-      for (int i = 0; i < static_cast<int>(dc.size()); i++) {
-	dc[i] = NEW DataCollector(feSpace, solutionVecs_[i], 
-				  level, flag, writeElem);
-      }
-    } else {
-      for (int i = 0; i < static_cast<int>(dc.size()); i++) {
-	dc[i] = NEW DataCollector(feSpace, solutionVecs_[i], 
-				  traverseLevel, flag | traverseFlag, writeElement);
+    if (delayWriting_) {
+      if (writeTecPlotFormat || writeAMDiSFormat || writePeriodicFormat) {
+	ERROR_EXIT("Delay writing only supported for ParaView file format!\n");
       }
+
+      for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+	dataCollectors_[i]->fillAllData();
       }
 
+      writingIsDelayed_ = true;
+      delayedFilename_ = fn;
+      return;
+    }
  
     if (writeTecPlotFormat) {
       TecPlotWriter<DOFVector<double> >::writeValues(solutionVecs_[0], 
@@ -191,42 +220,67 @@ namespace AMDiS {
     if (writeAMDiSFormat) {
       TEST_EXIT(mesh)("no mesh\n");
 
-      MacroWriter::writeMacro(dc[0], 
-			     const_cast<char*>( (fn +  amdisMeshExt).c_str()), 
+      MacroWriter::writeMacro(dataCollectors_[0], 
+			     const_cast<char*>((fn +  amdisMeshExt).c_str()), 
 			     adaptInfo ? adaptInfo->getTime() : 0.0);
       MSG("macro file written to %s\n", (fn + amdisMeshExt).c_str());
 
 
-      ValueWriter::writeValues(dc[0],
+      ValueWriter::writeValues(dataCollectors_[0],
 			       (fn + amdisDataExt).c_str(),
 			       adaptInfo ? adaptInfo->getTime() : 0.0);
       MSG("value file written to %s\n", (fn + amdisDataExt).c_str());   
     }
 
     if (writePeriodicFormat) {
-      MacroWriter::writePeriodicFile(dc[0], 
+      MacroWriter::writePeriodicFile(dataCollectors_[0], 
 				     (fn + periodicFileExt).c_str());
       MSG("periodic file written to %s\n", (fn + periodicFileExt).c_str());
     }
     
     if (writeParaViewFormat) {
-      VtkWriter vtkWriter(&dc);
+      VtkWriter vtkWriter(&dataCollectors_);     
+      vtkWriter.writeFile(const_cast<char*>((fn + paraViewFileExt).c_str()));
 
-      vtkWriter.writeFile(const_cast<char*>( (fn + paraViewFileExt).c_str()));
       MSG("ParaView file written to %s\n", (fn + paraViewFileExt).c_str());
     }
 
     if (writeParaViewAnimation) {
-      VtkWriter vtkWriter(&dc);
-
+      VtkWriter vtkWriter(&dataCollectors_);
       vtkWriter.updateAnimationFile(fn + paraViewFileExt, 
 				    &paraViewAnimationFrames_, 
-				    const_cast<char*>( (filename + ".pvd").c_str()));
+				    const_cast<char*>((filename + ".pvd").c_str()));
+    }
+    
+
+    for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+      DELETE dataCollectors_[i];
+    }
   }
 
+  void FileWriter::writeDelayedFiles()
+  {
+    if (!writingIsDelayed_) {
+      return;
+    }
 
-    for (int i = 0; i < static_cast<int>(dc.size()); i++) {
-      DELETE dc[i];
+    if (writeParaViewFormat) {
+      VtkWriter vtkWriter(&dataCollectors_);     
+      vtkWriter.writeFile(const_cast<char*>((delayedFilename_ + paraViewFileExt).c_str()));
     }
+
+    if (writeParaViewAnimation) {
+      VtkWriter vtkWriter(&dataCollectors_);
+      vtkWriter.updateAnimationFile(delayedFilename_ + paraViewFileExt, 
+				    &paraViewAnimationFrames_, 
+				    const_cast<char*>((filename + ".pvd").c_str()));
+    }
+
+    for (int i = 0; i < static_cast<int>(dataCollectors_.size()); i++) {
+      DELETE dataCollectors_[i];
+    }
+
+    writingIsDelayed_ = false;
+    delayedFilename_ = "";
   }
 }

AMDiS/src/FileWriter.h

@@ -31,6 +31,7 @@
 #include "MemoryManager.h"
 #include "MatrixVector.h"
 #include "Mesh.h"
+#include "DataCollector.h"
 
 namespace AMDiS {
 
@@ -69,6 +70,8 @@ namespace AMDiS {
 			    Flag traverseFlag = Mesh::CALL_LEAF_EL,
 			    bool (*writeElem)(ElInfo*) = NULL) = 0;
 
+    virtual void writeDelayedFiles() = 0;
+
     void setTraverseProperties(int level, 
 			       Flag flag,
 			       bool (*writeElem)(ElInfo*))
@@ -150,6 +153,8 @@ namespace AMDiS {
 			    Flag traverseFlag = Mesh::CALL_LEAF_EL,
 			    bool (*writeElem)(ElInfo*) = NULL);
 
+    virtual void writeDelayedFiles();
+
   protected:
     /** \brief
      * Initialization of the filewriter.
@@ -272,6 +277,28 @@ namespace AMDiS {
      * stored in solutionVecs_ must be deleted in the destructor.
      */
     int nTmpSolutions_;
+
+    /** \brief
+     * Containers, which store the data to be written;
+     */
+    ::std::vector< DataCollector* > dataCollectors_;
+
+    /** \brief
+     * If set to 1, the FileWriter will delay the file writing to the future, where
+     * it can be executed in parallel with some other independent calculations.
+     */
+    int delayWriting_;
+
+    /** \brief
+     * If set to true, the FileWriter was filled with data, but the Files are currently
+     * not written to disk.
+     */
+    bool writingIsDelayed_;
+
+    /** \brief
+     * Here the filename for the file, which should be written to the next, is stored.
+     */
+    ::std::string delayedFilename_;
   };
 
 }

AMDiS/src/ProblemInstat.cc

@@ -95,7 +95,7 @@ namespace AMDiS {
       oldSolution = NEW DOFVector<double>(problemStat->getFESpace(), name + "->uOld");
       oldSolution->refineInterpol(true);
       oldSolution->setCoarsenOperation(COARSE_INTERPOL);
-      if(problemStat->getEstimator()) {
+      if (problemStat->getEstimator()) {
 	dynamic_cast<Estimator*>(problemStat->getEstimator())
 	  ->addUhOldToSystem(0, oldSolution);
       }
@@ -184,4 +184,14 @@ namespace AMDiS {
     oldSolution->copy(*(problemStat->getSolution())); 
   }
 
+  void ProblemInstatScal::startDelayedTimestepCalculation()
+  {
+    problemStat->writeDelayedFiles();
+  }
+
+  void ProblemInstatVec::startDelayedTimestepCalculation()
+  {
+    problemStat->writeDelayedFiles();
+  }
+
 }

AMDiS/src/ProblemInstat.h

@@ -121,6 +121,11 @@ namespace AMDiS {
      */