Parallel paraview output.

AMDiS/src/FileWriter.cc

@@ -29,8 +29,8 @@ namespace AMDiS {
 
     feSpace = vec->getFESpace();
 
-    solutionVecs_.resize(1);
-    solutionVecs_[0] = vec;
+    solutionVecs.resize(1);
+    solutionVecs[0] = vec;
   }
 
 
@@ -49,7 +49,7 @@ namespace AMDiS {
 	("All FESpace have to be equal!\n");
 
     feSpace = vecs[0]->getFESpace();
-    solutionVecs_ = vecs;
+    solutionVecs = vecs;
   }
 
 
@@ -64,21 +64,18 @@ namespace AMDiS {
     initialize();
     
     // Create the temporal DOFVectors for all components of WorldVector.
-    nTmpSolutions_ = (*vec)[0].getSize();
-    solutionVecs_.resize(nTmpSolutions_);
-    for (int i = 0; i < nTmpSolutions_; i++) {
-      solutionVecs_[i] = new DOFVector<double>(vec->getFESpace(), "");
-    } 
+    nTmpSolutions = (*vec)[0].getSize();
+    solutionVecs.resize(nTmpSolutions);
+    for (int i = 0; i < nTmpSolutions; i++)
+      solutionVecs[i] = new DOFVector<double>(vec->getFESpace(), "");
 
     // Copy the components of the WorldVectors to different DOFVectors
     // of double values.
     DOFVector< WorldVector<double> >::Iterator it(vec, USED_DOFS);
     int counter = 0;
-    for (it.reset(); !it.end(); ++it, counter++) {
-      for (int i = 0; i < nTmpSolutions_; i++) {
-	(*solutionVecs_[i])[counter] = (*it)[i];
-      }
-    }
+    for (it.reset(); !it.end(); ++it, counter++)
+      for (int i = 0; i < nTmpSolutions; i++)
+	(*solutionVecs[i])[counter] = (*it)[i];
 
     feSpace = vec->getFESpace();
   }
@@ -88,9 +85,9 @@ namespace AMDiS {
   {
     // Do not forget to delete temporal solution vector, if there have been
     // some created in the constructor.
-    if (nTmpSolutions_ > 0)
-      for (int i = 0; i < nTmpSolutions_; i++)
-        delete solutionVecs_[i]; 
+    if (nTmpSolutions > 0)
+      for (int i = 0; i < nTmpSolutions; i++)
+        delete solutionVecs[i]; 
   }
   
 
@@ -99,7 +96,8 @@ namespace AMDiS {
     tecplotExt = ".tec";
     amdisMeshExt = ".mesh";
     amdisDataExt = ".dat";
-    paraViewFileExt = ".vtu";
+    paraviewFileExt = ".vtu";
+    paraviewParallelFileExt = ".pvtu";
     periodicFileExt = ".per";
     writeTecPlotFormat = 0;
     writeAMDiSFormat = 0;
@@ -112,8 +110,8 @@ namespace AMDiS {
     indexLength = 5;
     indexDecimals = 3;
     tsModulo = 1;
-    nTmpSolutions_ = 0;
-    paraViewAnimationFrames_.resize(0),
+    nTmpSolutions = 0;
+    paraviewAnimationFrames.resize(0),
     compression = NONE;
 
     readParameters();
@@ -131,7 +129,7 @@ namespace AMDiS {
     GET_PARAMETER(0, name + "->AMDiS data ext", &amdisDataExt);
     GET_PARAMETER(0, name + "->ParaView format", "%d", &writeParaViewFormat);
     GET_PARAMETER(0, name + "->ParaView animation", "%d", &writeParaViewAnimation);
-    GET_PARAMETER(0, name + "->ParaView ext", &paraViewFileExt);    
+    GET_PARAMETER(0, name + "->ParaView ext", &paraviewFileExt);    
     GET_PARAMETER(0, name + "->Periodic format", "%d", &writePeriodicFormat);
     GET_PARAMETER(0, name + "->Periodic ext", &periodicFileExt);
     GET_PARAMETER(0, name + "->PNG format", "%d", &writePngFormat);
@@ -162,15 +160,15 @@ namespace AMDiS {
       return;
 
     // Containers, which store the data to be written;
-    std::vector< DataCollector* > dataCollectors(solutionVecs_.size());
+    std::vector< DataCollector* > dataCollectors(solutionVecs.size());
 
     if (writeElem) {
       for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++)
-	dataCollectors[i] = new DataCollector(feSpace, solutionVecs_[i], 
+	dataCollectors[i] = new DataCollector(feSpace, solutionVecs[i], 
 					      level, flag, writeElem);
     } else {
       for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++)
-	dataCollectors[i] = new DataCollector(feSpace, solutionVecs_[i], 
+	dataCollectors[i] = new DataCollector(feSpace, solutionVecs[i], 
 					      traverseLevel, 
 					      flag | traverseFlag, 
 					      writeElement);
@@ -179,7 +177,9 @@ namespace AMDiS {
     std::string fn = filename;
 
 #if HAVE_PARALLEL_DOMAIN_AMDIS
-    char f[10];
+    std::string paraFilename = fn;
+    std::string postfix = "";
+    char f[15];
     sprintf(f, "-p%d-", MPI::COMM_WORLD.Get_rank());
     fn += f;
 #endif
@@ -196,12 +196,17 @@ namespace AMDiS {
       sprintf(timeStr, formatStr, adaptInfo ? adaptInfo->getTime() : 0.0);
 
       fn += timeStr;
+#if HAVE_PARALLEL_DOMAIN_AMDIS
+      paraFilename += timeStr;
+      postfix += timeStr;
+      postfix += paraviewFileExt;
+#endif
     }
 
     if (writeTecPlotFormat) {
-      TecPlotWriter<DOFVector<double> >::writeValues(solutionVecs_[0], 
+      TecPlotWriter<DOFVector<double> >::writeValues(solutionVecs[0], 
 						     const_cast<char*>((fn + tecplotExt).c_str()), 
-						     solutionVecs_[0]->getName().c_str());
+						     solutionVecs[0]->getName().c_str());
       MSG("TecPlot file written to %s\n", (fn + tecplotExt).c_str());
     }
 
@@ -229,16 +234,33 @@ namespace AMDiS {
     if (writeParaViewFormat) {
       VtkWriter vtkWriter(&dataCollectors);     
       vtkWriter.setCompression(compression);
-      vtkWriter.writeFile(fn + paraViewFileExt);      
+      vtkWriter.writeFile(fn + paraviewFileExt);      
+
+#if HAVE_PARALLEL_DOMAIN_AMDIS
+      if (MPI::COMM_WORLD.Get_rank() == 0)
+	vtkWriter.writeParallelFile(paraFilename + paraviewParallelFileExt,
+				    MPI::COMM_WORLD.Get_size(),
+				    filename, postfix);
+#endif
 
-      MSG("ParaView file written to %s\n", (fn + paraViewFileExt).c_str());
+      MSG("ParaView file written to %s\n", (fn + paraviewFileExt).c_str());
     }
 
     if (writeParaViewAnimation) {
+#if HAVE_PARALLEL_DOMAIN_AMDIS
+      if (MPI::COMM_WORLD.Get_rank() == 0) {
+	VtkWriter vtkWriter(&dataCollectors);
+	vtkWriter.updateAnimationFile(paraFilename + paraviewParallelFileExt, 
+				      &paraviewAnimationFrames,
+				      filename + ".pvd");
+
+      }
+#else
       VtkWriter vtkWriter(&dataCollectors);
-      vtkWriter.updateAnimationFile(fn + paraViewFileExt, 
-				    &paraViewAnimationFrames_, 
+      vtkWriter.updateAnimationFile(fn + paraviewFileExt, 
+				    &paraviewAnimationFrames,
 				    filename + ".pvd");
+#endif
     }
 
     if (writePngFormat) {
@@ -249,9 +271,8 @@ namespace AMDiS {
     }
     
 
-    for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++) {
+    for (int i = 0; i < static_cast<int>(dataCollectors.size()); i++)
       delete dataCollectors[i];
-    }
   }
 
 }

AMDiS/src/FileWriter.h

@@ -28,22 +28,12 @@
 
 #include <vector>
 #include <string>
-#include "MatrixVector.h"
+#include "AMDiS_fwd.h"
 #include "Mesh.h"
 #include "DataCollector.h"
 
 namespace AMDiS {
 
-  class ProblemScal;
-  class ProblemInstat;
-  class Mesh;
-  class SystemVector;
-  class FiniteElemSpace;
-  class AdaptInfo;
-
-  template<typename T> class DOFVector;
-
-
   typedef enum {
     NONE = 0,
     GZIP = 1,
@@ -158,7 +148,10 @@ namespace AMDiS {
     std::string amdisDataExt;
 
     /// VTK file extension.
-    std::string paraViewFileExt;
+    std::string paraviewFileExt;
+
+    /// Parallel VTK file extension.
+    std::string paraviewParallelFileExt;
 
     /// Periodic file extension.
     std::string periodicFileExt;
@@ -200,7 +193,7 @@ namespace AMDiS {
     int tsModulo;
 
     /// Stores all writen filename to a ParaView animation file.
-    std::vector< std::string > paraViewAnimationFrames_;
+    std::vector< std::string > paraviewAnimationFrames;
 
     ///
     int timestepNumber;
@@ -212,14 +205,14 @@ namespace AMDiS {
     const FiniteElemSpace *feSpace;
 
     /// Pointers to the vectors which store the solution.
-    std::vector< DOFVector<double>* > solutionVecs_;
+    std::vector< DOFVector<double>* > solutionVecs;
 
     /** \brief
      * Stores the number of temporal solutions vectors, which have been created
      * in the constructor. If this number is greater than zero, the vectors
      * stored in solutionVecs_ must be deleted in the destructor.
      */
-    int nTmpSolutions_;
+    int nTmpSolutions;
 
     /** \brief
      * Defines if, and with what kind of compression, the file should be compressed

AMDiS/src/VtkWriter.cc

 #include <stdio.h>
 #include <string>
 #include <fstream>
+#include <sstream>
 #include <cmath>
 
+#include "boost/filesystem.hpp" 
+
 #include "VtkWriter.h"
 #include "DataCollector.h"
 #include "DOFVector.h"
@@ -44,6 +47,49 @@ namespace AMDiS {
     return 0;
   }
 
+
+  void VtkWriter::writeParallelFile(std::string name, int nRanks,
+				    std::string fnPrefix, std::string fnPostfix)
+  {
+    std::ofstream file;
+    file.open(name.c_str());
+
+    file << "<?xml version=\"1.0\"?>\n";
+    file << "<VTKFile type=\"PUnstructuredGrid\">\n";
+    file << "  <PUnstructuredGrid GhostLevel=\"0\">\n";
+    file << "    <PPoints>\n"
+	 << "      <PDataArray type=\"Float32\" NumberOfComponents=\"3\" format=\"ascii\"/>\n"
+	 << "    </PPoints>\n";
+    file << "    <PCells>\n"
+	 << "      <PDataArray type=\"Int32\" Name=\"offsets\"/>\n"
+	 << "      <PDataArray type=\"UInt8\" Name=\"types\"/>\n"
+	 << "      <PDataArray type=\"Int32\" Name=\"connectivity\"/>\n"
+	 << "    </PCells>\n";
+    file << "    <PPointData>\n";
+
+    for (int i = 0; i < static_cast<int>(dataCollector->size()); i++)
+      file << "      <PDataArray type=\"Float32\" Name=\"value" 
+	   << i << "\" format=\"ascii\"/>\n";
+
+    file << "    </PPointData>\n";
+
+    for (int i = 0; i < nRanks; i++) {
+      std::stringstream oss;
+      oss << fnPrefix << "-p" << i << "-" << fnPostfix;
+      boost::filesystem::path filepath(oss.str());
+      file << "    <Piece Source=\"" 
+	   << boost::filesystem::basename(filepath)
+	   << boost::filesystem::extension(filepath) << "\"/>\n";
+      
+    }
+
+    file << "  </PUnstructuredGrid>\n";
+    file << "</VTKFile>\n";
+
+    file.close();
+  }
+
+
   int VtkWriter::updateAnimationFile(std::string valueFilename,
 				     std::vector< std::string > *paraViewAnimationFrames,
 				     std::string animationFilename)

AMDiS/src/VtkWriter.h

@@ -50,6 +50,10 @@ namespace AMDiS {
     /// Writes a ParaView-VTK file.
     int writeFile(std::string name);
 
+    /// Writes a pvtu file, which contains the links to all the rank files.
+    void writeParallelFile(std::string name, int nRanks,
+			   std::string fnPrefix, std::string fnPostfix);
+
     /// May be used to simply write ParaView files.
     static void writeFile(DOFVector<double> *values, std::string filename);
 

AMDiS/src/VtkWriter.hh

@@ -36,9 +36,8 @@ namespace AMDiS {
     file << "      <Cells>\n";
     file << "        <DataArray type=\"Int32\" Name=\"offsets\">\n";
     
-    for (int i = 0; i < nElements; i++) {
+    for (int i = 0; i < nElements; i++)
       file << " " << (i + 1) * vertices << "\n";
-    }
 
     file << "        </DataArray>\n";
     file << "        <DataArray type=\"UInt8\" Name=\"types\">\n";
@@ -119,24 +118,10 @@ namespace AMDiS {
   template<typename T>
   void VtkWriter::writeVertexValues(T &file, int componentNo)
   {
-    //    DOFVector<int> *interpPointInd;
-    //    DOFVector<double> *values;
-    //    DOFVector< std::list<WorldVector<double> > > *dofCoords;
-
     DOFVector<int> *interpPointInd = (*dataCollector)[componentNo]->getInterpPointInd();
     DOFVector<double> *values = (*dataCollector)[componentNo]->getValues();
     DOFVector< std::list<WorldVector<double> > > *dofCoords = (*dataCollector)[componentNo]->getDofCoords();
 
-    /*
-#ifdef _OPENMP
-#pragma omp critical 
-#endif
-    {   
-      interpPointInd = (*dataCollector)[componentNo]->getInterpPointInd();
-      values = (*dataCollector)[componentNo]->getValues();
-      dofCoords = (*dataCollector)[componentNo]->getDofCoords();
-    }
-    */
     DOFVector<int>::Iterator intPointIt(interpPointInd, USED_DOFS);
     DOFVector<double>::Iterator valueIt(values, USED_DOFS);
     DOFVector< std::list<WorldVector<double> > >::Iterator coordIt(dofCoords, USED_DOFS);