* Compressing file output using boost

AMDiS/src/FileWriter.cc

@@ -127,6 +127,7 @@ namespace AMDiS {
     writingIsDelayed_ = false;
     delayedFilename_ = "";
     paraViewAnimationFrames_.resize(0);
+    compression = NONE;
 
     readParameters();
   }
@@ -152,9 +153,16 @@ namespace AMDiS {
     GET_PARAMETER(0, name + "->write every i-th timestep", "%d", &tsModulo);
     GET_PARAMETER(0, name + "->delay", "%d", &delayWriting_);
 
+    std::string compressionStr = "";
+    GET_PARAMETER(0, name + "->compression", &compressionStr);
+    if ((compressionStr == "gzip") || (compressionStr == "gz")) {
+      compression = GZIP;
+    } else if ((compressionStr == "bzip2") || (compressionStr == "bz2")) {
+      compression = BZIP2;
+    }
+
     TEST_EXIT(!delayWriting_ || amdisHaveOpenMP)
       ("Delayed writing only possible with OpenMP support!\n");
-
   }
 
   void FileWriter::writeFiles(AdaptInfo *adaptInfo,
@@ -248,7 +256,8 @@ namespace AMDiS {
     
     if (writeParaViewFormat) {
       VtkWriter vtkWriter(&dataCollectors_);     
-      vtkWriter.writeFile(const_cast<char*>((fn + paraViewFileExt).c_str()));
+      vtkWriter.setCompression(compression);
+      vtkWriter.writeFile(const_cast<char*>((fn + paraViewFileExt).c_str()));      
 
       MSG("ParaView file written to %s\n", (fn + paraViewFileExt).c_str());
     }

AMDiS/src/FileWriter.h

@@ -44,6 +44,13 @@ namespace AMDiS {
 
   template<typename T> class DOFVector;
 
+
+  typedef enum {
+    NONE = 0,
+    GZIP = 1,
+    BZIP2 = 2
+  } FileCompression;
+
   // ============================================================================
   // ===== class FileWriterInterface ============================================
   // ============================================================================
@@ -311,6 +318,12 @@ namespace AMDiS {
      * Here the filename for the file, which should be written to the next, is stored.
      */
     std::string delayedFilename_;
+
+    /** \brief
+     * Defines if, and with what kind of compression, the file should be compressed
+     * during writing.
+     */
+    FileCompression compression;
   };
 
 }

AMDiS/src/VtkWriter.cc

@@ -13,373 +13,34 @@ namespace AMDiS {
 
   int VtkWriter::writeFile(const char *name)
   {
-    int nVertices = (*dataCollector)[0]->getNumberVertices();
-    int nElements = (*dataCollector)[0]->getNumberElements();
-    int vertices = (*dataCollector)[0]->getMesh()->getGeo(VERTEX);
-
-    if ((dim == 2) && (degree == 2)) {
-      nVertices += (*dataCollector)[0]->getNumberInterpPoints();
-      nElements *= 4;
-    } else if ((dim == 2) && (degree == 3)) {
-      nVertices += (*dataCollector)[0]->getNumberInterpPoints();
-      nElements *= 9;
-    } else if ((dim == 2) && (degree == 4)) {
-      nVertices += (*dataCollector)[0]->getNumberInterpPoints();
-      nElements *= 16;
-    }
+#ifdef HAVE_BOOST
     
+    boost::iostreams::filtering_ostream file;
+    switch (compress) {
+    case GZIP:
+      file.push(boost::iostreams::gzip_compressor());
+      file.push(boost::iostreams::file_descriptor_sink(strcat((char*)name, ".gz")));    
+      break;
+    case BZIP2:
+      file.push(boost::iostreams::bzip2_compressor());
+      file.push(boost::iostreams::file_descriptor_sink(strcat((char*)name, ".bz2")));
+      break;
+    default:
+      file.push(boost::iostreams::file_descriptor_sink(name));
+      break;
+    } 
+    writeFile(file);
+
+#else
+
     std::ofstream file;
     file.open(name);
-    
-    file << "<?xml version=\"1.0\"?>\n";
-    file << "<VTKFile type=\"UnstructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n";
-    file << "  <UnstructuredGrid>\n";
-    file << "    <Piece NumberOfPoints=\"" << nVertices 
-	 << "\" NumberOfCells=\"" << nElements << "\">\n";
-    file << "      <Points>\n";
-    file << "        <DataArray type=\"Float32\" NumberOfComponents=\"3\" format=\"ascii\">\n";
-    
-    writeVertexCoords(file);
-    
-    file << "        </DataArray>\n";
-    file << "      </Points>\n";
-    file << "      <Cells>\n";
-    file << "        <DataArray type=\"Int32\" Name=\"offsets\">\n";
-    
-    for (int i = 0; i < nElements; i++) {
-      file << " " << (i + 1) * vertices << "\n";
-    }
-
-    file << "        </DataArray>\n";
-    file << "        <DataArray type=\"UInt8\" Name=\"types\">\n";
-
-    for (int i = 0; i < nElements; i++) {
-      switch (vertices) {
-      case 2:
-	file << " 3\n";
-	break;
-      case 3: 
-	file << " 5\n";
-	break;
-      case 4:
-	file << " 10\n";
-	break;		    
-      default:
-	break;
-      }	   
-    }
-
-    file << "        </DataArray>\n";
-    file << "        <DataArray type=\"Int32\" Name=\"connectivity\">\n";
-    
-    writeConnectivity(file);
-    
-    file << "        </DataArray>\n";    
-    file << "      </Cells>\n";
-    file << "      <PointData>\n";
-    
-    for (int i = 0; i < static_cast<int>(dataCollector->size()); i++) {
-      file << "        <DataArray type=\"Float32\" Name=\"value" << i 
-	   << "\" format=\"ascii\">\n";
-      
-      writeVertexValues(file, i);
-      
-      file << "        </DataArray>\n";
-    }
-    
-    file << "      </PointData>\n";
-    file << "    </Piece>\n";
-    file << "  </UnstructuredGrid>\n";
-    file << "</VTKFile>\n";
-    
-    
+    writeFile(file);    
     file.close();
-    
-    return 0;
-  }
-
 
-  void VtkWriter::writeVertexCoords(std::ofstream &file)
-  {
-    DOFVector< std::list<VertexInfo> > *vertexInfos = (*dataCollector)[0]->getVertexInfos();
-    DOFVector< std::list<VertexInfo> >::Iterator it(vertexInfos, USED_DOFS);
-    int counter = 0;
-
-    // For all DOFs of vertices, write the coordinates.
-    for (it.reset(); !it.end(); ++it) {
-      // for all vertex infos of this DOF
-      std::list<VertexInfo>::iterator it2;
-      for (it2 = it->begin(); it2 != it->end(); ++it2) {
-	it2->outputIndex = counter++;
-	writeCoord(file, it2->coords);
-      }
-    }
-
-    // For the second dim case, write also the interpolation points.
-    if ((dim == 2) && (degree > 1)) {
-      DOFVector< std::list< WorldVector<double> > > *interpPointCoords = (*dataCollector)[0]->getInterpPointCoords();
-      DOFVector< std::list< WorldVector<double> > >::Iterator pointIt(interpPointCoords, USED_DOFS);
-      
-      for (pointIt.reset(); !pointIt.end(); ++pointIt) {
-	std::list< WorldVector<double> >::iterator it2;
-	for (it2 = pointIt->begin(); it2 != pointIt->end(); ++it2) {
-	  writeCoord(file, *it2);
-	}
-      }
-    }
-  }
-
-
-  void VtkWriter::writeVertexValues(std::ofstream &file, int componentNo)
-  { 
-    //    DOFVector<int> *interpPointInd;
-    //    DOFVector<double> *values;
-    //    DOFVector< std::list<WorldVector<double> > > *dofCoords;
-
-    DOFVector<int> *interpPointInd = (*dataCollector)[componentNo]->getInterpPointInd();
-    DOFVector<double> *values = (*dataCollector)[componentNo]->getValues();
-    DOFVector< std::list<WorldVector<double> > > *dofCoords = (*dataCollector)[componentNo]->getDofCoords();
-
-    /*
-#ifdef _OPENMP
-#pragma omp critical 
 #endif
-    {   
-      interpPointInd = (*dataCollector)[componentNo]->getInterpPointInd();
-      values = (*dataCollector)[componentNo]->getValues();
-      dofCoords = (*dataCollector)[componentNo]->getDofCoords();
-    }
-    */
-    DOFVector<int>::Iterator intPointIt(interpPointInd, USED_DOFS);
-    DOFVector<double>::Iterator valueIt(values, USED_DOFS);
-    DOFVector< std::list<WorldVector<double> > >::Iterator coordIt(dofCoords, USED_DOFS);
-          
-    // Write the values for all vertex DOFs.
-    for (intPointIt.reset(), valueIt.reset(), coordIt.reset();
-	 !intPointIt.end(); 
-	 ++intPointIt, ++valueIt, ++coordIt) {
-
-      if (*intPointIt == -2) {
-	for (int i = 0; i < static_cast<int>(coordIt->size()); i++) {
-	  file << " " << (fabs(*valueIt) < 1e-40 ? 0.0 : *valueIt) << "\n";
-	}
-      }
-    }        
-
-    // For the second dim case, write also the values of the interpolation points.
-    if ((dim == 2) && (degree > 1)) {
-      DOFVector< std::list<WorldVector<double> > >::Iterator 
-	interpCoordIt((*dataCollector)[componentNo]->getInterpPointCoords(), USED_DOFS);
-    
-      for (intPointIt.reset(), valueIt.reset(), interpCoordIt.reset();
-	   !intPointIt.end(); 
-	   ++intPointIt, ++valueIt, ++interpCoordIt) {      
-	
-	if (*intPointIt >= 0) {
-	  for (int i = 0; i < static_cast<int>(interpCoordIt->size()); i++) {
-	    file << " " << (fabs(*valueIt) < 1e-40 ? 0.0 : *valueIt) << "\n";
-	  }
-	}
-      }
-    }    
-  }
-
-
-  void VtkWriter::writeConnectivity(std::ofstream &file) 
-  {
-    // For the second dim case, and if higher order Lagrange elements are used,
-    // write the connectivity by extra functions.
-    if ((dim == 2) && (degree == 2)) {
-      writeConnectivity_dim2_degree2(file);
-    } else if ((dim == 2) && (degree == 3)) {
-      writeConnectivity_dim2_degree3(file);
-    } else if ((dim == 2) && (degree == 4)) {
-      writeConnectivity_dim2_degree4(file);   
-    } else {
-      std::list<ElementInfo> *elements = (*dataCollector)[0]->getElementInfos();
-      std::list<ElementInfo>::iterator elementIt;
-      int vertices = (*dataCollector)[0]->getMesh()->getGeo(VERTEX);
-      
-      for (elementIt = elements->begin(); elementIt != elements->end(); ++elementIt) {
-	// for all vertices
-	for (int i = 0; i < vertices; i++) {
-	  file << " " << elementIt->vertexInfo[i]->outputIndex;
-	}
-	file << "\n";
-      } 
-    }
-  }
-
-
-  void VtkWriter::writeConnectivity_dim2_degree2(std::ofstream &file)
-  {  
-    std::list<ElementInfo> *elements = (*dataCollector)[0]->getElementInfos();
-    std::list<ElementInfo>::iterator elementIt;
-
-    std::vector< std::vector<int> > *interpPoints = (*dataCollector)[0]->getInterpPoints();
-    std::vector< std::vector<int> >::iterator pointIt;
-
-    int nVertices = (*dataCollector)[0]->getNumberVertices();
-
-    for (pointIt = interpPoints->begin(), elementIt = elements->begin(); 
-	 pointIt != interpPoints->end(); 
-	 ++pointIt, ++elementIt) {
-    
-      file << " " << elementIt->vertexInfo[0]->outputIndex
-	   << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[2] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[2]->outputIndex 
-	   << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[1] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[1]->outputIndex
-	   << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[2] + nVertices << "\n";
-
-      file << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[1] + nVertices 
-	   << " " << (*pointIt)[2] + nVertices << "\n";
-    }
-
-  }
-
-
-  void VtkWriter::writeConnectivity_dim2_degree3(std::ofstream &file)
-  {  
-    std::list<ElementInfo> *elements = (*dataCollector)[0]->getElementInfos();
-    std::list<ElementInfo>::iterator elementIt;
-
-    std::vector< std::vector<int> > *interpPoints = (*dataCollector)[0]->getInterpPoints();
-    std::vector< std::vector<int> >::iterator pointIt;
-
-    int nVertices = (*dataCollector)[0]->getNumberVertices();
-
-    for (pointIt = interpPoints->begin(), elementIt = elements->begin(); 
-	 pointIt != interpPoints->end(); 
-	 ++pointIt, ++elementIt) {
-                          
-      file << " " << elementIt->vertexInfo[0]->outputIndex
-	   << " " << (*pointIt)[3] + nVertices
-	   << " " << (*pointIt)[4] + nVertices << "\n";
-
-      file << " " << (*pointIt)[4] + nVertices
-	   << " " << (*pointIt)[5] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << (*pointIt)[3] + nVertices
-	   << " " << (*pointIt)[4] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << (*pointIt)[2] + nVertices
-	   << " " << (*pointIt)[3] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[1]->outputIndex
-	   << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[5] + nVertices << "\n";
-
-      file << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[6] + nVertices
-	   << " " << (*pointIt)[5] + nVertices << "\n";
-
-      file << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[2] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[2]->outputIndex 
-	   << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[2] + nVertices << "\n";
-     
       
-    }
-
-  }
-
-
-  void VtkWriter::writeConnectivity_dim2_degree4(std::ofstream &file)
-  {
-    std::list<ElementInfo> *elements = (*dataCollector)[0]->getElementInfos();
-    std::list<ElementInfo>::iterator elementIt;
-
-    std::vector< std::vector<int> > *interpPoints = (*dataCollector)[0]->getInterpPoints();
-    std::vector< std::vector<int> >::iterator pointIt;
-
-    int nVertices = (*dataCollector)[0]->getNumberVertices();
-
-    for (pointIt = interpPoints->begin(), elementIt = elements->begin(); 
-	 pointIt != interpPoints->end(); 
-	 ++pointIt, ++elementIt) {
-
-      file << " " << elementIt->vertexInfo[0]->outputIndex 
-	   << " " << (*pointIt)[5] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << (*pointIt)[5] + nVertices
-	   << " " << (*pointIt)[9] + nVertices
-	   << " " << (*pointIt)[6] + nVertices << "\n";
-
-      file << " " << (*pointIt)[6] + nVertices
-	   << " " << (*pointIt)[7] + nVertices
-	   << " " << (*pointIt)[9] + nVertices << "\n";
-
-      file << " " << (*pointIt)[7] + nVertices
-	   << " " << (*pointIt)[9] + nVertices
-	   << " " << (*pointIt)[10] + nVertices << "\n";
-
-      file << " " << (*pointIt)[7] + nVertices
-	   << " " << (*pointIt)[8] + nVertices
-	   << " " << (*pointIt)[10] + nVertices << "\n";
-
-      file << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[8] + nVertices
-	   << " " << (*pointIt)[10] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[1]->outputIndex 
-	   << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[8] + nVertices << "\n";
-
-      file << " " << (*pointIt)[4] + nVertices
-	   << " " << (*pointIt)[5] + nVertices
-	   << " " << (*pointIt)[9] + nVertices << "\n";
-
-      file << " " << (*pointIt)[4] + nVertices
-	   << " " << (*pointIt)[9] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << (*pointIt)[9] + nVertices
-	   << " " << (*pointIt)[10] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[10] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << (*pointIt)[0] + nVertices
-	   << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[10] + nVertices << "\n";
-
-      file << " " << (*pointIt)[3] + nVertices
-	   << " " << (*pointIt)[4] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << (*pointIt)[2] + nVertices
-	   << " " << (*pointIt)[3] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << (*pointIt)[1] + nVertices
-	   << " " << (*pointIt)[2] + nVertices
-	   << " " << (*pointIt)[11] + nVertices << "\n";
-
-      file << " " << elementIt->vertexInfo[2]->outputIndex 
-	   << " " << (*pointIt)[2] + nVertices
-	   << " " << (*pointIt)[3] + nVertices << "\n";
-
-    }
+    return 0;
   }
 
   int VtkWriter::updateAnimationFile(std::string valueFilename,
@@ -414,18 +75,4 @@ namespace AMDiS {
   }
 
 
-  void VtkWriter::writeFile(DOFVector<double> *values,
-			    const char *filename)
-  {
-    DataCollector *dc = NEW DataCollector(values->getFESpace(), values);
-    std::vector<DataCollector*> dcList(0);
-    dcList.push_back(dc);
-
-    VtkWriter *writer = NEW VtkWriter(&dcList);
-    writer->writeFile(filename);
-
-    DELETE writer;
-    DELETE dc;
-  }
-
 }

AMDiS/src/VtkWriter.h

@@ -22,10 +22,17 @@
 #ifndef AMDIS_VTKWRITER_H
 #define AMDIS_VTKWRITER_H
 
-#include <fstream>
+#ifdef HAVE_BOOST
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#endif
 
+#include <fstream>
 #include "BasisFunction.h"
 #include "DataCollector.h"
+#include "FileWriter.h"
 
 namespace AMDiS {
 
@@ -33,7 +40,8 @@ namespace AMDiS {
   {
   public:
     VtkWriter(std::vector<DataCollector*> *dc)
-      : dataCollector(dc)
+      : dataCollector(dc),
+	compress(NONE)
     {
       degree = (*dataCollector)[0]->getFeSpace()->getBasisFcts()->getDegree();    
       dim = (*dataCollector)[0]->getMesh()->getDim();
@@ -45,11 +53,13 @@ namespace AMDiS {
      */
     int writeFile(const char *name);
 
+
     /** \brief
-     * May be used to simply write ParaView files.
+     * Set a compressing method for file output.
      */
-    static void writeFile(DOFVector<double> *values,
-			  const char *filename);
+    void setCompression(FileCompression c) {
+      compress = c;
+    }
 
     /** \brief
      * Adds a new entry to a ParaView animation file.
@@ -58,45 +68,71 @@ namespace AMDiS {
 			    std::vector< std::string > *paraViewAnimationFrames,
 			    const char *animationFilename);
   protected:
+    /** \brief
+     * Writes the VTU file to an arbitrary stream.
+     */
+    template<typename T>
+    void writeFile(T &file);
+
     /** \brief
      * Writes all coordinates of vertices and interpolation points to an
      * output file.
      */
-    void writeVertexCoords(std::ofstream &file);
+    template<typename T>
+    void writeVertexCoords(T &file);
 
 
     /** \brief
      * Writes all values of vertices and interpolation point to an output 
      * file.
      */
-    void writeVertexValues(std::ofstream &file, int componentNo);
+    template<typename T>
+    void writeVertexValues(T &file, int componentNo);
 
 
     /** \brief
      * Writes the connectivity of all simplices to an output file.
      */
-    void writeConnectivity(std::ofstream &file);
+    template<typename T>
+    void writeConnectivity(T &file);
 
     /** \brief
      * Writes the connectivity for the case dim = 2 and degree = 2 to
      * an output file.
      */
-    void writeConnectivity_dim2_degree2(std::ofstream &file);
+    template<typename T>
+    void writeConnectivity_dim2_degree2(T &file);
 
 
     /** \brief
      * Writes the connectivity for the case dim = 2 and degree = 3 to
      * an output file.
      */
-    void writeConnectivity_dim2_degree3(std::ofstream &file);
+    template<typename T>
+    void writeConnectivity_dim2_degree3(T &file);
 
 
     /** \brief
      * Writes the connectivity for the case dim = 2 and degree = 4 to
      * an output file.
      */
-    void writeConnectivity_dim2_degree4(std::ofstream &file);
+    template<typename T>
+    void writeConnectivity_dim2_degree4(T &file);
 
+#ifdef HAVE_BOOST
+    /** \brief
+     * Writes a world coordinate to a given file.
+     */
+    inline void writeCoord(boost::iostreams::filtering_ostream &file, WorldVector<double> coord) {
+      for (int i = 0; i < Global::getGeo(WORLD); i++) {
+	file << " " << std::scientific << coord[i];
+      }
+      for (int i = Global::getGeo(WORLD); i < 3; i++) {
+	file << " 0.0";
+      }
+      file << "\n";
+    }
+#endif
 
     /** \brief
      * Writes a world coordinate to a given file.
@@ -112,12 +148,18 @@ namespace AMDiS {
     }
 
   private:
-    /**
+    /** \brief
      * List of DataCollectors, for each component of the problem one.