Skip to content

GitLab

Commit 682a54ee authored by Praetorius, Simon's avatar Praetorius, Simon
Browse files
Options

Compression support added to VtkWriter

parent 76fe101c
Hide whitespace changes
Inline Side-by-side
Showing with 836 additions and 154 deletions
AMDiS/src/io/FileWriter.cc
View file @ 682a54ee
......@@ -57,7 +57,7 @@ namespace AMDiS
{
initialize();
/**
/*
* Removed by Siqi. not sure.
* for (int i = 0; i < static_cast<int>(vecs->getSize()); i++)
* TEST_EXIT(vecs->getDOFVector(0)->getFeSpace() == vecs->getDOFVector(i)->getFeSpace())
......@@ -171,10 +171,7 @@ namespace AMDiS
}
if (writeParaViewFormat) {
VtkWriter::Aux vtkWriter(&dataCollectors);
#ifdef HAVE_COMPRESSION
vtkWriter.setCompression(compression);
#endif
VtkWriter::Aux vtkWriter(&dataCollectors, VtkWriter::Vtuformat(paraViewMode), (paraViewPrecision == 1));
vtkWriter.writeFile(fn + paraviewFileExt);
#if HAVE_PARALLEL_DOMAIN_AMDIS
......@@ -187,7 +184,9 @@ namespace AMDiS
MPI::COMM_WORLD.Get_size(),
filename,
postfix,
componentNames);
componentNames,
VtkWriter::Vtuformat(paraViewMode),
(paraViewPrecision == 1));
}
#endif
......
AMDiS/src/io/FileWriter.h
View file @ 682a54ee
......@@ -161,6 +161,12 @@ namespace AMDiS {
/// 0: Don't write ParaView files; 1: Write ParaView files.
int writeParaViewFormat;
/// 0: ASCII mode; 1: Appended mode; 2:Appended_compressed mode.
int paraViewMode;
/// 0: FLOAT32 precision; 1: FLOAT64 precision. Only works in appended and appended_compressed mode.
int paraViewPrecision;
/// 0: Don't write ParaView std::vector files; 1: Write ParaView std::vector files.
int writeParaViewVectorFormat;
......
AMDiS/src/io/FileWriter.hh
View file @ 682a54ee
......@@ -117,6 +117,8 @@ namespace AMDiS
periodicFileExt = ".per";
writeAMDiSFormat = 0;
writeParaViewFormat = 0;
paraViewMode = 0;
paraViewPrecision = 0;
writeParaViewVectorFormat = 0;
writeAs3dVector = false;
writeParaViewAnimation = 0;
......@@ -149,6 +151,8 @@ namespace AMDiS
Parameters::get(name + "->AMDiS mesh ext", amdisMeshExt);
Parameters::get(name + "->AMDiS data ext", amdisDataExt);
Parameters::get(name + "->ParaView format", writeParaViewFormat);
Parameters::get(name + "->ParaView mode", paraViewMode);
Parameters::get(name + "->ParaView precision", paraViewPrecision);
Parameters::get(name + "->ParaView vector format", writeParaViewVectorFormat);
Parameters::get(name + "->write vector as 3d vector", writeAs3dVector);
Parameters::get(name + "->ParaView animation", writeParaViewAnimation);
......
AMDiS/src/io/VtkReader.hh
View file @ 682a54ee
......@@ -43,6 +43,7 @@ namespace AMDiS
std::vector<DOFVector<T>*> dofVectors,
std::vector<std::string> componentNames)
{
using namespace std;
using namespace pugi;
using namespace AMDiS::io::VtkReader;
......@@ -58,31 +59,103 @@ namespace AMDiS
TEST_EXIT(vtu.load_file(filename.c_str()))("Could not load vtu file! Error in xml structure.\n");
xml_node VTKFile = vtu.child("VTKFile");
string zlib = VTKFile.attribute("compressor").value();
TEST_EXIT(zlib == "" || zlib == "vtkZLibDataCompressor")("The only supported compressor is Zlib.\n Should not happen.\n ");
xml_node UnstructuredGrid = VTKFile.child("UnstructuredGrid");
xml_node Piece = UnstructuredGrid.child("Piece");
xml_node PointData = Piece.child("PointData");
xml_node Points = Piece.child("Points");
std::string points = Points.child_value("DataArray");
typedef std::vector<WorldVector<double> > PL;
xml_node PointsDataArray = Points.child("DataArray");
xml_node AppendedData = VTKFile.child("AppendedData");
typedef vector<WorldVector<double> > PL;
PL pointList;
detail::string2pointList(points, pointList);
std::vector<std::vector<T> > valueList(dofVectors.size());
T test;
for (xml_node DataArray = PointData.child("DataArray"); DataArray; DataArray = DataArray.next_sibling("DataArray")) {
std::string Name = DataArray.attribute("Name").value();
for (size_t i = 0; i < componentNames.size(); i++) {
if (Name == componentNames[i]) {
std::string values = DataArray.last_child().value();
int nComponents = -1;
if (DataArray.attribute("NumberOfComponents"))
nComponents = DataArray.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::string2valueList(values, valueList[i], vector_operations::num_rows(test), nComponents);
break;
vector<vector<T> > valueList(dofVectors.size());
if(!AppendedData) {
xml_node PointData = Piece.child("PointData");
string points = Points.child_value("DataArray");
detail::string2pointList(points, pointList);
T test;
for (xml_node DataArray = PointData.child("DataArray"); DataArray; DataArray = DataArray.next_sibling("DataArray")) {
string Name = DataArray.attribute("Name").value();
for (size_t i = 0; i < componentNames.size(); i++) {
if (Name == componentNames[i]) {
string values = DataArray.last_child().value();
int nComponents = -1;
if (DataArray.attribute("NumberOfComponents"))
nComponents = DataArray.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::string2valueList(values, valueList[i], vector_operations::num_rows(test), nComponents);
break;
}
}
}
} else {
string encoding = AppendedData.attribute("encoding").value();
TEST_EXIT(encoding == "base64")
("Currently the encoding of AppendedData only supports base64. But it's easy to extend to raw.\n");
string appendedData = AppendedData.last_child().value();
int start = appendedData.find("_");
appendedData = appendedData.substr(start + 1, appendedData.length() - 1);
vector<int> offsetVec;
int pointsIndex = 0, index = 0;
vector<pair<xml_node, int> > pointDataIndex;
for (xml_node Parent = Piece.first_child(); Parent; Parent = Parent.next_sibling()) {
for(xml_node Data = Parent.child("DataArray"); Data; Data = Data.next_sibling("DataArray"), index++) {
string parentName = Data.parent().name();
if(parentName == "Points")
pointsIndex = index;
if(parentName == "PointData")
pointDataIndex.push_back(make_pair(Data, index));
offsetVec.push_back(Data.attribute("offset").as_int());
}
}
string format = PointsDataArray.attribute("format").value();
string type = PointsDataArray.attribute("type").value();
TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n");
TEST_EXIT(type == "Float32" || type == "Float64")("Currently only supports Points with type Float32 and Float64.\n");
int len = ((unsigned)(pointsIndex + 1) == offsetVec.size()) ?
appendedData.length() - offsetVec[pointsIndex] :
offsetVec[pointsIndex + 1] - offsetVec[pointsIndex];
string points = appendedData.substr(offsetVec[pointsIndex], len);
detail::binary2pointList(points, type, (zlib != ""), pointList);
T test;
for (size_t i = 0 ; i < pointDataIndex.size(); i++) {
string Name = pointDataIndex[i].first.attribute("Name").value();
for (size_t j = 0; j < componentNames.size(); j++) {
if(Name == componentNames[j]) {
format = pointDataIndex[i].first.attribute("format").value();
type = pointDataIndex[i].first.attribute("type").value();
TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n");
TEST_EXIT(type == "Float32" || type == "Float64")("Currently only supports PointData with type Float32 and Float64.\n");
len = ((unsigned)(pointDataIndex[i].second + 1) == offsetVec.size())?
appendedData.length() - offsetVec[pointDataIndex[i].second]:
offsetVec[pointDataIndex[i].second + 1] - offsetVec[pointDataIndex[i].second];
string values = appendedData.substr(offsetVec[pointDataIndex[i].second], len);
int nComponents = -1;
if (pointDataIndex[i].first.attribute("NumberOfComponents"))
nComponents = pointDataIndex[i].first.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::binary2valueList(values, type, (zlib != ""), valueList[j], vector_operations::num_rows(test), nComponents);
break;
}
}
}
}
......@@ -100,7 +173,7 @@ namespace AMDiS
tree.index->buildIndex();
DOFIterator<WorldVector<double> > it_coords(&coords, USED_DOFS);
std::vector<DOFIterator<T>*> it_results;
vector<DOFIterator<T>*> it_results;
for (size_t i = 0; i < dofVectors.size(); i++) {
it_results.push_back(new DOFIterator<T>(dofVectors[i], USED_DOFS));
it_results[i]->reset();
......
AMDiS/src/io/VtkWriter.cc
View file @ 682a54ee
......@@ -40,7 +40,7 @@
#include "ElementRegion_ED.h"
#include "AdaptInfo.h"
#include "detail/VtkWriter.h"
namespace AMDiS
{
......@@ -53,23 +53,29 @@ namespace AMDiS
void writeFile(DOFVector<double> *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
DataCollector<> dc(values->getFeSpace(), values);
vector<DataCollector<>*> dcList(0);
dcList.push_back(&dc);
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
}
void writeFile(vector<DOFVector<double>* > &values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (size_t i = 0; i < values.size(); i++)
dcList.push_back(new DataCollector<>(values[i]->getFeSpace(), values[i]));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (size_t i = 0; i < values.size(); i++)
delete dcList[i];
}
......@@ -77,12 +83,15 @@ namespace AMDiS
void writeFile(WorldVector<DOFVector<double>* > &values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (int i = 0; i < values.getSize(); i++)
dcList.push_back(new DataCollector<>(values[i]->getFeSpace(), values[i]));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (int i = 0; i < values.getSize(); i++)
delete dcList[i];
}
......@@ -90,14 +99,17 @@ namespace AMDiS
void writeFile(DOFVector<WorldVector<double> > *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
WorldVector<DOFVector<double>*> valuesWV;
for (int i =0 ; i < valuesWV.getSize(); i++)
valuesWV[i] = new DOFVector<double>(values->getFeSpace(),
"values["+boost::lexical_cast<std::string>(i)+"]");
transform(values, &valuesWV);
writeFile(valuesWV, filename, writeParallel);
writeFile(valuesWV, filename, format, highPrecision, writeParallel);
for (int i = 0; i < valuesWV.getSize(); i++)
delete valuesWV[i];
}
......@@ -105,13 +117,16 @@ namespace AMDiS
void writeFile(SystemVector *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (int i = 0; i < values->getSize(); i++)
dcList.push_back(new DataCollector<>(values->getDOFVector(i)->getFeSpace(),
values->getDOFVector(i)));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (size_t i = 0; i < dcList.size(); i++)
delete dcList[i];
}
......@@ -119,9 +134,12 @@ namespace AMDiS
void writeFile(vector<DataCollector<>*> &dcList,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
::AMDiS::io::VtkWriter::Aux writer(&dcList);
::AMDiS::io::VtkWriter::Aux writer(&dcList, format, highPrecision);
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (writeParallel) {
......@@ -137,7 +155,7 @@ namespace AMDiS
componentNames.push_back(dcList[i]->getValues()->getName());
detail::writeParallelFile(name + ".pvtu", MPI::COMM_WORLD.Get_size(),
name, ".vtu", componentNames);
name, ".vtu", componentNames, format, highPrecision);
}
filename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.vtu";
......
AMDiS/src/io/VtkWriter.h
View file @ 682a54ee
......@@ -29,6 +29,7 @@
#include "DOFVector.h"
#include "FixVec.h"
#include "SystemVector.h"
#include "detail/VtkWriter.h"
namespace AMDiS { namespace io {
......@@ -39,11 +40,14 @@ namespace AMDiS { namespace io {
* VTU-files. The files generated are text-based XML-structured
* files and can be read by ParaView.
**/
namespace VtkWriter
{
/// Interface for general containers not implemented. Specializations below.
template<typename Container>
void writeFile(Container& vec, std::string filename,
void writeFile(Container& vec, std::string filename,
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true)
{
ERROR_EXIT("VtkWriter not implemented for this container type!\n");
......@@ -52,33 +56,48 @@ namespace AMDiS { namespace io {
/// Write a \ref DOFVector to a file. Using a container pointer.
void writeFile(DOFVector<double> *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector to file. Using a container reference.
inline
void writeFile(DOFVector<double> &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a vector of \ref DOFVector to file.
void writeFile(std::vector<DOFVector<double>*> &values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref WorldVector of \ref DOFVector to file.
void writeFile(WorldVector<DOFVector<double>*> &values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector <WorldVector> to file, by converting
/// to a \ref WorldVector of \ref DOFVector and calling the appropriate
/// method above. Using a container pointer.
void writeFile(DOFVector<WorldVector<double> > *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector <WorldVector> to file, by converting
/// to a \ref WorldVector of \ref DOFVector and calling the appropriate
......@@ -86,29 +105,41 @@ namespace AMDiS { namespace io {
inline
void writeFile(DOFVector<WorldVector<double> > &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a \ref SystemVector to file. Using a container pointer.
void writeFile(SystemVector *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref SystemVector to file. Using a container reference.
inline
void writeFile(SystemVector &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a vector of \ref DataCollector to file.
void writeFile(std::vector<DataCollector<>*> &dcList,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
} // end namespace VtkWriter
} } // end namespace io, AMDiS
......
AMDiS/src/io/detail/Arh2Reader.cc
View file @ 682a54ee
......@@ -7,6 +7,12 @@
#include "Debug.h"
#include "../Arh2Reader.h"
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#ifdef HAVE_COMPRESSION
#include <boost/iostreams/filter/zlib.hpp>
#endif
namespace AMDiS { namespace io {
using namespace std;
......@@ -210,6 +216,7 @@ namespace AMDiS { namespace io {
TEST_EXIT(file.is_open())
("Cannot open file %s\n", filename.c_str());
string cps(4, ' ');
uint32_t headerLen = 0;
uint32_t nMacroElements = 0;
uint32_t nValueVectors = 0;
......@@ -225,7 +232,16 @@ namespace AMDiS { namespace io {
file.read(reinterpret_cast<char*>(&nFeSpaces), 4);
file.read(reinterpret_cast<char*>(&nValueVectors), 4);
file.read(reinterpret_cast<char*>(&nMacroElements), 4);
file.read(const_cast<char*>(cps.data()), 4);
#ifdef HAVE_COMPRESSION
TEST_EXIT(cps == "null" || cps == "zlib")
("Cannot read Arh2 file. Currently only support zlib compression.\n");
#else
TEST_EXIT(cps == "null")
("HAVE_COMPRESSION OFF. Cannot read compressed Arh2 file.\n");
#endif
TEST_EXIT(dow == (unsigned)mesh->getGeo(WORLD))
("Dow is not match!\n");
TEST_EXIT(dim == (unsigned)mesh->getDim())
......@@ -238,6 +254,7 @@ namespace AMDiS { namespace io {
vector<string> vecsName(0);
vector<int> vecsFeSpaceNum(0);
vector<int> macroElIndex(nMacroElements);
vector<int> macroElSize(nMacroElements);
vector<vector<int> > feSpaceDOFs(0);
vector<int> perDOFs(4, 0);
......@@ -247,8 +264,7 @@ namespace AMDiS { namespace io {
for(unsigned int i = 0; i < nMacroElements; i++)
{
file.read(reinterpret_cast<char*>(&macroElIndex[i]), 4);
uint32_t elpos = 0;
file.read(reinterpret_cast<char*>(&elpos), 4);
file.read(reinterpret_cast<char*>(&macroElSize[i]), 4);
}
// Read feSpace table
for(unsigned int i = 0; i < nFeSpaces; i++)
......@@ -338,14 +354,30 @@ namespace AMDiS { namespace io {
// Read data: meshstructure and dof values
for (unsigned int i = 0; i < nMacroElements; i++) {
uint32_t nStructureCodes = 0;
stringstream dataStream(ios::out | ios::in | ios::binary);
int size = macroElSize[i];
char* buffer = new char[size];
file.read(buffer, size);
dataStream.write(buffer, size);
delete[] buffer;
#ifdef HAVE_COMPRESSION
if(cps == "zlib") {
stringstream tmp(ios::out | ios::in);
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::zlib_decompressor());
in.push(dataStream);
boost::iostreams::copy(in, tmp);
dataStream.str(tmp.str());
}
#endif
uint32_t nStructureCodes = 0;
uint32_t codeSize = 0;
file.read(reinterpret_cast<char*>(&nStructureCodes), 4);
file.read(reinterpret_cast<char*>(&codeSize), 4);
dataStream.read(reinterpret_cast<char*>(&nStructureCodes), 4);
dataStream.read(reinterpret_cast<char*>(&codeSize), 4);
vector<uint64_t> structureCode(nStructureCodes);
file.read(reinterpret_cast<char*>(&(structureCode[0])), 8 * nStructureCodes);
dataStream.read(reinterpret_cast<char*>(&(structureCode[0])), 8 * nStructureCodes);
MeshStructure elementStructure;
elementStructure.init(structureCode, codeSize);
......@@ -358,11 +390,11 @@ namespace AMDiS { namespace io {
for(unsigned int j = 0; j < nFeSpaces; j++)
{
uint32_t nValuesPerVector = 0;
file.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
dataStream.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
for(size_t k = 0; k < sortedFeSpaces[j].size(); k++)
{
values[valuePos + k].resize(nValuesPerVector);
file.read(reinterpret_cast<char*>(&(values[valuePos + k][0])), 8 * nValuesPerVector);
dataStream.read(reinterpret_cast<char*>(&(values[valuePos + k][0])), 8 * nValuesPerVector);
}
valuePos += sortedFeSpaces[j].size();
}
......
AMDiS/src/io/detail/Arh2Writer.cc
View file @ 682a54ee
......@@ -9,6 +9,12 @@
#include "DOFVector.h"
#include "../Arh2Writer.h"
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#ifdef HAVE_COMPRESSION
#include <boost/iostreams/filter/zlib.hpp>
#endif
namespace AMDiS { namespace io {
using namespace std;
......@@ -139,12 +145,15 @@ namespace AMDiS { namespace io {
uint32_t dow = mesh->getGeo(WORLD);
uint32_t dim = mesh->getDim();
uint32_t headerLen = 30 + //fixed part of header
uint32_t headerLen = 34 + //fixed part of header
nMacroElements * 8 + //macroElemnts table
nFeSpaces * 16 + //feSpaces table
namesLen + //value vector table
nValueVectors * 8; //also value vector table
string typeId = "arh2";
string typeId = "arh2", cps("null");
#ifdef HAVE_COMPRESSION
cps = "zlib";
#endif
uint8_t *major = const_cast<uint8_t*>(&(AMDiS::io::Arh2Writer::MAJOR));
uint8_t *minor = const_cast<uint8_t*>(&(AMDiS::io::Arh2Writer::MINOR));
......@@ -158,7 +167,7 @@ namespace AMDiS { namespace io {
file.write(reinterpret_cast<char*>(&nFeSpaces), 4);
file.write(reinterpret_cast<char*>(&nValueVectors), 4);
file.write(reinterpret_cast<char*>(&nMacroElements), 4);
file.write(cps.c_str(), 4);
//macro table
deque<MacroElement*>::const_iterator macroIter = mesh->firstMacroElement();
while(macroIter != mesh->endOfMacroElements())
......@@ -184,7 +193,7 @@ namespace AMDiS { namespace io {
posDOFs = (*nDOF)[j];
file.write(reinterpret_cast<char*>(&posDOFs), 4);
}
for(int j = nDOF->size(); j < 4 ; j++)
for(size_t j = nDOF->size(); j < 4 ; j++)
{