Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit 682a54ee authored by Praetorius, Simon's avatar Praetorius, Simon
Browse files

Compression support added to VtkWriter

parent 76fe101c
......@@ -57,7 +57,7 @@ namespace AMDiS
{
initialize();
/**
/*
* Removed by Siqi. not sure.
* for (int i = 0; i < static_cast<int>(vecs->getSize()); i++)
* TEST_EXIT(vecs->getDOFVector(0)->getFeSpace() == vecs->getDOFVector(i)->getFeSpace())
......@@ -171,10 +171,7 @@ namespace AMDiS
}
if (writeParaViewFormat) {
VtkWriter::Aux vtkWriter(&dataCollectors);
#ifdef HAVE_COMPRESSION
vtkWriter.setCompression(compression);
#endif
VtkWriter::Aux vtkWriter(&dataCollectors, VtkWriter::Vtuformat(paraViewMode), (paraViewPrecision == 1));
vtkWriter.writeFile(fn + paraviewFileExt);
#if HAVE_PARALLEL_DOMAIN_AMDIS
......@@ -187,7 +184,9 @@ namespace AMDiS
MPI::COMM_WORLD.Get_size(),
filename,
postfix,
componentNames);
componentNames,
VtkWriter::Vtuformat(paraViewMode),
(paraViewPrecision == 1));
}
#endif
......
......@@ -161,6 +161,12 @@ namespace AMDiS {
/// 0: Don't write ParaView files; 1: Write ParaView files.
int writeParaViewFormat;
/// 0: ASCII mode; 1: Appended mode; 2:Appended_compressed mode.
int paraViewMode;
/// 0: FLOAT32 precision; 1: FLOAT64 precision. Only works in appended and appended_compressed mode.
int paraViewPrecision;
/// 0: Don't write ParaView std::vector files; 1: Write ParaView std::vector files.
int writeParaViewVectorFormat;
......
......@@ -117,6 +117,8 @@ namespace AMDiS
periodicFileExt = ".per";
writeAMDiSFormat = 0;
writeParaViewFormat = 0;
paraViewMode = 0;
paraViewPrecision = 0;
writeParaViewVectorFormat = 0;
writeAs3dVector = false;
writeParaViewAnimation = 0;
......@@ -149,6 +151,8 @@ namespace AMDiS
Parameters::get(name + "->AMDiS mesh ext", amdisMeshExt);
Parameters::get(name + "->AMDiS data ext", amdisDataExt);
Parameters::get(name + "->ParaView format", writeParaViewFormat);
Parameters::get(name + "->ParaView mode", paraViewMode);
Parameters::get(name + "->ParaView precision", paraViewPrecision);
Parameters::get(name + "->ParaView vector format", writeParaViewVectorFormat);
Parameters::get(name + "->write vector as 3d vector", writeAs3dVector);
Parameters::get(name + "->ParaView animation", writeParaViewAnimation);
......
......@@ -43,6 +43,7 @@ namespace AMDiS
std::vector<DOFVector<T>*> dofVectors,
std::vector<std::string> componentNames)
{
using namespace std;
using namespace pugi;
using namespace AMDiS::io::VtkReader;
......@@ -58,31 +59,103 @@ namespace AMDiS
TEST_EXIT(vtu.load_file(filename.c_str()))("Could not load vtu file! Error in xml structure.\n");
xml_node VTKFile = vtu.child("VTKFile");
string zlib = VTKFile.attribute("compressor").value();
TEST_EXIT(zlib == "" || zlib == "vtkZLibDataCompressor")("The only supported compressor is Zlib.\n Should not happen.\n ");
xml_node UnstructuredGrid = VTKFile.child("UnstructuredGrid");
xml_node Piece = UnstructuredGrid.child("Piece");
xml_node PointData = Piece.child("PointData");
xml_node Points = Piece.child("Points");
std::string points = Points.child_value("DataArray");
typedef std::vector<WorldVector<double> > PL;
xml_node PointsDataArray = Points.child("DataArray");
xml_node AppendedData = VTKFile.child("AppendedData");
typedef vector<WorldVector<double> > PL;
PL pointList;
detail::string2pointList(points, pointList);
std::vector<std::vector<T> > valueList(dofVectors.size());
T test;
for (xml_node DataArray = PointData.child("DataArray"); DataArray; DataArray = DataArray.next_sibling("DataArray")) {
std::string Name = DataArray.attribute("Name").value();
for (size_t i = 0; i < componentNames.size(); i++) {
if (Name == componentNames[i]) {
std::string values = DataArray.last_child().value();
int nComponents = -1;
if (DataArray.attribute("NumberOfComponents"))
nComponents = DataArray.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::string2valueList(values, valueList[i], vector_operations::num_rows(test), nComponents);
break;
vector<vector<T> > valueList(dofVectors.size());
if(!AppendedData) {
xml_node PointData = Piece.child("PointData");
string points = Points.child_value("DataArray");
detail::string2pointList(points, pointList);
T test;
for (xml_node DataArray = PointData.child("DataArray"); DataArray; DataArray = DataArray.next_sibling("DataArray")) {
string Name = DataArray.attribute("Name").value();
for (size_t i = 0; i < componentNames.size(); i++) {
if (Name == componentNames[i]) {
string values = DataArray.last_child().value();
int nComponents = -1;
if (DataArray.attribute("NumberOfComponents"))
nComponents = DataArray.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::string2valueList(values, valueList[i], vector_operations::num_rows(test), nComponents);
break;
}
}
}
} else {
string encoding = AppendedData.attribute("encoding").value();
TEST_EXIT(encoding == "base64")
("Currently the encoding of AppendedData only supports base64. But it's easy to extend to raw.\n");
string appendedData = AppendedData.last_child().value();
int start = appendedData.find("_");
appendedData = appendedData.substr(start + 1, appendedData.length() - 1);
vector<int> offsetVec;
int pointsIndex = 0, index = 0;
vector<pair<xml_node, int> > pointDataIndex;
for (xml_node Parent = Piece.first_child(); Parent; Parent = Parent.next_sibling()) {
for(xml_node Data = Parent.child("DataArray"); Data; Data = Data.next_sibling("DataArray"), index++) {
string parentName = Data.parent().name();
if(parentName == "Points")
pointsIndex = index;
if(parentName == "PointData")
pointDataIndex.push_back(make_pair(Data, index));
offsetVec.push_back(Data.attribute("offset").as_int());
}
}
string format = PointsDataArray.attribute("format").value();
string type = PointsDataArray.attribute("type").value();
TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n");
TEST_EXIT(type == "Float32" || type == "Float64")("Currently only supports Points with type Float32 and Float64.\n");
int len = ((unsigned)(pointsIndex + 1) == offsetVec.size()) ?
appendedData.length() - offsetVec[pointsIndex] :
offsetVec[pointsIndex + 1] - offsetVec[pointsIndex];
string points = appendedData.substr(offsetVec[pointsIndex], len);
detail::binary2pointList(points, type, (zlib != ""), pointList);
T test;
for (size_t i = 0 ; i < pointDataIndex.size(); i++) {
string Name = pointDataIndex[i].first.attribute("Name").value();
for (size_t j = 0; j < componentNames.size(); j++) {
if(Name == componentNames[j]) {
format = pointDataIndex[i].first.attribute("format").value();
type = pointDataIndex[i].first.attribute("type").value();
TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n");
TEST_EXIT(type == "Float32" || type == "Float64")("Currently only supports PointData with type Float32 and Float64.\n");
len = ((unsigned)(pointDataIndex[i].second + 1) == offsetVec.size())?
appendedData.length() - offsetVec[pointDataIndex[i].second]:
offsetVec[pointDataIndex[i].second + 1] - offsetVec[pointDataIndex[i].second];
string values = appendedData.substr(offsetVec[pointDataIndex[i].second], len);
int nComponents = -1;
if (pointDataIndex[i].first.attribute("NumberOfComponents"))
nComponents = pointDataIndex[i].first.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(vector_operations::num_rows(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n");
detail::binary2valueList(values, type, (zlib != ""), valueList[j], vector_operations::num_rows(test), nComponents);
break;
}
}
}
}
......@@ -100,7 +173,7 @@ namespace AMDiS
tree.index->buildIndex();
DOFIterator<WorldVector<double> > it_coords(&coords, USED_DOFS);
std::vector<DOFIterator<T>*> it_results;
vector<DOFIterator<T>*> it_results;
for (size_t i = 0; i < dofVectors.size(); i++) {
it_results.push_back(new DOFIterator<T>(dofVectors[i], USED_DOFS));
it_results[i]->reset();
......
......@@ -40,7 +40,7 @@
#include "ElementRegion_ED.h"
#include "AdaptInfo.h"
#include "detail/VtkWriter.h"
namespace AMDiS
{
......@@ -53,23 +53,29 @@ namespace AMDiS
void writeFile(DOFVector<double> *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
DataCollector<> dc(values->getFeSpace(), values);
vector<DataCollector<>*> dcList(0);
dcList.push_back(&dc);
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
}
void writeFile(vector<DOFVector<double>* > &values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (size_t i = 0; i < values.size(); i++)
dcList.push_back(new DataCollector<>(values[i]->getFeSpace(), values[i]));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (size_t i = 0; i < values.size(); i++)
delete dcList[i];
}
......@@ -77,12 +83,15 @@ namespace AMDiS
void writeFile(WorldVector<DOFVector<double>* > &values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (int i = 0; i < values.getSize(); i++)
dcList.push_back(new DataCollector<>(values[i]->getFeSpace(), values[i]));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (int i = 0; i < values.getSize(); i++)
delete dcList[i];
}
......@@ -90,14 +99,17 @@ namespace AMDiS
void writeFile(DOFVector<WorldVector<double> > *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
WorldVector<DOFVector<double>*> valuesWV;
for (int i =0 ; i < valuesWV.getSize(); i++)
valuesWV[i] = new DOFVector<double>(values->getFeSpace(),
"values["+boost::lexical_cast<std::string>(i)+"]");
transform(values, &valuesWV);
writeFile(valuesWV, filename, writeParallel);
writeFile(valuesWV, filename, format, highPrecision, writeParallel);
for (int i = 0; i < valuesWV.getSize(); i++)
delete valuesWV[i];
}
......@@ -105,13 +117,16 @@ namespace AMDiS
void writeFile(SystemVector *values,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
vector<DataCollector<>*> dcList(0);
for (int i = 0; i < values->getSize(); i++)
dcList.push_back(new DataCollector<>(values->getDOFVector(i)->getFeSpace(),
values->getDOFVector(i)));
writeFile(dcList, filename, writeParallel);
writeFile(dcList, filename, format, highPrecision, writeParallel);
for (size_t i = 0; i < dcList.size(); i++)
delete dcList[i];
}
......@@ -119,9 +134,12 @@ namespace AMDiS
void writeFile(vector<DataCollector<>*> &dcList,
string filename,
bool writeParallel)
Vtuformat format,
bool highPrecision,
bool writeParallel
)
{
::AMDiS::io::VtkWriter::Aux writer(&dcList);
::AMDiS::io::VtkWriter::Aux writer(&dcList, format, highPrecision);
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (writeParallel) {
......@@ -137,7 +155,7 @@ namespace AMDiS
componentNames.push_back(dcList[i]->getValues()->getName());
detail::writeParallelFile(name + ".pvtu", MPI::COMM_WORLD.Get_size(),
name, ".vtu", componentNames);
name, ".vtu", componentNames, format, highPrecision);
}
filename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.vtu";
......
......@@ -29,6 +29,7 @@
#include "DOFVector.h"
#include "FixVec.h"
#include "SystemVector.h"
#include "detail/VtkWriter.h"
namespace AMDiS { namespace io {
......@@ -39,11 +40,14 @@ namespace AMDiS { namespace io {
* VTU-files. The files generated are text-based XML-structured
* files and can be read by ParaView.
**/
namespace VtkWriter
{
/// Interface for general containers not implemented. Specializations below.
template<typename Container>
void writeFile(Container& vec, std::string filename,
void writeFile(Container& vec, std::string filename,
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true)
{
ERROR_EXIT("VtkWriter not implemented for this container type!\n");
......@@ -52,33 +56,48 @@ namespace AMDiS { namespace io {
/// Write a \ref DOFVector to a file. Using a container pointer.
void writeFile(DOFVector<double> *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector to file. Using a container reference.
inline
void writeFile(DOFVector<double> &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a vector of \ref DOFVector to file.
void writeFile(std::vector<DOFVector<double>*> &values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref WorldVector of \ref DOFVector to file.
void writeFile(WorldVector<DOFVector<double>*> &values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector <WorldVector> to file, by converting
/// to a \ref WorldVector of \ref DOFVector and calling the appropriate
/// method above. Using a container pointer.
void writeFile(DOFVector<WorldVector<double> > *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref DOFVector <WorldVector> to file, by converting
/// to a \ref WorldVector of \ref DOFVector and calling the appropriate
......@@ -86,29 +105,41 @@ namespace AMDiS { namespace io {
inline
void writeFile(DOFVector<WorldVector<double> > &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a \ref SystemVector to file. Using a container pointer.
void writeFile(SystemVector *values,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
/// Write a \ref SystemVector to file. Using a container reference.
inline
void writeFile(SystemVector &values,
std::string filename,
bool writeParallel = true)
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
)
{
writeFile(&values, filename, writeParallel);
writeFile(&values, filename, format, highPrecision, writeParallel);
}
/// Write a vector of \ref DataCollector to file.
void writeFile(std::vector<DataCollector<>*> &dcList,
std::string filename,
bool writeParallel = true);
Vtuformat format = ASCII,
bool highPrecision = false,
bool writeParallel = true
);
} // end namespace VtkWriter
} } // end namespace io, AMDiS
......
......@@ -7,6 +7,12 @@
#include "Debug.h"
#include "../Arh2Reader.h"
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#ifdef HAVE_COMPRESSION
#include <boost/iostreams/filter/zlib.hpp>
#endif
namespace AMDiS { namespace io {
using namespace std;
......@@ -210,6 +216,7 @@ namespace AMDiS { namespace io {
TEST_EXIT(file.is_open())
("Cannot open file %s\n", filename.c_str());
string cps(4, ' ');
uint32_t headerLen = 0;
uint32_t nMacroElements = 0;
uint32_t nValueVectors = 0;
......@@ -225,7 +232,16 @@ namespace AMDiS { namespace io {
file.read(reinterpret_cast<char*>(&nFeSpaces), 4);
file.read(reinterpret_cast<char*>(&nValueVectors), 4);
file.read(reinterpret_cast<char*>(&nMacroElements), 4);
file.read(const_cast<char*>(cps.data()), 4);
#ifdef HAVE_COMPRESSION
TEST_EXIT(cps == "null" || cps == "zlib")
("Cannot read Arh2 file. Currently only support zlib compression.\n");
#else
TEST_EXIT(cps == "null")
("HAVE_COMPRESSION OFF. Cannot read compressed Arh2 file.\n");
#endif
TEST_EXIT(dow == (unsigned)mesh->getGeo(WORLD))
("Dow is not match!\n");
TEST_EXIT(dim == (unsigned)mesh->getDim())
......@@ -238,6 +254,7 @@ namespace AMDiS { namespace io {
vector<string> vecsName(0);
vector<int> vecsFeSpaceNum(0);
vector<int> macroElIndex(nMacroElements);
vector<int> macroElSize(nMacroElements);
vector<vector<int> > feSpaceDOFs(0);
vector<int> perDOFs(4, 0);
......@@ -247,8 +264,7 @@ namespace AMDiS { namespace io {
for(unsigned int i = 0; i < nMacroElements; i++)
{
file.read(reinterpret_cast<char*>(&macroElIndex[i]), 4);
uint32_t elpos = 0;
file.read(reinterpret_cast<char*>(&elpos), 4);
file.read(reinterpret_cast<char*>(&macroElSize[i]), 4);
}
// Read feSpace table
for(unsigned int i = 0; i < nFeSpaces; i++)
......@@ -338,14 +354,30 @@ namespace AMDiS { namespace io {
// Read data: meshstructure and dof values
for (unsigned int i = 0; i < nMacroElements; i++) {
uint32_t nStructureCodes = 0;
stringstream dataStream(ios::out | ios::in | ios::binary);
int size = macroElSize[i];
char* buffer = new char[size];
file.read(buffer, size);
dataStream.write(buffer, size);
delete[] buffer;
#ifdef HAVE_COMPRESSION
if(cps == "zlib") {
stringstream tmp(ios::out | ios::in);
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::zlib_decompressor());
in.push(dataStream);
boost::iostreams::copy(in, tmp);
dataStream.str(tmp.str());
}
#endif
uint32_t nStructureCodes = 0;
uint32_t codeSize = 0;
file.read(reinterpret_cast<char*>(&nStructureCodes), 4);
file.read(reinterpret_cast<char*>(&codeSize), 4);
dataStream.read(reinterpret_cast<char*>(&nStructureCodes), 4);
dataStream.read(reinterpret_cast<char*>(&codeSize), 4);
vector<uint64_t> structureCode(nStructureCodes);
file.read(reinterpret_cast<char*>(&(structureCode[0])), 8 * nStructureCodes);
dataStream.read(reinterpret_cast<char*>(&(structureCode[0])), 8 * nStructureCodes);
MeshStructure elementStructure;
elementStructure.init(structureCode, codeSize);
......@@ -358,11 +390,11 @@ namespace AMDiS { namespace io {
for(unsigned int j = 0; j < nFeSpaces; j++)
{
uint32_t nValuesPerVector = 0;
file.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
dataStream.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
for(size_t k = 0; k < sortedFeSpaces[j].size(); k++)
{
values[valuePos + k].resize(nValuesPerVector);
file.read(reinterpret_cast<char*>(&(values[valuePos + k][0])), 8 * nValuesPerVector);
dataStream.read(reinterpret_cast<char*>(&(values[valuePos + k][0])), 8 * nValuesPerVector);
}
valuePos += sortedFeSpaces[j].size();
}
......
......@@ -9,6 +9,12 @@
#include "DOFVector.h"
#include "../Arh2Writer.h"
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#ifdef HAVE_COMPRESSION
#include <boost/iostreams/filter/zlib.hpp>
#endif
namespace AMDiS { namespace io {
using namespace std;
......@@ -139,12 +145,15 @@ namespace AMDiS { namespace io {
uint32_t dow = mesh->getGeo(WORLD);
uint32_t dim = mesh->getDim();
uint32_t headerLen = 30 + //fixed part of header
uint32_t headerLen = 34 + //fixed part of header
nMacroElements * 8 + //macroElemnts table
nFeSpaces * 16 + //feSpaces table
namesLen + //value vector table
nValueVectors * 8; //also value vector table
string typeId = "arh2";
string typeId = "arh2", cps("null");
#ifdef HAVE_COMPRESSION
cps = "zlib";
#endif
uint8_t *major = const_cast<uint8_t*>(&(AMDiS::io::Arh2Writer::MAJOR));
uint8_t *minor = const_cast<uint8_t*>(&(AMDiS::io::Arh2Writer::MINOR));
......@@ -158,7 +167,7 @@ namespace AMDiS { namespace io {
file.write(reinterpret_cast<char*>(&nFeSpaces), 4);
file.write(reinterpret_cast<char*>(&nValueVectors), 4);
file.write(reinterpret_cast<char*>(&nMacroElements), 4);
file.write(cps.c_str(), 4);
//macro table
deque<MacroElement*>::const_iterator macroIter = mesh->firstMacroElement();
while(macroIter != mesh->endOfMacroElements())
......@@ -184,7 +193,7 @@ namespace AMDiS { namespace io {
posDOFs = (*nDOF)[j];
file.write(reinterpret_cast<char*>(&posDOFs), 4);
}
for(int j = nDOF->size(); j < 4 ; j++)
for(size_t j = nDOF->size(); j <