Commit d43c997f authored by Siqi Ling's avatar Siqi Ling
Browse files

vtu writer write only raw, vtu reader read raw and base64

parent 41f5850e
...@@ -207,7 +207,7 @@ namespace AMDiS { namespace io { ...@@ -207,7 +207,7 @@ namespace AMDiS { namespace io {
string typeId(4, ' '); string typeId(4, ' ');
uint8_t major = 0, minor = 0; uint8_t major = 0, minor = 0;
file.read(const_cast<char*>(typeId.data()), 4); file.read(&typeId[0], 4);
file.read(reinterpret_cast<char*>(&major), 1); file.read(reinterpret_cast<char*>(&major), 1);
file.read(reinterpret_cast<char*>(&minor), 1); file.read(reinterpret_cast<char*>(&minor), 1);
......
...@@ -58,7 +58,8 @@ namespace AMDiS ...@@ -58,7 +58,8 @@ namespace AMDiS
TEST_EXIT(boost::filesystem::exists(filename))((filename + " does not exist!\n").c_str()); TEST_EXIT(boost::filesystem::exists(filename))((filename + " does not exist!\n").c_str());
xml_document vtu; xml_document vtu;
TEST_EXIT(vtu.load_file(filename.c_str()))("Could not load vtu file! Error in xml structure.\n"); // TEST_EXIT(vtu.load_file(filename.c_str()))("Could not load vtu file! Error in xml structure.\n");
vtu.load_file(filename.c_str());
xml_node VTKFile = vtu.child("VTKFile"); xml_node VTKFile = vtu.child("VTKFile");
string zlib = VTKFile.attribute("compressor").value(); string zlib = VTKFile.attribute("compressor").value();
...@@ -98,14 +99,7 @@ namespace AMDiS ...@@ -98,14 +99,7 @@ namespace AMDiS
} }
} }
} else { } else {
string encoding = AppendedData.attribute("encoding").value(); // Get offset of all data array
TEST_EXIT(encoding == "base64")
("Currently the encoding of AppendedData only supports base64. But it's easy to extend to raw.\n");
string appendedData = AppendedData.last_child().value();
int start = appendedData.find("_");
appendedData = appendedData.substr(start + 1, appendedData.length() - 1);
vector<int> offsetVec; vector<int> offsetVec;
int pointsIndex = 0, index = 0; int pointsIndex = 0, index = 0;
vector<pair<xml_node, int> > pointDataIndex; vector<pair<xml_node, int> > pointDataIndex;
...@@ -121,6 +115,45 @@ namespace AMDiS ...@@ -121,6 +115,45 @@ namespace AMDiS
} }
} }
// Get appended data
string encoding = AppendedData.attribute("encoding").value();
string appendedData = "";
if (encoding == "base64") {
appendedData = AppendedData.last_child().value();
int start = appendedData.find("_");
appendedData = appendedData.substr(start + 1);
} else if (encoding == "raw") {
// Binary AppendedData block cannot be regonized by pugixml. So we have to
// extract the data manually.
ifstream file(filename.c_str());
TEST_EXIT(file.is_open())("Cannot open file: %s\n", filename.c_str());
long startPos = 0, endPos = 0;
string line = "";
while (getline(file, line))
if (line.find("<AppendedData") != std::string::npos)
break;
TEST_EXIT_DBG(!file.eof()) ("Should not happen.\n");
startPos = file.tellg();
file.seekg(offsetVec[offsetVec.size() - 1], ios_base::cur);
while (getline(file, line))
if (line.find("</AppendedData") != std::string::npos)
break;
TEST_EXIT_DBG(!file.eof()) ("Should not happen.\n");
file.seekg(-line.length(), ios_base::cur);
endPos = file.tellg();
appendedData.resize(endPos - startPos);
file.seekg(startPos);
file.read(&appendedData[0], endPos - startPos);
int start = appendedData.find("_");
appendedData = appendedData.substr(start + 1);
} else
ERROR_EXIT("Currently the encoding of AppendedData only supports base64 or raw.\n");
// Read appended data
string format = PointsDataArray.attribute("format").value(); string format = PointsDataArray.attribute("format").value();
string type = PointsDataArray.attribute("type").value(); string type = PointsDataArray.attribute("type").value();
TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n"); TEST_EXIT(format == "appended")("The format of DataArray is not appended in appended mode. Should not happen.\n");
...@@ -131,7 +164,11 @@ namespace AMDiS ...@@ -131,7 +164,11 @@ namespace AMDiS
offsetVec[pointsIndex + 1] - offsetVec[pointsIndex]; offsetVec[pointsIndex + 1] - offsetVec[pointsIndex];
string points = appendedData.substr(offsetVec[pointsIndex], len); string points = appendedData.substr(offsetVec[pointsIndex], len);
detail::binary2pointList(points, type, (zlib != ""), pointList); detail::binary2pointList(points,
type,
(zlib != ""),
encoding == "base64",
pointList);
T test; T test;
...@@ -155,7 +192,12 @@ namespace AMDiS ...@@ -155,7 +192,12 @@ namespace AMDiS
nComponents = pointDataIndex[i].first.attribute("NumberOfComponents").as_int(); nComponents = pointDataIndex[i].first.attribute("NumberOfComponents").as_int();
TEST_EXIT(nComponents == -1 || static_cast<int>(size(test)) <= nComponents) TEST_EXIT(nComponents == -1 || static_cast<int>(size(test)) <= nComponents)
("Can not read values in DOFVector with given value type. Too many components!\n"); ("Can not read values in DOFVector with given value type. Too many components!\n");
detail::binary2valueList(values, type, (zlib != ""), valueList[j], size(test), nComponents); detail::binary2valueList(values, type,
(zlib != ""),
encoding == "base64",
valueList[j],
size(test),
nComponents);
break; break;
} }
} }
......
...@@ -47,7 +47,7 @@ namespace AMDiS { namespace io { ...@@ -47,7 +47,7 @@ namespace AMDiS { namespace io {
uint8_t minor_ = 0; uint8_t minor_ = 0;
string typeId(4, ' '); string typeId(4, ' ');
file.read(const_cast<char*>(typeId.data()), 4); file.read(&typeId[0], 4);
file.read(reinterpret_cast<char*>(&major_), 1); file.read(reinterpret_cast<char*>(&major_), 1);
file.read(reinterpret_cast<char*>(&minor_), 1); file.read(reinterpret_cast<char*>(&minor_), 1);
...@@ -298,7 +298,7 @@ namespace AMDiS { namespace io { ...@@ -298,7 +298,7 @@ namespace AMDiS { namespace io {
{ {
uint32_t tmpInt = 0; uint32_t tmpInt = 0;
file.read(reinterpret_cast<char*>(&tmpInt), 4); file.read(reinterpret_cast<char*>(&tmpInt), 4);
file.read(const_cast<char*>(AFEDfileName[i].data()), tmpInt); file.read(&AFEDfileName[i][0], tmpInt);
for(int j = 0; j < 4; j++) for(int j = 0; j < 4; j++)
{ {
file.read(reinterpret_cast<char*>(&perDOFs[j]), 4); file.read(reinterpret_cast<char*>(&perDOFs[j]), 4);
...@@ -313,13 +313,13 @@ namespace AMDiS { namespace io { ...@@ -313,13 +313,13 @@ namespace AMDiS { namespace io {
file.read(reinterpret_cast<char*>(&tmpInt), 4); file.read(reinterpret_cast<char*>(&tmpInt), 4);
vecsNameLen.push_back(tmpInt); vecsNameLen.push_back(tmpInt);
tmpString.resize(tmpInt, ' '); tmpString.resize(tmpInt, ' ');
file.read(const_cast<char*>(tmpString.data()), tmpInt); // file.read(&tmpString[0], tmpInt); //
vecsName.push_back(tmpString); vecsName.push_back(tmpString);
file.read(reinterpret_cast<char*>(&tmpInt), 4); file.read(reinterpret_cast<char*>(&tmpInt), 4);
sortedFeSpaces[tmpInt].push_back(i); sortedFeSpaces[tmpInt].push_back(i);
vecsFeSpaceNum.push_back(tmpInt); vecsFeSpaceNum.push_back(tmpInt);
tmpString.resize(4, ' '); tmpString.resize(4, ' ');
file.read(const_cast<char*>(tmpString.data()), 4); file.read(&tmpString[0], 4);
dataformat.push_back(tmpString); dataformat.push_back(tmpString);
} }
...@@ -329,7 +329,7 @@ namespace AMDiS { namespace io { ...@@ -329,7 +329,7 @@ namespace AMDiS { namespace io {
if (macroFile_nl > 0) { if (macroFile_nl > 0) {
string tmpString(""); string tmpString("");
tmpString.resize(macroFile_nl, ' '); tmpString.resize(macroFile_nl, ' ');
file.read(const_cast<char*>(tmpString.data()), macroFile_nl); file.read(&tmpString[0], macroFile_nl);
} }
//================header is over==================// //================header is over==================//
...@@ -715,11 +715,11 @@ namespace AMDiS { namespace io { ...@@ -715,11 +715,11 @@ namespace AMDiS { namespace io {
uint32_t tmpInt = 0; uint32_t tmpInt = 0;
file.read(reinterpret_cast<char*>(&tmpInt), 4); file.read(reinterpret_cast<char*>(&tmpInt), 4);
tmpString.resize(tmpInt, ' '); tmpString.resize(tmpInt, ' ');
file.read(const_cast<char*>(tmpString.data()), tmpInt); // file.read(&tmpString[0], tmpInt); //
file.read(reinterpret_cast<char*>(&tmpInt), 4); file.read(reinterpret_cast<char*>(&tmpInt), 4);
sortedFeSpaces[tmpInt].push_back(i); sortedFeSpaces[tmpInt].push_back(i);
tmpString.resize(4, ' '); tmpString.resize(4, ' ');
file.read(const_cast<char*>(tmpString.data()), 4); file.read(&tmpString[0], 4);
dataformat.push_back(tmpString); dataformat.push_back(tmpString);
} }
...@@ -727,7 +727,7 @@ namespace AMDiS { namespace io { ...@@ -727,7 +727,7 @@ namespace AMDiS { namespace io {
uint32_t macroFile_nl = 0; uint32_t macroFile_nl = 0;
file.read(reinterpret_cast<char*>(&macroFile_nl), 4); file.read(reinterpret_cast<char*>(&macroFile_nl), 4);
string tmpString(""); string tmpString("");
file.read(const_cast<char*>(tmpString.data()), macroFile_nl); file.read(&tmpString[0], macroFile_nl);
//================header is over==================// //================header is over==================//
...@@ -843,7 +843,7 @@ namespace AMDiS { namespace io { ...@@ -843,7 +843,7 @@ namespace AMDiS { namespace io {
("Cannot open file %s\n", filename.c_str()); ("Cannot open file %s\n", filename.c_str());
string fd(16, ' '); string fd(16, ' ');
file.read(const_cast<char*>(fd.data()), 16); file.read(&fd[0], 16);
string Id = fd.substr(0, 4); string Id = fd.substr(0, 4);
int major = boost::lexical_cast<int>(fd.substr(5, 1)); int major = boost::lexical_cast<int>(fd.substr(5, 1));
int minor = boost::lexical_cast<int>(fd.substr(7, 1)); int minor = boost::lexical_cast<int>(fd.substr(7, 1));
...@@ -854,12 +854,12 @@ namespace AMDiS { namespace io { ...@@ -854,12 +854,12 @@ namespace AMDiS { namespace io {
file.read(reinterpret_cast<char*>(&baseDirLen), 4); file.read(reinterpret_cast<char*>(&baseDirLen), 4);
baseDir.resize(baseDirLen, ' '); baseDir.resize(baseDirLen, ' ');
file.read(const_cast<char*>(baseDir.data()), baseDirLen); file.read(&baseDir[0], baseDirLen);
file.read(reinterpret_cast<char*>(&nFiles), 4); file.read(reinterpret_cast<char*>(&nFiles), 4);
file.read(reinterpret_cast<char*>(&macroFile_nl), 4); file.read(reinterpret_cast<char*>(&macroFile_nl), 4);
if (macroFile_nl > 0) { //TODO if (macroFile_nl > 0) { //TODO
macroFilename.resize(macroFile_nl, ' '); macroFilename.resize(macroFile_nl, ' ');
file.read(const_cast<char*>(macroFilename.data()), macroFile_nl); file.read(&macroFilename[0], macroFile_nl);
} }
file.read(reinterpret_cast<char*>(&nMacros), 4); file.read(reinterpret_cast<char*>(&nMacros), 4);
......
...@@ -108,7 +108,7 @@ namespace AMDiS ...@@ -108,7 +108,7 @@ namespace AMDiS
} }
#endif #endif
inline std::string getInnerDataArray(std::string& input, bool zlib) inline std::string getInnerDataArray(std::string& input, bool zlib, bool base64)
{ {
FUNCNAME("VtkReader::detail::getInnerDataArray()"); FUNCNAME("VtkReader::detail::getInnerDataArray()");
...@@ -122,16 +122,20 @@ namespace AMDiS ...@@ -122,16 +122,20 @@ namespace AMDiS
if(zlib) { if(zlib) {
#ifdef HAVE_COMPRESSION #ifdef HAVE_COMPRESSION
string s = input.substr(0, 8); string s = input.substr(0, 8);
if (base64)
s = detail::base64ToStr(s); s = detail::base64ToStr(s);
tmp = const_cast<char*>(s.c_str()); tmp = const_cast<char*>(input.c_str());
ptr = reinterpret_cast<int*>(tmp); ptr = reinterpret_cast<int*>(tmp);
int nBlocks = *ptr; int nBlocks = *ptr;
int headerSize = (((4 * nBlocks + 12) % 3) > 0) ? 4 * ((4 * nBlocks + 12) / 3 + 1) : 4 * ((4 * nBlocks + 12) / 3); int headerSize = (base64) ? (((4 * nBlocks + 12) % 3) > 0) ? 4 * ((4 * nBlocks + 12) / 3 + 1) : 4 * ((4 * nBlocks + 12) / 3)
: 4 * nBlocks + 12;
header = input.substr(0, headerSize); header = input.substr(0, headerSize);
body = input.substr(headerSize); body = input.substr(headerSize);
if (base64) {
header = detail::base64ToStr(header); header = detail::base64ToStr(header);
body = detail::base64ToStr(body); body = detail::base64ToStr(body);
}
int blockSize, finalSize, offset = 0; int blockSize, finalSize, offset = 0;
tmp = const_cast<char*>(header.c_str()); tmp = const_cast<char*>(header.c_str());
...@@ -150,7 +154,7 @@ namespace AMDiS ...@@ -150,7 +154,7 @@ namespace AMDiS
ERROR_EXIT("HAVE_COMPRESSION OFF. VtkReader cannot read APPENDED_COMPRESSED vtu files.\n"); ERROR_EXIT("HAVE_COMPRESSION OFF. VtkReader cannot read APPENDED_COMPRESSED vtu files.\n");
#endif #endif
} else { } else {
header = detail::base64ToStr(input); header = (base64) ? detail::base64ToStr(input) : input;
tmp = const_cast<char*>(header.c_str()); tmp = const_cast<char*>(header.c_str());
ptr = reinterpret_cast<int*>(tmp); ptr = reinterpret_cast<int*>(tmp);
nBytes = *ptr; nBytes = *ptr;
...@@ -163,12 +167,13 @@ namespace AMDiS ...@@ -163,12 +167,13 @@ namespace AMDiS
inline void binary2pointList(std::string& input, inline void binary2pointList(std::string& input,
std::string type, std::string type,
bool zlib, bool zlib,
bool base64,
std::vector<WorldVector<double> >& pointList) std::vector<WorldVector<double> >& pointList)
{ {
int dow = Global::getGeo(WORLD); int dow = Global::getGeo(WORLD);
int dowExplicit = 3; int dowExplicit = 3;
std::string inner = getInnerDataArray(input, zlib); std::string inner = getInnerDataArray(input, zlib, base64);
int nBytes = inner.length(); int nBytes = inner.length();
char* tmp = const_cast<char*>(inner.c_str()); char* tmp = const_cast<char*>(inner.c_str());
...@@ -232,6 +237,7 @@ namespace AMDiS ...@@ -232,6 +237,7 @@ namespace AMDiS
void binary2valueList(std::string& input, void binary2valueList(std::string& input,
std::string type, std::string type,
bool zlib, bool zlib,
bool base64,
std::vector<T>& valueList, std::vector<T>& valueList,
int numComponent = 1, int numComponent = 1,
int numComponentMax = -1) int numComponentMax = -1)
...@@ -239,7 +245,7 @@ namespace AMDiS ...@@ -239,7 +245,7 @@ namespace AMDiS
if (numComponentMax < 0) if (numComponentMax < 0)
numComponentMax = numComponent; numComponentMax = numComponent;
std::string inner = getInnerDataArray(input, zlib); std::string inner = getInnerDataArray(input, zlib, base64);
int nBytes = inner.length(); int nBytes = inner.length();
char* tmp = const_cast<char*>(inner.c_str()); char* tmp = const_cast<char*>(inner.c_str());
......
...@@ -75,14 +75,14 @@ namespace AMDiS { namespace io { ...@@ -75,14 +75,14 @@ namespace AMDiS { namespace io {
using namespace std; using namespace std;
string result(""), header(""), body(""); string result(""), header(""), body("");
switch(format) { switch(format) {
case APPENDED: { case APPENDED: {
BinaryStream hstream; BinaryStream hstream;
hstream << bstream.getSize(); hstream << bstream.getSize();
body = bstream.str(); body = bstream.str();
header = hstream.str(); header = hstream.str();
result = detail::base64Encode(header + body); // result = detail::base64Encode(header + body);
result = header + body;
break; break;
} }
case APPENDED_COMPRESSED: { case APPENDED_COMPRESSED: {
...@@ -109,7 +109,8 @@ namespace AMDiS { namespace io { ...@@ -109,7 +109,8 @@ namespace AMDiS { namespace io {
body += subData; body += subData;
} }
header = hstream.str(); header = hstream.str();
result = detail::base64Encode(header) + detail::base64Encode(body); // result = detail::base64Encode(header) + detail::base64Encode(body);
result = header + body;
#endif #endif
break; break;
} }
...@@ -124,13 +125,13 @@ namespace AMDiS { namespace io { ...@@ -124,13 +125,13 @@ namespace AMDiS { namespace io {
namespace detail namespace detail
{ {
std::string base64Encode(std::string text) // std::string base64Encode(std::string text)
{ // {
unsigned int writePaddChars = (3-text.length()%3)%3; // unsigned int writePaddChars = (3-text.length()%3)%3;
std::string base64(binary_base64(text.begin()),binary_base64(text.end())); // std::string base64(binary_base64(text.begin()),binary_base64(text.end()));
base64.append(writePaddChars,'='); // base64.append(writePaddChars,'=');
return base64; // return base64;
} // }
std::string extract_relative_path(std::string valueFilename, std::string animationFilename) std::string extract_relative_path(std::string valueFilename, std::string animationFilename)
{ {
......
...@@ -28,8 +28,8 @@ ...@@ -28,8 +28,8 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <sstream> #include <sstream>
#include <boost/archive/iterators/base64_from_binary.hpp> // #include <boost/archive/iterators/base64_from_binary.hpp>
#include <boost/archive/iterators/transform_width.hpp> // #include <boost/archive/iterators/transform_width.hpp>
#include "AdaptInfo.h" #include "AdaptInfo.h"
#include "io/DataCollector.h" #include "io/DataCollector.h"
...@@ -272,14 +272,14 @@ namespace AMDiS { namespace io { ...@@ -272,14 +272,14 @@ namespace AMDiS { namespace io {
namespace detail namespace detail
{ {
typedef // typedef
boost::archive::iterators::base64_from_binary< // convert binary values to base64 characters // boost::archive::iterators::base64_from_binary< // convert binary values to base64 characters
boost::archive::iterators::transform_width< // retrieve 6 bit integers from a sequence of 8 bit bytes // boost::archive::iterators::transform_width< // retrieve 6 bit integers from a sequence of 8 bit bytes
std::string::const_iterator, 6, 8 // std::string::const_iterator, 6, 8
> // >
> binary_base64; // > binary_base64;
//
std::string base64Encode(std::string text); // std::string base64Encode(std::string text);
std::string extract_relative_path(std::string valueFilename, std::string animationFilename); std::string extract_relative_path(std::string valueFilename, std::string animationFilename);
......
...@@ -232,7 +232,8 @@ namespace AMDiS { namespace io { ...@@ -232,7 +232,8 @@ namespace AMDiS { namespace io {
file << " </Cells>\n"; file << " </Cells>\n";
file << " </Piece>\n"; file << " </Piece>\n";
file << " </UnstructuredGrid>\n"; file << " </UnstructuredGrid>\n";
file << " <AppendedData encoding=\"base64\">\n"; // file << " <AppendedData encoding=\"base64\">\n";
file << " <AppendedData encoding=\"raw\">\n";
file << " _" << finalData << "\n"; file << " _" << finalData << "\n";
file << " </AppendedData>\n"; file << " </AppendedData>\n";
file << "</VTKFile>\n"; file << "</VTKFile>\n";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment