Skip to content

GitLab

Commit 8d1f6a40 authored by Siqi Ling's avatar Siqi Ling
Browse files
Options

A fast implementation for readMetaFromSgArh of Arh2Reader

parent a559d671
Hide whitespace changes
Inline Side-by-side
Showing with 123 additions and 143 deletions
AMDiS/src/io/Arh2Reader.cc
View file @ 8d1f6a40
......@@ -361,42 +361,25 @@ namespace AMDiS { namespace io {
int nProcs = -1;
#endif
std::vector<std::set<std::pair<int, int> > > data(nProcs);
// data format: (rank ; (elIndex; elCodeSize) )
// check for consistency
// That is, if there are no more or less files as nProcs
int i=0;
for (; i < nProcs+1; i++) {
int i = 0;
for (; i < nProcs + 1; i++) {
string fn = filename + "-p" + boost::lexical_cast<string>(i) + "-.arh";
if(!boost::filesystem::exists(fn)) break;
}
TEST_EXIT(i == nProcs)
("Number of arh files doesn't match number of processors \n");
// data format: (rank ; (elIndex; elCodeSize) )
vector<std::set<pair<int, int> > > data(nProcs);
//collect data
for (int i = 0; i < nProcs; i++) {
string fn = filename + "-p" + boost::lexical_cast<string>(i) + "-.arh";
ifstream file;
file.open(fn.c_str(), ios::in | ios::binary);
if (!file.is_open()) {
cout << "Cannot open file \"" << fn << "\"\n";
exit(0);
}
string typeId(4, ' ');
file.read(const_cast<char*>(typeId.data()), 4);
if(typeId == "arh2")
readMetaFromSgArh(file, typeId, i, data);
else
{
cout << "file \"" << fn << "\"is not arh file.\n";
cout << "should not happen.\n";
exit(0);
}
file.close();
detail::readMetaFromSgArh(fn, i, data);
}
//make elInRank and elCodeSize-Map
for (int i = 0; i < nProcs; i++) {
for (std::set<pair<int, int> >::iterator it = data[i].begin();
......@@ -407,117 +390,6 @@ namespace AMDiS { namespace io {
}
return nProcs;
}
void readMetaFromSgArh(std::ifstream& file, string typeId, int nProc,
std::vector< std::set<std::pair<int, int> > >& data)
{
uint8_t major = 0, minor = 0;
uint32_t dow = 0, dim = 0;
uint32_t headerLen = 0;
uint32_t nMacroElements = 0;
uint32_t nValueVectors = 0;
uint32_t nFeSpaces = 0;
string cps(4, ' ');
file.read(reinterpret_cast<char*>(&major), 1);
file.read(reinterpret_cast<char*>(&minor), 1);
file.read(reinterpret_cast<char*>(&headerLen), 4);
file.read(reinterpret_cast<char*>(&dow), 4);
file.read(reinterpret_cast<char*>(&dim), 4);
file.read(reinterpret_cast<char*>(&nFeSpaces), 4);
file.read(reinterpret_cast<char*>(&nValueVectors), 4);
file.read(reinterpret_cast<char*>(&nMacroElements), 4);
switch(minor) {
case 1:
file.read(const_cast<char*>(cps.data()), 4);
break;
default:
cps = "null";break;
}
vector<int> macroElIndex(nMacroElements);
vector<int> macroElSize(nMacroElements);
vector<vector<int> > sortedFeSpaces(nFeSpaces);
// Read macro table
for(int i = 0; i < nMacroElements; i++)
{
file.read(reinterpret_cast<char*>(&macroElIndex[i]), 4);
uint32_t elpos = 0;
file.read(reinterpret_cast<char*>(&macroElSize[i]), 4);
}
// Read feSpace table
for(int i = 0; i < nFeSpaces; i++)
{
for(int j = 0; j < 4; j++)
{
uint32_t perDOFs = 0;
file.read(reinterpret_cast<char*>(&perDOFs), 4);
}
}
// Read value table
for(int i = 0; i < nValueVectors; i++)
{
string tmpString("");
uint32_t tmpInt = 0;
file.read(reinterpret_cast<char*>(&tmpInt), 4);
tmpString.resize(tmpInt, ' ');
file.read(const_cast<char*>(tmpString.data()), tmpInt); //
file.read(reinterpret_cast<char*>(&tmpInt), 4);
sortedFeSpaces[tmpInt].push_back(i);
}
for (unsigned int i = 0; i < nMacroElements; i++) {
stringstream dataStream(ios::out | ios::in | ios::binary);
int size = macroElSize[i];
char* buffer = new char[size];
file.read(buffer, size);
dataStream.write(buffer, size);
delete[] buffer;
if(cps == "zlib") {
stringstream tmp(ios::out | ios::in);
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::zlib_decompressor());
in.push(dataStream);
boost::iostreams::copy(in, tmp);
dataStream.str(tmp.str());
}
uint32_t nStructureCodes = 0;
uint32_t codeSize = 0;
dataStream.read(reinterpret_cast<char*>(&nStructureCodes), 4);
dataStream.read(reinterpret_cast<char*>(&codeSize), 4);
cout << " -> macro el " << macroElIndex[i] << "\n";
data[nProc].insert(make_pair(macroElIndex[i], codeSize));
vector<uint64_t> structureCode(nStructureCodes);
dataStream.read(reinterpret_cast<char*>(&(structureCode[0])), 8 * nStructureCodes);
if (nValueVectors > 0) {
vector<vector<double> > values(nValueVectors);
int valuePos = 0;
for(unsigned int j = 0; j < nFeSpaces; j++)
{
uint32_t nValuesPerVector = 0;
dataStream.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
for(unsigned k = 0; k < sortedFeSpaces[j].size(); k++)
{
values[valuePos + k].resize(nValuesPerVector);
dataStream.read(reinterpret_cast<char*>(&(values[valuePos + k][0])), 8 * nValuesPerVector);
}
valuePos += sortedFeSpaces[j].size();
}
}
}
}
} // end namespace Arh2Reader
} } // end namespace io, AMDiS
AMDiS/src/io/Arh2Reader.h
View file @ 8d1f6a40
......@@ -188,13 +188,7 @@ namespace AMDiS { namespace io {
int readMetaFromArh(std::string filename,
std::map<int, int> &elInRank,
std::map<int, int> &elCodeSize);
/// read meta data from a single ARH-file (just copied from arhTools
/// (readMetaFromArh2) )
void readMetaFromSgArh(std::ifstream& file, std::string typeId, int nProc,
std::vector< std::set<std::pair<int, int> > >& data);
/// Only returns just the number of subdomains a meta ARH file is defined for.
int readMetaData(std::string filename);
......
AMDiS/src/io/detail/Arh2Reader.cc
View file @ 8d1f6a40
......@@ -485,7 +485,117 @@ namespace AMDiS { namespace io {
}
MSG("ARH file read from: %s\n", filename.c_str());
}
} //end namespace detail
void readMetaFromSgArh(std::string filename, int nProc,
std::vector< std::set<std::pair<int, int> > >& data)
{
ifstream file;
file.open(filename.c_str(), ios::in | ios::binary);
TEST_EXIT(file.is_open())
("Cannot open file %s\n", filename.c_str());
file.seekg (0, file.end);
int fileSize = file.tellg();
file.seekg (0, file.beg);
uint32_t dow = 0, dim = 0;
uint32_t headerLen = 0;
uint32_t nMacroElements = 0;
uint32_t nValueVectors = 0;
uint32_t nFeSpaces = 0;
string cps = "null";
uint8_t minor = firstRead(file);
file.read(reinterpret_cast<char*>(&headerLen), 4);
file.read(reinterpret_cast<char*>(&dow), 4);
file.read(reinterpret_cast<char*>(&dim), 4);
file.read(reinterpret_cast<char*>(&nFeSpaces), 4);
file.read(reinterpret_cast<char*>(&nValueVectors), 4);
file.read(reinterpret_cast<char*>(&nMacroElements), 4);
if(minor > 0)
file.read(const_cast<char*>(cps.data()), 4);
#ifdef HAVE_COMPRESSION
TEST_EXIT(cps == "null" || cps == "zlib")
("Cannot read Arh2 file. Currently only support zlib compression.\n");
#else
TEST_EXIT(cps == "null")
("HAVE_COMPRESSION OFF. Cannot read compressed Arh2 file.\n");
#endif
vector<int> macroElIndex(nMacroElements);
vector<int> macroElSize(nMacroElements);
vector<vector<int> > sortedFeSpaces(nFeSpaces);
// Read macro table
for(int i = 0; i < nMacroElements; i++) {
file.read(reinterpret_cast<char*>(&macroElIndex[i]), 4);
file.read(reinterpret_cast<char*>(&macroElSize[i]), 4);
}
// Read feSpace table
file.seekg(nFeSpaces * 4 * 4, ios_base::cur);
// Read value table
for(int i = 0; i < nValueVectors; i++) {
string tmpString("");
uint32_t tmpInt = 0;
file.read(reinterpret_cast<char*>(&tmpInt), 4);
tmpString.resize(tmpInt, ' ');
file.read(const_cast<char*>(tmpString.data()), tmpInt); //
file.read(reinterpret_cast<char*>(&tmpInt), 4);
sortedFeSpaces[tmpInt].push_back(i);
}
for (unsigned int i = 0; i < nMacroElements; i++) {
stringstream dataStream(ios::out | ios::in | ios::binary);
int size = 0;
if(minor > 0)
size = macroElSize[i];
else
size = (i != nMacroElements - 1) ? macroElSize[i + 1] - macroElSize[i] : fileSize - macroElSize[i];
char* buffer = new char[size];
file.read(buffer, size);
dataStream.write(buffer, size);
delete[] buffer;
#ifdef HAVE_COMPRESSION
if(cps == "zlib") {
stringstream tmp(ios::out | ios::in);
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::zlib_decompressor());
in.push(dataStream);
boost::iostreams::copy(in, tmp);
dataStream.str(tmp.str());
}
#endif
uint32_t nStructureCodes = 0;
uint32_t codeSize = 0;
dataStream.read(reinterpret_cast<char*>(&nStructureCodes), 4);
dataStream.read(reinterpret_cast<char*>(&codeSize), 4);
data[nProc].insert(make_pair(macroElIndex[i], codeSize));
// We only need codeSize for each macro element data block, so skip all of the unnecessary.
dataStream.seekg(8 * nStructureCodes, ios_base::cur);
if (nValueVectors > 0) {
for(unsigned int j = 0; j < nFeSpaces; j++) {
uint32_t nValuesPerVector = 0;
dataStream.read(reinterpret_cast<char*>(&nValuesPerVector), 4);
dataStream.seekg(sortedFeSpaces[j].size() * 8 * nValuesPerVector, ios_base::cur);
}
}
}
} // end readMetaFromSgArh
} // end namespace detail
} // end namespace Arh2Reader
} } // end namespace io, AMDiS
......
AMDiS/src/io/detail/Arh2Reader.h
View file @ 8d1f6a40
......@@ -67,6 +67,10 @@ namespace AMDiS { namespace io {
int nProcs = -1,
bool byName = false);
/// read meta data from a single ARH-file
void readMetaFromSgArh(std::string filename, int nProc,
std::vector< std::set<std::pair<int, int> > >& data);
}//end namespace detail
} // end namespace Arh2Reader
} } // end namespace io, AMDiS
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment