Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
Backofen, Rainer
amdis
Commits
8d1f6a40
Commit
8d1f6a40
authored
May 09, 2014
by
Siqi Ling
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
A fast implementation for readMetaFromSgArh of Arh2Reader
parent
a559d671
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
123 additions
and
143 deletions
+123
-143
AMDiS/src/io/Arh2Reader.cc
AMDiS/src/io/Arh2Reader.cc
+8
-136
AMDiS/src/io/Arh2Reader.h
AMDiS/src/io/Arh2Reader.h
+0
-6
AMDiS/src/io/detail/Arh2Reader.cc
AMDiS/src/io/detail/Arh2Reader.cc
+111
-1
AMDiS/src/io/detail/Arh2Reader.h
AMDiS/src/io/detail/Arh2Reader.h
+4
-0
No files found.
AMDiS/src/io/Arh2Reader.cc
View file @
8d1f6a40
...
...
@@ -361,42 +361,25 @@ namespace AMDiS { namespace io {
int
nProcs
=
-
1
;
#endif
std
::
vector
<
std
::
set
<
std
::
pair
<
int
,
int
>
>
>
data
(
nProcs
);
// data format: (rank ; (elIndex; elCodeSize) )
// check for consistency
// That is, if there are no more or less files as nProcs
int
i
=
0
;
for
(;
i
<
nProcs
+
1
;
i
++
)
{
int
i
=
0
;
for
(;
i
<
nProcs
+
1
;
i
++
)
{
string
fn
=
filename
+
"-p"
+
boost
::
lexical_cast
<
string
>
(
i
)
+
"-.arh"
;
if
(
!
boost
::
filesystem
::
exists
(
fn
))
break
;
}
TEST_EXIT
(
i
==
nProcs
)
(
"Number of arh files doesn't match number of processors
\n
"
);
// data format: (rank ; (elIndex; elCodeSize) )
vector
<
std
::
set
<
pair
<
int
,
int
>
>
>
data
(
nProcs
);
//collect data
for
(
int
i
=
0
;
i
<
nProcs
;
i
++
)
{
string
fn
=
filename
+
"-p"
+
boost
::
lexical_cast
<
string
>
(
i
)
+
"-.arh"
;
ifstream
file
;
file
.
open
(
fn
.
c_str
(),
ios
::
in
|
ios
::
binary
);
if
(
!
file
.
is_open
())
{
cout
<<
"Cannot open file
\"
"
<<
fn
<<
"
\"\n
"
;
exit
(
0
);
}
string
typeId
(
4
,
' '
);
file
.
read
(
const_cast
<
char
*>
(
typeId
.
data
()),
4
);
if
(
typeId
==
"arh2"
)
readMetaFromSgArh
(
file
,
typeId
,
i
,
data
);
else
{
cout
<<
"file
\"
"
<<
fn
<<
"
\"
is not arh file.
\n
"
;
cout
<<
"should not happen.
\n
"
;
exit
(
0
);
}
file
.
close
();
detail
::
readMetaFromSgArh
(
fn
,
i
,
data
);
}
//make elInRank and elCodeSize-Map
for
(
int
i
=
0
;
i
<
nProcs
;
i
++
)
{
for
(
std
::
set
<
pair
<
int
,
int
>
>::
iterator
it
=
data
[
i
].
begin
();
...
...
@@ -407,117 +390,6 @@ namespace AMDiS { namespace io {
}
return
nProcs
;
}
void
readMetaFromSgArh
(
std
::
ifstream
&
file
,
string
typeId
,
int
nProc
,
std
::
vector
<
std
::
set
<
std
::
pair
<
int
,
int
>
>
>&
data
)
{
uint8_t
major
=
0
,
minor
=
0
;
uint32_t
dow
=
0
,
dim
=
0
;
uint32_t
headerLen
=
0
;
uint32_t
nMacroElements
=
0
;
uint32_t
nValueVectors
=
0
;
uint32_t
nFeSpaces
=
0
;
string
cps
(
4
,
' '
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
major
),
1
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
minor
),
1
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
headerLen
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
dow
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
dim
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nFeSpaces
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nValueVectors
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nMacroElements
),
4
);
switch
(
minor
)
{
case
1
:
file
.
read
(
const_cast
<
char
*>
(
cps
.
data
()),
4
);
break
;
default:
cps
=
"null"
;
break
;
}
vector
<
int
>
macroElIndex
(
nMacroElements
);
vector
<
int
>
macroElSize
(
nMacroElements
);
vector
<
vector
<
int
>
>
sortedFeSpaces
(
nFeSpaces
);
// Read macro table
for
(
int
i
=
0
;
i
<
nMacroElements
;
i
++
)
{
file
.
read
(
reinterpret_cast
<
char
*>
(
&
macroElIndex
[
i
]),
4
);
uint32_t
elpos
=
0
;
file
.
read
(
reinterpret_cast
<
char
*>
(
&
macroElSize
[
i
]),
4
);
}
// Read feSpace table
for
(
int
i
=
0
;
i
<
nFeSpaces
;
i
++
)
{
for
(
int
j
=
0
;
j
<
4
;
j
++
)
{
uint32_t
perDOFs
=
0
;
file
.
read
(
reinterpret_cast
<
char
*>
(
&
perDOFs
),
4
);
}
}
// Read value table
for
(
int
i
=
0
;
i
<
nValueVectors
;
i
++
)
{
string
tmpString
(
""
);
uint32_t
tmpInt
=
0
;
file
.
read
(
reinterpret_cast
<
char
*>
(
&
tmpInt
),
4
);
tmpString
.
resize
(
tmpInt
,
' '
);
file
.
read
(
const_cast
<
char
*>
(
tmpString
.
data
()),
tmpInt
);
//
file
.
read
(
reinterpret_cast
<
char
*>
(
&
tmpInt
),
4
);
sortedFeSpaces
[
tmpInt
].
push_back
(
i
);
}
for
(
unsigned
int
i
=
0
;
i
<
nMacroElements
;
i
++
)
{
stringstream
dataStream
(
ios
::
out
|
ios
::
in
|
ios
::
binary
);
int
size
=
macroElSize
[
i
];
char
*
buffer
=
new
char
[
size
];
file
.
read
(
buffer
,
size
);
dataStream
.
write
(
buffer
,
size
);
delete
[]
buffer
;
if
(
cps
==
"zlib"
)
{
stringstream
tmp
(
ios
::
out
|
ios
::
in
);
boost
::
iostreams
::
filtering_streambuf
<
boost
::
iostreams
::
input
>
in
;
in
.
push
(
boost
::
iostreams
::
zlib_decompressor
());
in
.
push
(
dataStream
);
boost
::
iostreams
::
copy
(
in
,
tmp
);
dataStream
.
str
(
tmp
.
str
());
}
uint32_t
nStructureCodes
=
0
;
uint32_t
codeSize
=
0
;
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
nStructureCodes
),
4
);
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
codeSize
),
4
);
cout
<<
" -> macro el "
<<
macroElIndex
[
i
]
<<
"
\n
"
;
data
[
nProc
].
insert
(
make_pair
(
macroElIndex
[
i
],
codeSize
));
vector
<
uint64_t
>
structureCode
(
nStructureCodes
);
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
(
structureCode
[
0
])),
8
*
nStructureCodes
);
if
(
nValueVectors
>
0
)
{
vector
<
vector
<
double
>
>
values
(
nValueVectors
);
int
valuePos
=
0
;
for
(
unsigned
int
j
=
0
;
j
<
nFeSpaces
;
j
++
)
{
uint32_t
nValuesPerVector
=
0
;
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
nValuesPerVector
),
4
);
for
(
unsigned
k
=
0
;
k
<
sortedFeSpaces
[
j
].
size
();
k
++
)
{
values
[
valuePos
+
k
].
resize
(
nValuesPerVector
);
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
(
values
[
valuePos
+
k
][
0
])),
8
*
nValuesPerVector
);
}
valuePos
+=
sortedFeSpaces
[
j
].
size
();
}
}
}
}
}
// end namespace Arh2Reader
}
}
// end namespace io, AMDiS
AMDiS/src/io/Arh2Reader.h
View file @
8d1f6a40
...
...
@@ -188,13 +188,7 @@ namespace AMDiS { namespace io {
int
readMetaFromArh
(
std
::
string
filename
,
std
::
map
<
int
,
int
>
&
elInRank
,
std
::
map
<
int
,
int
>
&
elCodeSize
);
/// read meta data from a single ARH-file (just copied from arhTools
/// (readMetaFromArh2) )
void
readMetaFromSgArh
(
std
::
ifstream
&
file
,
std
::
string
typeId
,
int
nProc
,
std
::
vector
<
std
::
set
<
std
::
pair
<
int
,
int
>
>
>&
data
);
/// Only returns just the number of subdomains a meta ARH file is defined for.
int
readMetaData
(
std
::
string
filename
);
...
...
AMDiS/src/io/detail/Arh2Reader.cc
View file @
8d1f6a40
...
...
@@ -485,7 +485,117 @@ namespace AMDiS { namespace io {
}
MSG
(
"ARH file read from: %s
\n
"
,
filename
.
c_str
());
}
}
//end namespace detail
void
readMetaFromSgArh
(
std
::
string
filename
,
int
nProc
,
std
::
vector
<
std
::
set
<
std
::
pair
<
int
,
int
>
>
>&
data
)
{
ifstream
file
;
file
.
open
(
filename
.
c_str
(),
ios
::
in
|
ios
::
binary
);
TEST_EXIT
(
file
.
is_open
())
(
"Cannot open file %s
\n
"
,
filename
.
c_str
());
file
.
seekg
(
0
,
file
.
end
);
int
fileSize
=
file
.
tellg
();
file
.
seekg
(
0
,
file
.
beg
);
uint32_t
dow
=
0
,
dim
=
0
;
uint32_t
headerLen
=
0
;
uint32_t
nMacroElements
=
0
;
uint32_t
nValueVectors
=
0
;
uint32_t
nFeSpaces
=
0
;
string
cps
=
"null"
;
uint8_t
minor
=
firstRead
(
file
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
headerLen
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
dow
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
dim
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nFeSpaces
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nValueVectors
),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
nMacroElements
),
4
);
if
(
minor
>
0
)
file
.
read
(
const_cast
<
char
*>
(
cps
.
data
()),
4
);
#ifdef HAVE_COMPRESSION
TEST_EXIT
(
cps
==
"null"
||
cps
==
"zlib"
)
(
"Cannot read Arh2 file. Currently only support zlib compression.
\n
"
);
#else
TEST_EXIT
(
cps
==
"null"
)
(
"HAVE_COMPRESSION OFF. Cannot read compressed Arh2 file.
\n
"
);
#endif
vector
<
int
>
macroElIndex
(
nMacroElements
);
vector
<
int
>
macroElSize
(
nMacroElements
);
vector
<
vector
<
int
>
>
sortedFeSpaces
(
nFeSpaces
);
// Read macro table
for
(
int
i
=
0
;
i
<
nMacroElements
;
i
++
)
{
file
.
read
(
reinterpret_cast
<
char
*>
(
&
macroElIndex
[
i
]),
4
);
file
.
read
(
reinterpret_cast
<
char
*>
(
&
macroElSize
[
i
]),
4
);
}
// Read feSpace table
file
.
seekg
(
nFeSpaces
*
4
*
4
,
ios_base
::
cur
);
// Read value table
for
(
int
i
=
0
;
i
<
nValueVectors
;
i
++
)
{
string
tmpString
(
""
);
uint32_t
tmpInt
=
0
;
file
.
read
(
reinterpret_cast
<
char
*>
(
&
tmpInt
),
4
);
tmpString
.
resize
(
tmpInt
,
' '
);
file
.
read
(
const_cast
<
char
*>
(
tmpString
.
data
()),
tmpInt
);
//
file
.
read
(
reinterpret_cast
<
char
*>
(
&
tmpInt
),
4
);
sortedFeSpaces
[
tmpInt
].
push_back
(
i
);
}
for
(
unsigned
int
i
=
0
;
i
<
nMacroElements
;
i
++
)
{
stringstream
dataStream
(
ios
::
out
|
ios
::
in
|
ios
::
binary
);
int
size
=
0
;
if
(
minor
>
0
)
size
=
macroElSize
[
i
];
else
size
=
(
i
!=
nMacroElements
-
1
)
?
macroElSize
[
i
+
1
]
-
macroElSize
[
i
]
:
fileSize
-
macroElSize
[
i
];
char
*
buffer
=
new
char
[
size
];
file
.
read
(
buffer
,
size
);
dataStream
.
write
(
buffer
,
size
);
delete
[]
buffer
;
#ifdef HAVE_COMPRESSION
if
(
cps
==
"zlib"
)
{
stringstream
tmp
(
ios
::
out
|
ios
::
in
);
boost
::
iostreams
::
filtering_streambuf
<
boost
::
iostreams
::
input
>
in
;
in
.
push
(
boost
::
iostreams
::
zlib_decompressor
());
in
.
push
(
dataStream
);
boost
::
iostreams
::
copy
(
in
,
tmp
);
dataStream
.
str
(
tmp
.
str
());
}
#endif
uint32_t
nStructureCodes
=
0
;
uint32_t
codeSize
=
0
;
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
nStructureCodes
),
4
);
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
codeSize
),
4
);
data
[
nProc
].
insert
(
make_pair
(
macroElIndex
[
i
],
codeSize
));
// We only need codeSize for each macro element data block, so skip all of the unnecessary.
dataStream
.
seekg
(
8
*
nStructureCodes
,
ios_base
::
cur
);
if
(
nValueVectors
>
0
)
{
for
(
unsigned
int
j
=
0
;
j
<
nFeSpaces
;
j
++
)
{
uint32_t
nValuesPerVector
=
0
;
dataStream
.
read
(
reinterpret_cast
<
char
*>
(
&
nValuesPerVector
),
4
);
dataStream
.
seekg
(
sortedFeSpaces
[
j
].
size
()
*
8
*
nValuesPerVector
,
ios_base
::
cur
);
}
}
}
}
// end readMetaFromSgArh
}
// end namespace detail
}
// end namespace Arh2Reader
}
}
// end namespace io, AMDiS
...
...
AMDiS/src/io/detail/Arh2Reader.h
View file @
8d1f6a40
...
...
@@ -67,6 +67,10 @@ namespace AMDiS { namespace io {
int
nProcs
=
-
1
,
bool
byName
=
false
);
/// read meta data from a single ARH-file
void
readMetaFromSgArh
(
std
::
string
filename
,
int
nProc
,
std
::
vector
<
std
::
set
<
std
::
pair
<
int
,
int
>
>
>&
data
);
}
//end namespace detail
}
// end namespace Arh2Reader
}
}
// end namespace io, AMDiS
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment