Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
iwr
amdis
Commits
dfa8ce49
Commit
dfa8ce49
authored
Mar 09, 2011
by
Thomas Witkowski
Browse files
Fixed PETSc nnz problem.
parent
4f18875c
Changes
2
Hide whitespace changes
Inline
Side-by-side
AMDiS/libtool
View file @
dfa8ce49
...
...
@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host
deimos104
:
# Libtool was configured on host
p1d098
:
# Shell to use when invoking shell scripts.
SHELL
=
"/bin/sh"
...
...
@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS
=
"cru"
# A C compiler.
LTCC
=
"
g
cc"
LTCC
=
"
/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpi
cc"
# LTCC compiler flags.
LTCFLAGS
=
"-g -O3"
# A language-specific compiler.
CC
=
"
g
cc"
CC
=
"
/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpi
cc"
# Is the compiler the GNU C compiler?
with_gcc
=
yes
...
...
@@ -174,7 +174,7 @@ dlopen_self=unknown
dlopen_self_static
=
unknown
# Compiler flag to prevent dynamic linking.
link_static_flag
=
"
-static
"
link_static_flag
=
""
# Compiler flag to turn off builtin functions.
no_builtin_flag
=
" -fno-builtin"
...
...
@@ -6763,7 +6763,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host
deimos104
:
# Libtool was configured on host
p1d098
:
# Shell to use when invoking shell scripts.
SHELL
=
"/bin/sh"
...
...
@@ -6801,13 +6801,13 @@ AR="ar"
AR_FLAGS
=
"cru"
# A C compiler.
LTCC
=
"
g
cc"
LTCC
=
"
/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpi
cc"
# LTCC compiler flags.
LTCFLAGS
=
"-g -O3"
# A language-specific compiler.
CC
=
"
g++
"
CC
=
"
/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx
"
# Is the compiler the GNU C compiler?
with_gcc
=
yes
...
...
@@ -6893,7 +6893,7 @@ dlopen_self=unknown
dlopen_self_static
=
unknown
# Compiler flag to prevent dynamic linking.
link_static_flag
=
"
-static
"
link_static_flag
=
""
# Compiler flag to turn off builtin functions.
no_builtin_flag
=
" -fno-builtin"
...
...
@@ -6960,11 +6960,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdeps
=
"-lstdc++ -lm -lgcc_s -lc -lgcc_s"
postdeps
=
"
-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl
-lstdc++ -lm -lgcc_s
-lpthread
-lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path
=
`
echo
"-L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
|
$SED
-e
"s@
${
gcc_dir
}
@
\$
{gcc_dir}@g;s@
${
gcc_ver
}
@
\$
{gcc_ver}@g"
`
compiler_lib_search_path
=
`
echo
"-L/usr/lib64
-L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64
/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
|
$SED
-e
"s@
${
gcc_dir
}
@
\$
{gcc_dir}@g;s@
${
gcc_ver
}
@
\$
{gcc_ver}@g"
`
# Method to check whether dependent libraries are shared objects.
deplibs_check_method
=
"pass_all"
...
...
@@ -7071,7 +7071,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host
deimos104
:
# Libtool was configured on host
p1d098
:
# Shell to use when invoking shell scripts.
SHELL
=
"/bin/sh"
...
...
@@ -7109,7 +7109,7 @@ AR="ar"
AR_FLAGS
=
"cru"
# A C compiler.
LTCC
=
"
g
cc"
LTCC
=
"
/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpi
cc"
# LTCC compiler flags.
LTCFLAGS
=
"-g -O3"
...
...
AMDiS/src/parallel/PetscSolver.cc
View file @
dfa8ce49
...
...
@@ -24,10 +24,12 @@
namespace
AMDiS
{
using
namespace
std
;
PetscErrorCode
myKSPMonitor
(
KSP
ksp
,
PetscInt
iter
,
PetscReal
rnorm
,
void
*
)
{
if
(
iter
%
100
==
0
&&
MPI
::
COMM_WORLD
.
Get_rank
()
==
0
)
std
::
cout
<<
"[0] Petsc-Iteration "
<<
iter
<<
": "
<<
rnorm
<<
std
::
endl
;
cout
<<
"[0] Petsc-Iteration "
<<
iter
<<
": "
<<
rnorm
<<
endl
;
return
0
;
}
...
...
@@ -66,12 +68,12 @@ namespace AMDiS {
typedef
traits
::
range_generator
<
row
,
Matrix
>::
type
cursor_type
;
typedef
traits
::
range_generator
<
nz
,
cursor_type
>::
type
icursor_type
;
std
::
vector
<
int
>
cols
;
std
::
vector
<
double
>
values
;
vector
<
int
>
cols
;
vector
<
double
>
values
;
cols
.
reserve
(
300
);
values
.
reserve
(
300
);
std
::
vector
<
int
>
globalCols
;
vector
<
int
>
globalCols
;
// === Traverse all rows of the dof matrix and insert row wise the values ===
...
...
@@ -86,7 +88,9 @@ namespace AMDiS {
bool
periodicRow
=
meshDistributor
->
isPeriodicDof
(
globalRowDof
);
if
(
!
periodicRow
)
{
// Calculate petsc row index.
// === Row DOF index is not periodic. ===
// Calculate PETSc row index.
int
rowIndex
=
globalRowDof
*
dispMult
+
dispAddRow
;
cols
.
clear
();
...
...
@@ -99,35 +103,54 @@ namespace AMDiS {
int
globalColDof
=
meshDistributor
->
mapLocalToGlobal
(
col
(
*
icursor
));
// Test if the current col dof is a periodic dof.
bool
periodicCol
=
meshDistributor
->
isPeriodicDof
(
globalColDof
);
// Calculate PETSc col index.
int
colIndex
=
globalColDof
*
dispMult
+
dispAddCol
;
if
(
value
(
*
icursor
)
==
0.0
&&
globalRowDof
!=
globalColDof
)
// Ignore all zero entries, expect it is a diagonal entry.
// if (value(*icursor) == 0.0 && globalRowDof != globalColDof)
if
(
value
(
*
icursor
)
==
0.0
&&
rowIndex
!=
colIndex
)
continue
;
if
(
!
periodicCol
)
{
// Calculate the exact position of the column index in the
pets
c matrix.
// Calculate the exact position of the column index in the
PETS
c matrix.
cols
.
push_back
(
globalColDof
*
dispMult
+
dispAddCol
);
values
.
push_back
(
value
(
*
icursor
));
}
else
{
std
::
set
<
int
>&
perColAsc
=
meshDistributor
->
getPerDofAssociations
(
globalColDof
);
// === Row index is not periodic, but column index is. ===
// Create set of all periodic associations of the column index.
std
::
set
<
int
>
perAsc
;
std
::
set
<
int
>&
perColAsc
=
meshDistributor
->
getPerDofAssociations
(
globalColDof
);
for
(
std
::
set
<
int
>::
iterator
it
=
perColAsc
.
begin
();
it
!=
perColAsc
.
end
();
++
it
)
if
(
*
it
>=
-
3
)
perAsc
.
insert
(
*
it
);
// Scale value to the number of periodic associations of the column index.
double
scaledValue
=
value
(
*
icursor
)
*
std
::
pow
(
0.5
,
static_cast
<
double
>
(
perAsc
.
size
()));
std
::
vector
<
int
>
newCols
;
value
(
*
icursor
)
*
pow
(
0.5
,
static_cast
<
double
>
(
perAsc
.
size
()));
// === Create set of all matrix column indices due to the periodic ===
// === associations of the column DOF index. ===
vector
<
int
>
newCols
;
// First, add the original matrix index.
newCols
.
push_back
(
globalColDof
);
// And add all periodic matrix indices.
for
(
std
::
set
<
int
>::
iterator
it
=
perAsc
.
begin
();
it
!=
perAsc
.
end
();
++
it
)
{
int
nCols
=
static_cast
<
int
>
(
newCols
.
size
());
for
(
int
i
=
0
;
i
<
nCols
;
i
++
)
{
for
(
unsigned
int
i
=
0
;
i
<
newCols
.
size
();
i
++
)
{
TEST_EXIT
(
meshDistributor
->
isPeriodicDof
(
newCols
[
i
],
*
it
))
(
"Should not happen: %d %d
\n
"
,
*
it
,
newCols
[
i
]);
(
"Wrong periodic DOF associations at boundary %d with DOF %d!
\n
"
,
*
it
,
newCols
[
i
]);
MSG
(
"MAP DOF %d -> %d
\n
"
,
newCols
[
i
],
meshDistributor
->
getPeriodicMapping
(
newCols
[
i
],
*
it
));
newCols
.
push_back
(
meshDistributor
->
getPeriodicMapping
(
newCols
[
i
],
*
it
));
}
...
...
@@ -143,18 +166,30 @@ namespace AMDiS {
MatSetValues
(
petscMatrix
,
1
,
&
rowIndex
,
cols
.
size
(),
&
(
cols
[
0
]),
&
(
values
[
0
]),
ADD_VALUES
);
}
else
{
std
::
map
<
int
,
std
::
vector
<
int
>
>
colsMap
;
std
::
map
<
int
,
std
::
vector
<
double
>
>
valsMap
;
// === Row DOF index is periodic. ===
// Because this row is periodic, we will have to add the entries of this
// matrix row to multiple rows. The following maps store to each row an
// array of column indices and values of the entries that must be added to
// the PETSc matrix.
map
<
int
,
vector
<
int
>
>
colsMap
;
map
<
int
,
vector
<
double
>
>
valsMap
;
// Traverse all column entries.
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
// Global index of the current column index.
int
globalColDof
=
meshDistributor
->
mapLocalToGlobal
(
col
(
*
icursor
));
// Ignore all zero entries, expect it is a diagonal entry.
if
(
value
(
*
icursor
)
==
0.0
&&
globalRowDof
!=
globalColDof
)
continue
;
// === Add all periodic associations of both, the row and the column ===
// === indices to the set perAsc. ===
std
::
set
<
int
>
perAsc
;
if
(
meshDistributor
->
isPeriodicDof
(
globalColDof
))
{
...
...
@@ -173,11 +208,21 @@ namespace AMDiS {
if
(
*
it
>=
-
3
)
perAsc
.
insert
(
*
it
);
// Scale the value with respect to the number of periodic associations.
double
scaledValue
=
value
(
*
icursor
)
*
std
::
pow
(
0.5
,
static_cast
<
double
>
(
perAsc
.
size
()));
std
::
vector
<
std
::
pair
<
int
,
int
>
>
entry
;
entry
.
push_back
(
std
::
make_pair
(
globalRowDof
,
globalColDof
));
value
(
*
icursor
)
*
pow
(
0.5
,
static_cast
<
double
>
(
perAsc
.
size
()));
// === Create all matrix entries with respect to the periodic ===
// === associations of the row and column indices. ===
vector
<
pair
<
int
,
int
>
>
entry
;
// First, add the original entry.
entry
.
push_back
(
make_pair
(
globalRowDof
,
globalColDof
));
// Then, traverse the periodic associations of the row and column indices
// and create the corresponding entries.
for
(
std
::
set
<
int
>::
iterator
it
=
perAsc
.
begin
();
it
!=
perAsc
.
end
();
++
it
)
{
int
nEntry
=
static_cast
<
int
>
(
entry
.
size
());
for
(
int
i
=
0
;
i
<
nEntry
;
i
++
)
{
...
...
@@ -194,23 +239,28 @@ namespace AMDiS {
perColDof
=
entry
[
i
].
second
;
entry
.
push_back
(
std
::
make_pair
(
perRowDof
,
perColDof
));
entry
.
push_back
(
make_pair
(
perRowDof
,
perColDof
));
}
}
for
(
std
::
vector
<
std
::
pair
<
int
,
int
>
>::
iterator
eIt
=
entry
.
begin
();
// === Translate the matrix entries to PETSc's matrix.
for
(
vector
<
pair
<
int
,
int
>
>::
iterator
eIt
=
entry
.
begin
();
eIt
!=
entry
.
end
();
++
eIt
)
{
// Calculate
petsc row inde
x.
// Calculate
row and column indices of the PETSc matri
x.
int
rowIndex
=
eIt
->
first
*
dispMult
+
dispAddRow
;
int
colIndex
=
eIt
->
second
*
dispMult
+
dispAddCol
;
colsMap
[
rowIndex
].
push_back
(
colIndex
);
valsMap
[
rowIndex
].
push_back
(
scaledValue
);
}
}
for
(
std
::
map
<
int
,
std
::
vector
<
int
>
>::
iterator
rowIt
=
colsMap
.
begin
();
// === Finally, add all periodic rows to the PETSc matrix. ===
for
(
map
<
int
,
vector
<
int
>
>::
iterator
rowIt
=
colsMap
.
begin
();
rowIt
!=
colsMap
.
end
();
++
rowIt
)
{
TEST_EXIT_DBG
(
rowIt
->
second
.
size
()
==
valsMap
[
rowIt
->
first
].
size
())
(
"Should not happen!
\n
"
);
...
...
@@ -275,18 +325,18 @@ namespace AMDiS {
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
namespace
traits
=
mtl
::
traits
;
typedef
DOFMatrix
::
base_matrix_type
Matrix
;
typedef
std
::
vector
<
std
::
pair
<
int
,
int
>
>
MatrixNnzEntry
;
typedef
std
::
map
<
int
,
DofContainer
>
RankToDofContainer
;
typedef
vector
<
pair
<
int
,
int
>
>
MatrixNnzEntry
;
typedef
map
<
int
,
DofContainer
>
RankToDofContainer
;
// Stores to each rank a list of nnz entries (i.e. pairs of row and column index)
// that this rank will send to. These nnz entries will be assembled on this rank,
// but because the row DOFs are not DOFs of this rank they will be send to the
// owner of the row DOFs.
std
::
map
<
int
,
MatrixNnzEntry
>
sendMatrixEntry
;
map
<
int
,
MatrixNnzEntry
>
sendMatrixEntry
;
// Maps to each DOF that must be send to another rank the rank number of the
// receiving rank.
std
::
map
<
DegreeOfFreedom
,
int
>
sendDofToRank
;
map
<
DegreeOfFreedom
,
int
>
sendDofToRank
;
// First, create for all ranks we send data to MatrixNnzEntry object with 0 entries.
...
...
@@ -324,9 +374,9 @@ namespace AMDiS {
int
globalRowDof
=
meshDistributor
->
mapLocalToGlobal
(
*
cursor
);
std
::
vector
<
int
>
rows
;
vector
<
int
>
rows
;
rows
.
push_back
(
globalRowDof
);
std
::
vector
<
int
>
rowRank
;
vector
<
int
>
rowRank
;
if
(
meshDistributor
->
getIsRankDof
(
*
cursor
))
{
rowRank
.
push_back
(
meshDistributor
->
getMpiRank
());
}
else
{
...
...
@@ -387,7 +437,7 @@ namespace AMDiS {
meshDistributor
->
mapLocalToGlobal
(
col
(
*
icursor
))
*
nComponents
+
j
;
sendMatrixEntry
[
sendToRank
].
push_back
(
std
::
make_pair
(
petscRowIdx
,
petscColIdx
));
push_back
(
make_pair
(
petscRowIdx
,
petscColIdx
));
}
}
...
...
@@ -409,7 +459,7 @@ namespace AMDiS {
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
// === the PETSc nnz data structure. ===
for
(
std
::
map
<
int
,
MatrixNnzEntry
>::
iterator
it
=
stdMpi
.
getRecvData
().
begin
();
for
(
map
<
int
,
MatrixNnzEntry
>::
iterator
it
=
stdMpi
.
getRecvData
().
begin
();
it
!=
stdMpi
.
getRecvData
().
end
();
++
it
)
{
if
(
it
->
second
.
size
()
>
0
)
{
for
(
unsigned
int
i
=
0
;
i
<
it
->
second
.
size
();
i
++
)
{
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment