Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
A
amdis
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Backofen, Rainer
amdis
Commits
2bbf10f7
Commit
2bbf10f7
authored
Jun 26, 2012
by
Thomas Witkowski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fixed some problems with nnz computation with coarse grid, still not finished.
parent
53c5f965
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
184 additions
and
139 deletions
+184
-139
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
+133
-128
AMDiS/src/parallel/PetscSolverGlobalMatrix.h
AMDiS/src/parallel/PetscSolverGlobalMatrix.h
+51
-11
No files found.
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
View file @
2bbf10f7
...
@@ -51,7 +51,9 @@ namespace AMDiS {
...
@@ -51,7 +51,9 @@ namespace AMDiS {
MatCreateAIJ
(
mpiCommGlobal
,
nRankRows
,
nRankRows
,
MatCreateAIJ
(
mpiCommGlobal
,
nRankRows
,
nRankRows
,
nOverallRows
,
nOverallRows
,
nOverallRows
,
nOverallRows
,
0
,
matInteriorDiagNnz
,
0
,
matInteriorOffdiagNnz
,
&
matIntInt
);
0
,
nnzInterior
.
dnnz
,
0
,
nnzInterior
.
onnz
,
&
matIntInt
);
MatSetOption
(
matIntInt
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
MatSetOption
(
matIntInt
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
...
@@ -132,28 +134,40 @@ namespace AMDiS {
...
@@ -132,28 +134,40 @@ namespace AMDiS {
bool
localMatrix
=
(
subdomainLevel
==
0
);
bool
localMatrix
=
(
subdomainLevel
==
0
);
if
(
checkMeshChange
(
mat
,
localMatrix
))
{
if
(
checkMeshChange
(
mat
,
localMatrix
))
{
createPetscNnzStructureWithCoarseSpace
(
mat
,
createPetscNnzStructureWithCoarseSpace
(
mat
,
*
interiorMap
,
*
interiorMap
,
matInteriorDiagNnz
,
*
interiorMap
,
matInteriorOffdiagNnz
,
nnzInterior
,
localMatrix
);
localMatrix
);
if
(
coarseSpaceMap
)
if
(
coarseSpaceMap
)
{
createPetscNnzStructureWithCoarseSpace
(
mat
,
*
coarseSpaceMap
,
*
coarseSpaceMap
,
nnzCoarse
);
createPetscNnzStructureWithCoarseSpace
(
mat
,
createPetscNnzStructureWithCoarseSpace
(
mat
,
*
coarseSpaceMap
,
*
coarseSpaceMap
,
matCoarseDiagNnz
,
*
interiorMap
,
matCoarseOffdiagNnz
);
nnzCoarseInt
);
createPetscNnzStructureWithCoarseSpace
(
mat
,
*
interiorMap
,
*
coarseSpaceMap
,
nnzIntCoarse
);
}
}
}
if
(
localMatrix
)
{
if
(
localMatrix
)
{
MatCreateSeqAIJ
(
mpiCommLocal
,
nRowsRankInterior
,
nRowsRankInterior
,
MatCreateSeqAIJ
(
mpiCommLocal
,
nRowsRankInterior
,
nRowsRankInterior
,
0
,
matInteriorDiagNnz
,
&
matIntInt
);
0
,
nnzInterior
.
dnnz
,
&
matIntInt
);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
else
{
}
else
{
MatCreateAIJ
(
mpiCommLocal
,
MatCreateAIJ
(
mpiCommLocal
,
nRowsRankInterior
,
nRowsRankInterior
,
nRowsRankInterior
,
nRowsRankInterior
,
nRowsOverallInterior
,
nRowsOverallInterior
,
nRowsOverallInterior
,
nRowsOverallInterior
,
0
,
matInteriorDiagNnz
,
0
,
matInteriorOffdiagN
nz
,
0
,
nnzInterior
.
dnnz
,
0
,
nnzInterior
.
on
nz
,
&
matIntInt
);
&
matIntInt
);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
}
if
(
coarseSpaceMap
)
{
if
(
coarseSpaceMap
)
{
...
@@ -163,13 +177,16 @@ namespace AMDiS {
...
@@ -163,13 +177,16 @@ namespace AMDiS {
MatCreateAIJ
(
mpiCommGlobal
,
MatCreateAIJ
(
mpiCommGlobal
,
nRowsRankCoarse
,
nRowsRankCoarse
,
nRowsRankCoarse
,
nRowsRankCoarse
,
nRowsOverallCoarse
,
nRowsOverallCoarse
,
nRowsOverallCoarse
,
nRowsOverallCoarse
,
0
,
matCoarseDiagNnz
,
0
,
matCoarseOffdiagNnz
,
0
,
nnzCoarse
.
dnnz
,
0
,
nnzCoarse
.
onnz
,
// 100, PETSC_NULL, 100, PETSC_NULL,
&
matCoarseCoarse
);
&
matCoarseCoarse
);
// MatSetOption(matCoarseCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
MatCreateAIJ
(
mpiCommGlobal
,
MatCreateAIJ
(
mpiCommGlobal
,
nRowsRankCoarse
,
nRowsRankInterior
,
nRowsRankCoarse
,
nRowsRankInterior
,
nRowsOverallCoarse
,
nGlobalOverallInterior
,
nRowsOverallCoarse
,
nGlobalOverallInterior
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
// 0, nnzCoarseInt.dnnz, 0, nnzCoarseInt.onnz,
&
matCoarseInt
);
&
matCoarseInt
);
MatSetOption
(
matCoarseInt
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
MatSetOption
(
matCoarseInt
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
...
@@ -177,7 +194,7 @@ namespace AMDiS {
...
@@ -177,7 +194,7 @@ namespace AMDiS {
nRowsRankInterior
,
nRowsRankCoarse
,
nRowsRankInterior
,
nRowsRankCoarse
,
nGlobalOverallInterior
,
nRowsOverallCoarse
,
nGlobalOverallInterior
,
nRowsOverallCoarse
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
100
,
PETSC_NULL
,
// 0,
matInteriorDiagNnz, 0, matCoarseOffdiagN
nz,
// 0,
nnzIntCoarse.dnnz, 0, nnzIntCoarse.on
nz,
&
matIntCoarse
);
&
matIntCoarse
);
MatSetOption
(
matIntCoarse
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
MatSetOption
(
matIntCoarse
,
MAT_NEW_NONZERO_ALLOCATION_ERR
,
PETSC_FALSE
);
}
}
...
@@ -259,14 +276,19 @@ namespace AMDiS {
...
@@ -259,14 +276,19 @@ namespace AMDiS {
if
(
colsOther
.
size
())
{
if
(
colsOther
.
size
())
{
if
(
subdomainLevel
==
0
)
{
if
(
subdomainLevel
==
0
)
{
for
(
unsigned
int
k
=
0
;
k
<
colsOther
.
size
();
k
++
)
for
(
unsigned
int
k
=
0
;
k
<
colsOther
.
size
();
k
++
)
{
int
local
=
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
])
-
interiorMap
->
getStartDofs
();
if
(
rowIndex
==
0
&&
local
==
9297
)
MSG
(
"FOUND: %d %d %d
\n
"
,
colsOther
[
k
],
j
,
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
]));
colsOther
[
k
]
=
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
]);
colsOther
[
k
]
=
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
]);
}
}
else
{
}
else
{
for
(
unsigned
int
k
=
0
;
k
<
colsOther
.
size
();
k
++
)
for
(
unsigned
int
k
=
0
;
k
<
colsOther
.
size
();
k
++
)
colsOther
[
k
]
=
colsOther
[
k
]
=
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
])
+
rStartInterior
;
interiorMap
->
getMatIndex
(
j
,
colsOther
[
k
])
+
rStartInterior
;
}
}
MatSetValues
(
matCoarseInt
,
1
,
&
rowIndex
,
colsOther
.
size
(),
MatSetValues
(
matCoarseInt
,
1
,
&
rowIndex
,
colsOther
.
size
(),
&
(
colsOther
[
0
]),
&
(
valuesOther
[
0
]),
ADD_VALUES
);
&
(
colsOther
[
0
]),
&
(
valuesOther
[
0
]),
ADD_VALUES
);
}
}
...
@@ -318,6 +340,8 @@ namespace AMDiS {
...
@@ -318,6 +340,8 @@ namespace AMDiS {
MatAssemblyEnd
(
matCoarseInt
,
MAT_FINAL_ASSEMBLY
);
MatAssemblyEnd
(
matCoarseInt
,
MAT_FINAL_ASSEMBLY
);
}
}
AMDiS
::
finalize
();
exit
(
0
);
// === Remove Dirichlet BC DOFs. ===
// === Remove Dirichlet BC DOFs. ===
...
@@ -577,30 +601,18 @@ namespace AMDiS {
...
@@ -577,30 +601,18 @@ namespace AMDiS {
static_cast
<
int
>
(
meshDistributor
->
getLastMeshChangeIndex
()
!=
lastMeshNnz
);
static_cast
<
int
>
(
meshDistributor
->
getLastMeshChangeIndex
()
!=
lastMeshNnz
);
mpiCommGlobal
.
Allreduce
(
&
sendValue
,
&
recvAllValues
,
1
,
MPI_INT
,
MPI_SUM
);
mpiCommGlobal
.
Allreduce
(
&
sendValue
,
&
recvAllValues
,
1
,
MPI_INT
,
MPI_SUM
);
if
(
!
matInteriorDiagN
nz
||
recvAllValues
!=
0
||
alwaysCreateNnzStructure
)
{
if
(
!
nnzInterior
.
dn
nz
||
recvAllValues
!=
0
||
alwaysCreateNnzStructure
)
{
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
interiorMap
->
setComputeMatIndex
(
true
,
!
localMatrix
);
interiorMap
->
setComputeMatIndex
(
true
,
!
localMatrix
);
interiorMap
->
update
(
feSpaces
);
interiorMap
->
update
(
feSpaces
);
if
(
matInteriorDiagNnz
)
{
nnzInterior
.
clear
();
delete
[]
matInteriorDiagNnz
;
matInteriorDiagNnz
=
NULL
;
}
if
(
matInteriorOffdiagNnz
)
{
delete
[]
matInteriorOffdiagNnz
;
matInteriorOffdiagNnz
=
NULL
;
}
if
(
matCoarseDiagNnz
)
{
if
(
coarseSpaceMap
)
{
delete
[]
matCoarseDiagNnz
;
nnzCoarse
.
clear
();
matCoarseDiagNnz
=
NULL
;
nnzCoarseInt
.
clear
();
}
nnzIntCoarse
.
clear
();
if
(
matCoarseOffdiagNnz
)
{
delete
[]
matCoarseOffdiagNnz
;
matCoarseOffdiagNnz
=
NULL
;
}
}
updateSubdomainData
();
updateSubdomainData
();
...
@@ -1034,18 +1046,11 @@ namespace AMDiS {
...
@@ -1034,18 +1046,11 @@ namespace AMDiS {
{
{
FUNCNAME
(
"PetscSolverGlobalMatrix::createPetscNnzStructure()"
);
FUNCNAME
(
"PetscSolverGlobalMatrix::createPetscNnzStructure()"
);
TEST_EXIT_DBG
(
!
matInteriorDiagNnz
)(
"There is something wrong!
\n
"
);
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
int
nRankRows
=
interiorMap
->
getRankDofs
();
int
nRankRows
=
interiorMap
->
getRankDofs
();
int
rankStartIndex
=
interiorMap
->
getStartDofs
();
int
rankStartIndex
=
interiorMap
->
getStartDofs
();
matInteriorDiagNnz
=
new
int
[
nRankRows
];
nnzInterior
.
create
(
nRankRows
,
nRankRows
);
matInteriorOffdiagNnz
=
new
int
[
nRankRows
];
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
{
matInteriorDiagNnz
[
i
]
=
0
;
matInteriorOffdiagNnz
[
i
]
=
0
;
}
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
namespace
traits
=
mtl
::
traits
;
namespace
traits
=
mtl
::
traits
;
...
@@ -1143,9 +1148,9 @@ namespace AMDiS {
...
@@ -1143,9 +1148,9 @@ namespace AMDiS {
// otherwise the matInteriorDiagNnz value.
// otherwise the matInteriorDiagNnz value.
if
(
petscColIdx
>=
rankStartIndex
&&
if
(
petscColIdx
>=
rankStartIndex
&&
petscColIdx
<
rankStartIndex
+
nRankRows
)
petscColIdx
<
rankStartIndex
+
nRankRows
)
matInteriorDiagN
nz
[
localPetscRowIdx
]
++
;
nnzInterior
.
dn
nz
[
localPetscRowIdx
]
++
;
else
else
matInteriorOffdiagN
nz
[
localPetscRowIdx
]
++
;
nnzInterior
.
on
nz
[
localPetscRowIdx
]
++
;
}
}
}
}
}
else
{
}
else
{
...
@@ -1202,9 +1207,9 @@ namespace AMDiS {
...
@@ -1202,9 +1207,9 @@ namespace AMDiS {
r
,
localRowIdx
,
nRankRows
,
it
->
first
);
r
,
localRowIdx
,
nRankRows
,
it
->
first
);
if
(
c
<
rankStartIndex
||
c
>=
rankStartIndex
+
nRankRows
)
if
(
c
<
rankStartIndex
||
c
>=
rankStartIndex
+
nRankRows
)
matInteriorOffdiagN
nz
[
localRowIdx
]
++
;
nnzInterior
.
on
nz
[
localRowIdx
]
++
;
else
else
matInteriorDiagN
nz
[
localRowIdx
]
++
;
nnzInterior
.
dn
nz
[
localRowIdx
]
++
;
}
}
}
}
}
}
...
@@ -1217,33 +1222,26 @@ namespace AMDiS {
...
@@ -1217,33 +1222,26 @@ namespace AMDiS {
if
(
nRankRows
<
100
)
if
(
nRankRows
<
100
)
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
matInteriorDiagNnz
[
i
]
=
std
::
min
(
matInteriorDiagN
nz
[
i
],
nRankRows
);
nnzInterior
.
dnnz
[
i
]
=
std
::
min
(
nnzInterior
.
dn
nz
[
i
],
nRankRows
);
}
}
void
PetscSolverGlobalMatrix
::
createPetscNnzStructureWithCoarseSpace
(
Matrix
<
DOFMatrix
*>
*
mat
,
void
PetscSolverGlobalMatrix
::
createPetscNnzStructureWithCoarseSpace
(
Matrix
<
DOFMatrix
*>
*
mat
,
ParallelDofMapping
&
d
ofMap
,
ParallelDofMapping
&
rowD
ofMap
,
int
*&
diagNnz
,
ParallelDofMapping
&
colDofMap
,
int
*&
offdiagN
nz
,
NnzStructure
&
n
nz
,
bool
localMatrix
)
bool
localMatrix
)
{
{
FUNCNAME
(
"PetscSolverGlobalMatrix::createPetscNnzStructure()"
);
FUNCNAME
(
"PetscSolverGlobalMatrix::createPetscNnzStructure()"
);
TEST_EXIT_DBG
(
!
diagNnz
)(
"There is something wrong!
\n
"
);
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
vector
<
const
FiniteElemSpace
*>
feSpaces
=
getFeSpaces
(
mat
);
int
nRankRows
=
d
ofMap
.
getRankDofs
();
int
nRankRows
=
rowD
ofMap
.
getRankDofs
();
int
rankStart
Index
=
d
ofMap
.
getStartDofs
();
int
rankStart
RowIndex
=
rowD
ofMap
.
getStartDofs
();
diagNnz
=
new
int
[
nRankRows
];
int
nRankCols
=
colDofMap
.
getRankDofs
();
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
int
rankStartColIndex
=
colDofMap
.
getStartDofs
();
diagNnz
[
i
]
=
0
;
if
(
localMatrix
==
false
)
{
nnz
.
create
(
nRankRows
,
(
!
localMatrix
?
nRankRows
:
-
1
));
offdiagNnz
=
new
int
[
nRankRows
];
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
offdiagNnz
[
i
]
=
0
;
}
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
namespace
traits
=
mtl
::
traits
;
namespace
traits
=
mtl
::
traits
;
...
@@ -1308,21 +1306,21 @@ namespace AMDiS {
...
@@ -1308,21 +1306,21 @@ namespace AMDiS {
for
(
cursor_type
cursor
=
begin
<
row
>
(
bmat
),
for
(
cursor_type
cursor
=
begin
<
row
>
(
bmat
),
cend
=
end
<
row
>
(
bmat
);
cursor
!=
cend
;
++
cursor
)
{
cend
=
end
<
row
>
(
bmat
);
cursor
!=
cend
;
++
cursor
)
{
if
(
d
ofMap
[
feSpaces
[
i
]].
find
(
*
cursor
,
rowDofIndex
)
==
false
)
if
(
rowD
ofMap
[
feSpaces
[
i
]].
find
(
*
cursor
,
rowDofIndex
)
==
false
)
continue
;
continue
;
// The corresponding global matrix row index of the current row DOF.
// The corresponding global matrix row index of the current row DOF.
int
petscRowIdx
=
0
;
int
petscRowIdx
=
0
;
if
(
localMatrix
)
{
if
(
localMatrix
)
{
petscRowIdx
=
d
ofMap
.
getLocalMatIndex
(
i
,
*
cursor
);
petscRowIdx
=
rowD
ofMap
.
getLocalMatIndex
(
i
,
*
cursor
);
}
else
{
}
else
{
if
(
d
ofMap
.
isMatIndexFromGlobal
())
if
(
rowD
ofMap
.
isMatIndexFromGlobal
())
petscRowIdx
=
d
ofMap
.
getMatIndex
(
i
,
rowDofIndex
.
global
);
petscRowIdx
=
rowD
ofMap
.
getMatIndex
(
i
,
rowDofIndex
.
global
);
else
else
petscRowIdx
=
d
ofMap
.
getMatIndex
(
i
,
*
cursor
);
petscRowIdx
=
rowD
ofMap
.
getMatIndex
(
i
,
*
cursor
);
}
}
if
(
localMatrix
||
d
ofMap
[
feSpaces
[
i
]].
isRankDof
(
*
cursor
))
{
if
(
localMatrix
||
rowD
ofMap
[
feSpaces
[
i
]].
isRankDof
(
*
cursor
))
{
// === The current row DOF is a rank DOF, so create the ===
// === The current row DOF is a rank DOF, so create the ===
// === corresponding nnz values directly on rank's nnz data. ===
// === corresponding nnz values directly on rank's nnz data. ===
...
@@ -1330,50 +1328,47 @@ namespace AMDiS {
...
@@ -1330,50 +1328,47 @@ namespace AMDiS {
// This is the local row index of the local PETSc matrix.
// This is the local row index of the local PETSc matrix.
int
localPetscRowIdx
=
petscRowIdx
;
int
localPetscRowIdx
=
petscRowIdx
;
if
(
localMatrix
==
false
)
{
if
(
localMatrix
==
false
)
localPetscRowIdx
-=
rankStartIndex
;
localPetscRowIdx
-=
rankStart
Row
Index
;
TEST_EXIT_DBG
(
localPetscRowIdx
>=
0
&&
localPetscRowIdx
<
nRankRows
)
TEST_EXIT_DBG
(
localPetscRowIdx
>=
0
&&
localPetscRowIdx
<
nRankRows
)
(
"Should not happen!
\n
Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d
\n
"
,
(
"Should not happen!
\n
Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d
\n
"
,
*
cursor
,
*
cursor
,
dofMap
[
feSpaces
[
i
]][
*
cursor
].
global
,
rowDofMap
[
feSpaces
[
i
]][
*
cursor
].
global
,
petscRowIdx
,
petscRowIdx
,
localPetscRowIdx
,
localPetscRowIdx
,
rankStartIndex
,
rankStartRowIndex
,
i
,
i
,
nComponents
,
nComponents
,
nRankRows
);
nRankRows
);
}
if
(
localMatrix
)
{
if
(
localMatrix
)
{
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
if
(
d
ofMap
[
feSpaces
[
j
]].
isSet
(
col
(
*
icursor
)))
if
(
colD
ofMap
[
feSpaces
[
j
]].
isSet
(
col
(
*
icursor
)))
diagN
nz
[
localPetscRowIdx
]
++
;
nnz
.
dn
nz
[
localPetscRowIdx
]
++
;
}
else
{
}
else
{
MultiIndex
colDofIndex
;
MultiIndex
colDofIndex
;
// Traverse all non zero entries in this row.
// Traverse all non zero entries in this row.
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
if
(
d
ofMap
[
feSpaces
[
j
]].
find
(
col
(
*
icursor
),
colDofIndex
)
==
false
)
if
(
colD
ofMap
[
feSpaces
[
j
]].
find
(
col
(
*
icursor
),
colDofIndex
)
==
false
)
continue
;
continue
;
int
petscColIdx
=
(
dofMap
.
isMatIndexFromGlobal
()
?
int
petscColIdx
=
(
colDofMap
.
isMatIndexFromGlobal
()
?
dofMap
.
getMatIndex
(
j
,
colDofIndex
.
global
)
:
colDofMap
.
getMatIndex
(
j
,
colDofIndex
.
global
)
:
dofMap
.
getMatIndex
(
j
,
col
(
*
icursor
)));
colDofMap
.
getMatIndex
(
j
,
col
(
*
icursor
)));
if
(
value
(
*
icursor
)
!=
0.0
||
petscRowIdx
==
petscColIdx
)
{
// The row DOF is a rank DOF, if also the column is a rank DOF,
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the diagNnz values for this row,
// increment the diagNnz values for this row,
// otherwise the offdiagNnz value.
// otherwise the offdiagNnz value.
if
(
petscColIdx
>=
rankStartColIndex
&&
if
(
petscColIdx
>=
rankStartIndex
&&
petscColIdx
<
rankStartColIndex
+
nRankCols
)
petscColIdx
<
rankStartIndex
+
nRankRows
)
nnz
.
dnnz
[
localPetscRowIdx
]
++
;
diagNnz
[
localPetscRowIdx
]
++
;
else
else
nnz
.
onnz
[
localPetscRowIdx
]
++
;
offdiagNnz
[
localPetscRowIdx
]
++
;
}
}
}
}
}
}
else
{
}
else
{
...
@@ -1389,17 +1384,15 @@ namespace AMDiS {
...
@@ -1389,17 +1384,15 @@ namespace AMDiS {
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
if
(
d
ofMap
[
feSpaces
[
j
]].
find
(
col
(
*
icursor
),
colDofIndex
)
==
false
)
if
(
colD
ofMap
[
feSpaces
[
j
]].
find
(
col
(
*
icursor
),
colDofIndex
)
==
false
)
continue
;
continue
;
if
(
value
(
*
icursor
)
!=
0.0
)
{
int
petscColIdx
=
(
colDofMap
.
isMatIndexFromGlobal
()
?
int
petscColIdx
=
(
dofMap
.
isMatIndexFromGlobal
()
?
colDofMap
.
getMatIndex
(
j
,
colDofIndex
.
global
)
:
dofMap
.
getMatIndex
(
j
,
colDofIndex
.
global
)
:
colDofMap
.
getMatIndex
(
j
,
col
(
*
icursor
)));
dofMap
.
getMatIndex
(
j
,
col
(
*
icursor
)));
sendMatrixEntry
[
sendToRank
].
sendMatrixEntry
[
sendToRank
].
push_back
(
make_pair
(
petscRowIdx
,
petscColIdx
));
push_back
(
make_pair
(
petscRowIdx
,
petscColIdx
));
}
}
}
}
// if (isRankDof[*cursor]) ... else ...
}
// if (isRankDof[*cursor]) ... else ...
...
@@ -1407,6 +1400,7 @@ namespace AMDiS {
...
@@ -1407,6 +1400,7 @@ namespace AMDiS {
}
}
}
}
if
(
localMatrix
==
false
)
{
if
(
localMatrix
==
false
)
{
// === Send and recv the nnz row structure to/from other ranks. ===
// === Send and recv the nnz row structure to/from other ranks. ===
...
@@ -1428,43 +1422,54 @@ namespace AMDiS {
...
@@ -1428,43 +1422,54 @@ namespace AMDiS {
int
r
=
it
->
second
[
i
].
first
;
int
r
=
it
->
second
[
i
].
first
;
int
c
=
it
->
second
[
i
].
second
;
int
c
=
it
->
second
[
i
].
second
;
int
localRowIdx
=
r
-
rankStartIndex
;
int
localRowIdx
=
r
-
rankStart
Row
Index
;
TEST_EXIT_DBG
(
localRowIdx
>=
0
&&
localRowIdx
<
nRankRows
)
TEST_EXIT_DBG
(
localRowIdx
>=
0
&&
localRowIdx
<
nRankRows
)
(
"Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!
\n
"
,
(
"Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!
\n
"
,
r
,
localRowIdx
,
nRankRows
,
it
->
first
);
r
,
localRowIdx
,
nRankRows
,
it
->
first
);
if
(
c
<
rankStart
Index
||
c
>=
rankStartIndex
+
nRankRow
s
)
if
(
c
<
rankStart
ColIndex
||
c
>=
rankStartColIndex
+
nRankCol
s
)
offdiagN
nz
[
localRowIdx
]
++
;
nnz
.
on
nz
[
localRowIdx
]
++
;
else
else
diagN
nz
[
localRowIdx
]
++
;
nnz
.
dn
nz
[
localRowIdx
]
++
;
}
}
}
}
}
}
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
if
(
nRankRows
<
100
)
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
diagNnz
[
i
]
=
std
::
min
(
diagNnz
[
i
],
nRankRows
);
}
}
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
// if (nRankRows < 100)
// for (int i = 0; i < nRankRows; i++)
// nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankCols);
#if (DEBUG != 0)
#if (DEBUG != 0)
int
nMax
=
0
;
int
nMax
=
0
;
int
nSum
=
0
;
int
nSum
=
0
;
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
{
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
{
nMax
=
std
::
max
(
nMax
,
diagN
nz
[
i
]);
nMax
=
std
::
max
(
nMax
,
nnz
.
dn
nz
[
i
]);
nSum
+=
diagN
nz
[
i
];
nSum
+=
nnz
.
dn
nz
[
i
];
}
}
MSG
(
"NNZ in matrix: max = %d, avrg = %.0f
\n
"
,
MSG
(
"NNZ in diag block: max = %d, avrg = %.0f
\n
"
,
nMax
,
static_cast
<
double
>
(
nSum
)
/
nRankRows
);
nMax
,
(
nSum
>
0
?
(
static_cast
<
double
>
(
nSum
)
/
nRankRows
)
:
0
));
#endif
if
(
!
localMatrix
)
{
nMax
=
0
;
nSum
=
0
;
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
{
nMax
=
std
::
max
(
nMax
,
nnz
.
onnz
[
i
]);
nSum
+=
nnz
.
onnz
[
i
];
}
MSG
(
"NNZ in offdiag block: max = %d, avrg = %.0f
\n
"
,
nMax
,
(
nSum
>
0
?
(
static_cast
<
double
>
(
nSum
)
/
nRankRows
)
:
0
));
}
}
}
#endif
}
}
AMDiS/src/parallel/PetscSolverGlobalMatrix.h
View file @
2bbf10f7
...
@@ -31,16 +31,56 @@ namespace AMDiS {
...
@@ -31,16 +31,56 @@ namespace AMDiS {
using
namespace
std
;
using
namespace
std
;
struct
NnzStructure
{
NnzStructure
()
:
dnnz
(
NULL
),