Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
Backofen, Rainer
amdis
Commits
334a275f
Commit
334a275f
authored
Jan 19, 2010
by
Thomas Witkowski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Moved all general debug functions to Debug.h and Debug.cc
parent
73afa800
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
360 additions
and
280 deletions
+360
-280
AMDiS/bin/Makefile.am
AMDiS/bin/Makefile.am
+1
-0
AMDiS/bin/Makefile.in
AMDiS/bin/Makefile.in
+13
-3
AMDiS/src/Debug.cc
AMDiS/src/Debug.cc
+139
-0
AMDiS/src/Debug.h
AMDiS/src/Debug.h
+53
-0
AMDiS/src/ParallelDomainBase.cc
AMDiS/src/ParallelDomainBase.cc
+116
-246
AMDiS/src/ParallelDomainBase.h
AMDiS/src/ParallelDomainBase.h
+21
-26
AMDiS/src/StdMpi.h
AMDiS/src/StdMpi.h
+17
-5
No files found.
AMDiS/bin/Makefile.am
View file @
334a275f
...
...
@@ -216,6 +216,7 @@ $(SOURCE_DIR)/SubElInfo.h $(SOURCE_DIR)/SubElInfo.cc \
$(SOURCE_DIR)
/SolutionDataStorage.h
$(SOURCE_DIR)
/SolutionDataStorage.hh
\
$(SOURCE_DIR)
/InteriorBoundary.h
$(SOURCE_DIR)
/InteriorBoundary.cc
\
$(SOURCE_DIR)
/ElementDofIteartor.h
$(SOURCE_DIR)
/ElementDofIterator.cc
\
$(SOURCE_DIR)
/Debug.h
$(SOURCE_DIR)
/Debug.cc
\
$(SOURCE_DIR)
/parareal/ProblemBase.h
\
$(SOURCE_DIR)
/parareal/AdaptParaReal.h
$(SOURCE_DIR)
/parareal/AdaptParaReal.cc
...
...
AMDiS/bin/Makefile.in
View file @
334a275f
...
...
@@ -225,8 +225,8 @@ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ParallelDomainBase.h \
$(SOURCE_DIR)
/InteriorBoundary.h
\
$(SOURCE_DIR)
/InteriorBoundary.cc
\
$(SOURCE_DIR)
/ElementDofIteartor.h
\
$(SOURCE_DIR)
/ElementDofIterator.cc
\
$(SOURCE_DIR)
/parareal/ProblemBase.h
\
$(SOURCE_DIR)
/ElementDofIterator.cc
$(SOURCE_DIR)
/Debug.h
\
$(SOURCE_DIR)
/Debug.cc
$(SOURCE_DIR)
/parareal/ProblemBase.h
\
$(SOURCE_DIR)
/parareal/AdaptParaReal.h
\
$(SOURCE_DIR)
/parareal/AdaptParaReal.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@
am__objects_1
=
libamdis_la-ParallelDomainBase.lo
\
...
...
@@ -290,7 +290,8 @@ am_libamdis_la_OBJECTS = $(am__objects_2) libamdis_la-DOFIndexed.lo \
libamdis_la-PngWriter.lo libamdis_la-PovrayWriter.lo
\
libamdis_la-DataCollector.lo libamdis_la-ScalableQuadrature.lo
\
libamdis_la-SubElInfo.lo libamdis_la-InteriorBoundary.lo
\
libamdis_la-ElementDofIterator.lo libamdis_la-AdaptParaReal.lo
libamdis_la-ElementDofIterator.lo libamdis_la-Debug.lo
\
libamdis_la-AdaptParaReal.lo
libamdis_la_OBJECTS
=
$(am_libamdis_la_OBJECTS)
libcompositeFEM_la_LIBADD
=
am_libcompositeFEM_la_OBJECTS
=
libcompositeFEM_la-CFE_Integration.lo
\
...
...
@@ -629,6 +630,7 @@ $(SOURCE_DIR)/SubElInfo.h $(SOURCE_DIR)/SubElInfo.cc \
$(SOURCE_DIR)
/SolutionDataStorage.h
$(SOURCE_DIR)
/SolutionDataStorage.hh
\
$(SOURCE_DIR)
/InteriorBoundary.h
$(SOURCE_DIR)
/InteriorBoundary.cc
\
$(SOURCE_DIR)
/ElementDofIteartor.h
$(SOURCE_DIR)
/ElementDofIterator.cc
\
$(SOURCE_DIR)
/Debug.h
$(SOURCE_DIR)
/Debug.cc
\
$(SOURCE_DIR)
/parareal/ProblemBase.h
\
$(SOURCE_DIR)
/parareal/AdaptParaReal.h
$(SOURCE_DIR)
/parareal/AdaptParaReal.cc
...
...
@@ -747,6 +749,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-DOFMatrix.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-DOFVector.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-DataCollector.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-Debug.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-DirichletBC.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-DualTraverse.Plo@am__quote@
@AMDEP_TRUE@@am__include@
@am__quote@./$(DEPDIR)/libamdis_la-ElInfo.Plo@am__quote@
...
...
@@ -1532,6 +1535,13 @@ libamdis_la-ElementDofIterator.lo: $(SOURCE_DIR)/ElementDofIterator.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@
DEPDIR
=
$(DEPDIR)
$(CXXDEPMODE)
$(depcomp)
@AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@
$(LIBTOOL)
--tag
=
CXX
--mode
=
compile
$(CXX)
$(DEFS)
$(DEFAULT_INCLUDES)
$(INCLUDES)
$(AM_CPPFLAGS)
$(CPPFLAGS)
$(libamdis_la_CXXFLAGS)
$(CXXFLAGS)
-c
-o
libamdis_la-ElementDofIterator.lo
`
test
-f
'
$(SOURCE_DIR)
/ElementDofIterator.cc'
||
echo
'
$(srcdir)
/'
`
$(SOURCE_DIR)
/ElementDofIterator.cc
libamdis_la-Debug.lo
:
$(SOURCE_DIR)/Debug.cc
@am__fastdepCXX_TRUE@
if
$(LIBTOOL)
--tag
=
CXX
--mode
=
compile
$(CXX)
$(DEFS)
$(DEFAULT_INCLUDES)
$(INCLUDES)
$(AM_CPPFLAGS)
$(CPPFLAGS)
$(libamdis_la_CXXFLAGS)
$(CXXFLAGS)
-MT
libamdis_la-Debug.lo
-MD
-MP
-MF
"
$(DEPDIR)
/libamdis_la-Debug.Tpo"
-c
-o
libamdis_la-Debug.lo
`
test
-f
'
$(SOURCE_DIR)
/Debug.cc'
||
echo
'
$(srcdir)
/'
`
$(SOURCE_DIR)
/Debug.cc
;
\
@am__fastdepCXX_TRUE@
then
mv
-f
"
$(DEPDIR)
/libamdis_la-Debug.Tpo"
"
$(DEPDIR)
/libamdis_la-Debug.Plo"
;
else
rm
-f
"
$(DEPDIR)
/libamdis_la-Debug.Tpo"
;
exit
1
;
fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@
source
=
'
$(SOURCE_DIR)
/Debug.cc'
object
=
'libamdis_la-Debug.lo'
libtool
=
yes
@AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@
DEPDIR
=
$(DEPDIR)
$(CXXDEPMODE)
$(depcomp)
@AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@
$(LIBTOOL)
--tag
=
CXX
--mode
=
compile
$(CXX)
$(DEFS)
$(DEFAULT_INCLUDES)
$(INCLUDES)
$(AM_CPPFLAGS)
$(CPPFLAGS)
$(libamdis_la_CXXFLAGS)
$(CXXFLAGS)
-c
-o
libamdis_la-Debug.lo
`
test
-f
'
$(SOURCE_DIR)
/Debug.cc'
||
echo
'
$(srcdir)
/'
`
$(SOURCE_DIR)
/Debug.cc
libamdis_la-AdaptParaReal.lo
:
$(SOURCE_DIR)/parareal/AdaptParaReal.cc
@am__fastdepCXX_TRUE@
if
$(LIBTOOL)
--tag
=
CXX
--mode
=
compile
$(CXX)
$(DEFS)
$(DEFAULT_INCLUDES)
$(INCLUDES)
$(AM_CPPFLAGS)
$(CPPFLAGS)
$(libamdis_la_CXXFLAGS)
$(CXXFLAGS)
-MT
libamdis_la-AdaptParaReal.lo
-MD
-MP
-MF
"
$(DEPDIR)
/libamdis_la-AdaptParaReal.Tpo"
-c
-o
libamdis_la-AdaptParaReal.lo
`
test
-f
'
$(SOURCE_DIR)
/parareal/AdaptParaReal.cc'
||
echo
'
$(srcdir)
/'
`
$(SOURCE_DIR)
/parareal/AdaptParaReal.cc
;
\
@am__fastdepCXX_TRUE@
then
mv
-f
"
$(DEPDIR)
/libamdis_la-AdaptParaReal.Tpo"
"
$(DEPDIR)
/libamdis_la-AdaptParaReal.Plo"
;
else
rm
-f
"
$(DEPDIR)
/libamdis_la-AdaptParaReal.Tpo"
;
exit
1
;
fi
...
...
AMDiS/src/Debug.cc
0 → 100644
View file @
334a275f
#include <boost/lexical_cast.hpp>
#include "Debug.h"
#include "DOFVector.h"
#include "MacroElement.h"
#include "VtkWriter.h"
namespace
AMDiS
{
namespace
debug
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void
writeLocalElementDofs
(
int
rank
,
int
elIdx
,
FiniteElemSpace
*
feSpace
)
{
using
boost
::
lexical_cast
;
if
(
MPI
::
COMM_WORLD
.
Get_rank
()
==
rank
)
{
DOFVector
<
double
>
tmp
(
feSpace
,
"tmp"
);
colorDofVectorByLocalElementDofs
(
tmp
,
feSpace
->
getMesh
(),
elIdx
);
VtkWriter
::
writeFile
(
tmp
,
"tmp"
+
lexical_cast
<
std
::
string
>
(
elIdx
)
+
".vtu"
);
}
}
void
writeDofMesh
(
int
rank
,
DegreeOfFreedom
dof
,
FiniteElemSpace
*
feSpace
)
{
using
boost
::
lexical_cast
;
if
(
MPI
::
COMM_WORLD
.
Get_rank
()
==
rank
)
{
DOFVector
<
double
>
tmp
(
feSpace
,
"tmp"
);
tmp
.
set
(
0.0
);
tmp
[
dof
]
=
1.0
;
VtkWriter
::
writeFile
(
tmp
,
"dofmesh"
+
lexical_cast
<
std
::
string
>
(
rank
)
+
".vtu"
);
}
}
void
writeMesh
(
FiniteElemSpace
*
feSpace
,
int
rank
,
std
::
string
filename
)
{
using
boost
::
lexical_cast
;
int
myRank
=
MPI
::
COMM_WORLD
.
Get_rank
();
if
(
rank
==
-
1
||
myRank
==
rank
)
{
DOFVector
<
double
>
tmp
(
feSpace
,
"tmp"
);
VtkWriter
::
writeFile
(
tmp
,
filename
+
lexical_cast
<
std
::
string
>
(
myRank
)
+
".vtu"
);
}
}
#endif
void
colorDofVectorByLocalElementDofs
(
DOFVector
<
double
>&
vec
,
Element
*
el
)
{
// === Get local indices of the given element. ===
const
BasisFunction
*
basisFcts
=
vec
.
getFESpace
()
->
getBasisFcts
();
int
nBasisFcts
=
basisFcts
->
getNumber
();
std
::
vector
<
DegreeOfFreedom
>
localDofs
(
nBasisFcts
);
basisFcts
->
getLocalIndices
(
el
,
vec
.
getFESpace
()
->
getAdmin
(),
localDofs
);
// === Set the values of the dof vector. ===
vec
.
set
(
0.0
);
for
(
int
i
=
0
;
i
<
nBasisFcts
;
i
++
)
vec
[
localDofs
[
i
]]
=
static_cast
<
double
>
(
i
);
}
bool
colorDofVectorByLocalElementDofs
(
DOFVector
<
double
>&
vec
,
Mesh
*
mesh
,
int
elIndex
)
{
FUNCNAME
(
"colorDofVectorByLocalElementDofs()"
);
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh
,
-
1
,
Mesh
::
CALL_LEAF_EL
);
while
(
elInfo
)
{
if
(
elInfo
->
getElement
()
->
getIndex
()
==
elIndex
)
{
colorDofVectorByLocalElementDofs
(
vec
,
elInfo
->
getElement
());
return
true
;
}
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
return
false
;
}
Element
*
getDofIndexElement
(
FiniteElemSpace
*
feSpace
,
DegreeOfFreedom
dof
)
{
const
BasisFunction
*
basFcts
=
feSpace
->
getBasisFcts
();
int
nBasFcts
=
basFcts
->
getNumber
();
std
::
vector
<
DegreeOfFreedom
>
dofVec
(
nBasFcts
);
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
feSpace
->
getMesh
(),
-
1
,
Mesh
::
CALL_EVERY_EL_PREORDER
);
while
(
elInfo
)
{
basFcts
->
getLocalIndices
(
elInfo
->
getElement
(),
feSpace
->
getAdmin
(),
dofVec
);
for
(
int
i
=
0
;
i
<
nBasFcts
;
i
++
)
if
(
dofVec
[
i
]
==
dof
)
return
elInfo
->
getElement
();
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
return
NULL
;
}
Element
*
getLevel0ParentElement
(
Mesh
*
mesh
,
Element
*
el
)
{
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh
,
-
1
,
Mesh
::
CALL_EVERY_EL_PREORDER
);
while
(
elInfo
)
{
if
(
elInfo
->
getElement
()
==
el
)
return
elInfo
->
getMacroElement
()
->
getElement
();
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
return
NULL
;
}
void
printInfoByDof
(
FiniteElemSpace
*
feSpace
,
DegreeOfFreedom
dof
)
{
Element
*
el
=
getDofIndexElement
(
feSpace
,
dof
);
Element
*
parEl
=
getLevel0ParentElement
(
feSpace
->
getMesh
(),
el
);
std
::
cout
<<
"DOF-INFO: dof = "
<<
dof
<<
" elidx = "
<<
el
->
getIndex
()
<<
" pelidx = "
<<
parEl
->
getIndex
()
<<
std
::
endl
;
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
feSpace
->
getMesh
(),
-
1
,
Mesh
::
CALL_EVERY_EL_PREORDER
);
while
(
elInfo
)
{
if
(
elInfo
->
getElement
()
->
getIndex
()
==
parEl
->
getIndex
())
std
::
cout
<<
"EL INFO TO "
<<
parEl
->
getIndex
()
<<
": "
<<
elInfo
->
getType
()
<<
std
::
endl
;
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
}
}
// namespace debug
}
// namespace AMDiS
AMDiS/src/Debug.h
0 → 100644
View file @
334a275f
// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// ============================================================================
// == ==
// == TU Dresden ==
// == ==
// == Institut fr Wissenschaftliches Rechnen ==
// == Zellescher Weg 12-14 ==
// == 01069 Dresden ==
// == germany ==
// == ==
// ============================================================================
// == ==
// == https://gforge.zih.tu-dresden.de/projects/amdis/ ==
// == ==
// ============================================================================
/** \file Debug.h */
#ifndef AMDIS_DEBUG_H
#define AMDIS_DEBUG_H
#include "AMDiS_fwd.h"
#include "Global.h"
namespace
AMDiS
{
namespace
debug
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void
writeLocalElementDofs
(
int
rank
,
int
elIdx
,
FiniteElemSpace
*
feSpace
);
void
writeMesh
(
FiniteElemSpace
*
feSpace
,
int
rank
,
std
::
string
filename
);
void
writeDofMesh
(
int
rank
,
DegreeOfFreedom
dof
,
FiniteElemSpace
*
feSpace
);
#endif
void
colorDofVectorByLocalElementDofs
(
DOFVector
<
double
>&
vec
,
Element
*
el
);
bool
colorDofVectorByLocalElementDofs
(
DOFVector
<
double
>&
vec
,
Mesh
*
mesh
,
int
elIndex
);
Element
*
getDofIndexElement
(
FiniteElemSpace
*
feSpace
,
DegreeOfFreedom
dof
);
Element
*
getLevel0ParentElement
(
Mesh
*
mesh
,
Element
*
el
);
void
printInfoByDof
(
FiniteElemSpace
*
feSpace
,
DegreeOfFreedom
dof
);
}
}
#endif
AMDiS/src/ParallelDomainBase.cc
View file @
334a275f
#include <algorithm>
#include <boost/lexical_cast.hpp>
#include "ParallelDomainBase.h"
#include "ParMetisPartitioner.h"
...
...
@@ -20,6 +19,7 @@
#include "VertexVector.h"
#include "StdMpi.h"
#include "MeshStructure.h"
#include "Debug.h"
#include "petscksp.h"
...
...
@@ -119,6 +119,10 @@ namespace AMDiS {
removeMacroElements
();
// === Reset all DOFAdmins of the mesh. ===
updateDofAdmins
();
// === If in debug mode, make some tests. ===
#if (DEBUG != 0)
...
...
@@ -127,11 +131,9 @@ namespace AMDiS {
dbgTestInteriorBoundary
();
dbgTestCommonDofs
(
true
);
MSG
(
"Debug mode tests finished!
\n
"
);
#endif
// === Reset all DOFAdmins of the mesh. ===
updateDofAdmins
();
debug
::
writeMesh
(
feSpace
,
-
1
,
"macromesh"
);
#endif
// === Create periodic dof mapping, if there are periodic boundaries. ===
...
...
@@ -167,17 +169,17 @@ namespace AMDiS {
// There must be always more allocated DOFs than used DOFs in DOFAdmin. Otherwise,
// it is not possible to define a first DOF hole.
if
(
static_cast
<
unsigned
int
>
(
admin
.
getSize
())
==
mapLocalGlobalD
OF
s
.
size
())
if
(
static_cast
<
unsigned
int
>
(
admin
.
getSize
())
==
mapLocalGlobalD
of
s
.
size
())
admin
.
enlargeDOFLists
();
for
(
int
j
=
0
;
j
<
admin
.
getSize
();
j
++
)
admin
.
setDOFFree
(
j
,
true
);
for
(
unsigned
int
j
=
0
;
j
<
mapLocalGlobalD
OF
s
.
size
();
j
++
)
for
(
unsigned
int
j
=
0
;
j
<
mapLocalGlobalD
of
s
.
size
();
j
++
)
admin
.
setDOFFree
(
j
,
false
);
admin
.
setUsedSize
(
mapLocalGlobalD
OF
s
.
size
());
admin
.
setUsedCount
(
mapLocalGlobalD
OF
s
.
size
());
admin
.
setFirstHole
(
mapLocalGlobalD
OF
s
.
size
());
admin
.
setUsedSize
(
mapLocalGlobalD
of
s
.
size
());
admin
.
setUsedCount
(
mapLocalGlobalD
of
s
.
size
());
admin
.
setFirstHole
(
mapLocalGlobalD
of
s
.
size
());
}
}
...
...
@@ -236,7 +238,7 @@ namespace AMDiS {
values
.
clear
();
// Global index of the current row dof.
int
globalRowDof
=
mapLocalGlobalD
OF
s
[
*
cursor
];
int
globalRowDof
=
mapLocalGlobalD
of
s
[
*
cursor
];
// Test if the current row dof is a periodic dof.
bool
periodicRow
=
(
periodicDof
.
count
(
globalRowDof
)
>
0
);
...
...
@@ -251,7 +253,7 @@ namespace AMDiS {
// Set only non null values.
if
(
value
(
*
icursor
)
!=
0.0
)
{
// Global index of the current column index.
int
globalColDof
=
mapLocalGlobalD
OF
s
[
col
(
*
icursor
)];
int
globalColDof
=
mapLocalGlobalD
of
s
[
col
(
*
icursor
)];
// Calculate the exact position of the column index in the petsc matrix.
int
colIndex
=
globalColDof
*
dispMult
+
dispAddCol
;
...
...
@@ -337,7 +339,7 @@ namespace AMDiS {
DOFVector
<
double
>::
Iterator
dofIt
(
vec
,
USED_DOFS
);
for
(
dofIt
.
reset
();
!
dofIt
.
end
();
++
dofIt
)
{
// Calculate global row index of the dof.
int
globalRow
=
mapLocalGlobalD
OF
s
[
dofIt
.
getDOFIndex
()];
int
globalRow
=
mapLocalGlobalD
of
s
[
dofIt
.
getDOFIndex
()];
// Calculate petsc index of the row dof.
int
index
=
globalRow
*
dispMult
+
dispAdd
;
...
...
@@ -368,6 +370,8 @@ namespace AMDiS {
clock_t
first
=
clock
();
// === Create PETSc vector (rhs, solution and a temporary vector). ===
VecCreate
(
PETSC_COMM_WORLD
,
&
petscRhsVec
);
VecSetSizes
(
petscRhsVec
,
nRankRows
,
nOverallRows
);
VecSetType
(
petscRhsVec
,
VECMPI
);
...
...
@@ -381,19 +385,25 @@ namespace AMDiS {
VecSetType
(
petscTmpVec
,
VECMPI
);
using
mtl
::
tag
::
row
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
namespace
traits
=
mtl
::
traits
;
namespace
traits
=
mtl
::
traits
;
typedef
DOFMatrix
::
base_matrix_type
Matrix
;
typedef
std
::
vector
<
std
::
pair
<
int
,
int
>
>
MatrixNnzEntry
;
int
d_nnz
[
nRankRows
];
int
o_nnz
[
nRankRows
];
std
::
map
<
int
,
std
::
vector
<
std
::
pair
<
int
,
int
>
>
>
sendMatrixEntry
;
for
(
int
i
=
0
;
i
<
nRankRows
;
i
++
)
{
d_nnz
[
i
]
=
0
;
o_nnz
[
i
]
=
0
;
}
MSG
(
"Is rank DOF: %d
\n
"
,
isRankDof
[
6717
-
rstart
*
nComponents
]);
// Stores to each rank a list of nnz entries (i.e. pairs of row and column index)
// that this rank will send to. This nnz entries will be assembled on this rank,
// but because the row DOFs are not DOFs of this rank they will be send to the
// owner of the row DOFs.
std
::
map
<
int
,
MatrixNnzEntry
>
sendMatrixEntry
;
for
(
int
i
=
0
;
i
<
nComponents
;
i
++
)
{
for
(
int
j
=
0
;
j
<
nComponents
;
j
++
)
{
if
((
*
mat
)[
i
][
j
])
{
...
...
@@ -408,41 +418,63 @@ namespace AMDiS {
for
(
cursor_type
cursor
=
begin
<
row
>
(
bmat
),
cend
=
end
<
row
>
(
bmat
);
cursor
!=
cend
;
++
cursor
)
{
int
r
=
mapLocalGlobalDOFs
[
*
cursor
]
*
nComponents
+
i
;
// Map the local row number to the global DOF index and create from it
// the global PETSc row index of this DOF.
int
petscRowIdx
=
mapLocalGlobalDofs
[
*
cursor
]
*
nComponents
+
i
;
if
(
isRankDof
[
*
cursor
])
{
r
-=
rstart
*
nComponents
;
// === The current row DOF is a rank dof, so create the corresponding ===
// === nnz values directly on rank's nnz data. ===
// This is the local row index of the local PETSc matrix.
int
localPetscRowIdx
=
petscRowIdx
-
rstart
*
nComponents
;
#if (DEBUG != 0)
if
(
r
<
0
||
r
>=
nRankRows
)
{
if
(
localPetscRowIdx
<
0
||
localPetscRowIdx
>=
nRankRows
)
{
std
::
cout
<<
"ERROR in rank: "
<<
mpiRank
<<
std
::
endl
;
std
::
cout
<<
" Wrong r = "
<<
r
<<
" "
<<
*
cursor
<<
" "
<<
mapLocalGlobalD
OF
s
[
*
cursor
]
<<
" "
std
::
cout
<<
" Wrong r = "
<<
localPetscRowIdx
<<
" "
<<
*
cursor
<<
" "
<<
mapLocalGlobalD
of
s
[
*
cursor
]
<<
" "
<<
nRankRows
<<
std
::
endl
;
ERROR_EXIT
(
"Should not happen!
\n
"
);
}
#endif
// Traverse all non zero entries in this row.
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
if
(
value
(
*
icursor
)
!=
0.0
)
{
int
c
=
mapLocalGlobalD
OF
s
[
col
(
*
icursor
)]
*
nComponents
+
j
;
int
petscColIdx
=
mapLocalGlobalD
of
s
[
col
(
*
icursor
)]
*
nComponents
+
j
;
if
(
c
>=
rstart
*
nComponents
&&
c
<
rstart
*
nComponents
+
nRankRows
)
d_nnz
[
r
]
++
;
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the d_nnz values for this row, otherwise the o_nnz value.
if
(
petscColIdx
>=
rstart
*
nComponents
&&
petscColIdx
<
rstart
*
nComponents
+
nRankRows
)
d_nnz
[
localPetscRowIdx
]
++
;
else
o_nnz
[
r
]
++
;
o_nnz
[
localPetscRowIdx
]
++
;
}
}
}
else
{
int
sendToRank
=
-
1
;
// === The current row DOF is not a rank dof, i.e., it will be created ===
// === on this rank, but after this it will be send to another rank ===
// === matrix. So we need to send also the corresponding nnz structure ===
// === of this row to the corresponding rank. ===
// Find out who is the member of this DOF.
int
sendToRank
=
-
1
;
for
(
RankToDofContainer
::
iterator
it
=
recvDofs
.
begin
();
it
!=
recvDofs
.
end
();
++
it
)
{
for
(
DofContainer
::
iterator
dofIt
=
it
->
second
.
begin
();
dofIt
!=
it
->
second
.
end
();
++
dofIt
)
{
if
(
**
dofIt
==
*
cursor
)
{
if
(
petscRowIdx
==
6717
)
{
debug
::
writeDofMesh
(
mpiRank
,
*
cursor
,
feSpace
);
MSG
(
"SEND DOF TO: %d/%d
\n
"
,
it
->
first
,
*
cursor
);
}
sendToRank
=
it
->
first
;
break
;
}
...
...
@@ -454,117 +486,56 @@ namespace AMDiS {
TEST_EXIT_DBG
(
sendToRank
!=
-
1
)(
"Should not happen!
\n
"
);
// Send all non zero entries to the member of the row DOF.
for
(
icursor_type
icursor
=
begin
<
nz
>
(
cursor
),
icend
=
end
<
nz
>
(
cursor
);
icursor
!=
icend
;
++
icursor
)
{
if
(
value
(
*
icursor
)
!=
0.0
)
{
int
c
=
mapLocalGlobalD
OF
s
[
col
(
*
icursor
)]
*
nComponents
+
j
;
int
petscColIdx
=
mapLocalGlobalD
of
s
[
col
(
*
icursor
)]
*
nComponents
+
j
;
sendMatrixEntry
[
sendToRank
].
push_back
(
std
::
make_pair
(
r
,
c
));
sendMatrixEntry
[
sendToRank
].
push_back
(
std
::
make_pair
(
petscRowIdx
,
petscColIdx
));
}
}
}
}
}
}
}
MPI
::
Request
request
[
sendDofs
.
size
()
+
recvDofs
.
size
()];
int
requestCounter
=
0
;
std
::
vector
<
int
*>
sendBuffers
;
sendBuffers
.
reserve
(
recvDofs
.
size
());
for
(
RankToDofContainer
::
iterator
it
=
recvDofs
.
begin
();
it
!=
recvDofs
.
end
();
++
it
)
{
int
nSend
=
sendMatrixEntry
[
it
->
first
].
size
();
request
[
requestCounter
++
]
=
mpiComm
.
Isend
(
&
nSend
,
1
,
MPI_INT
,
it
->
first
,
0
);
if
(
nSend
>
0
)
{
int
*
sendbuf
=
new
int
[
nSend
*
2
];
for
(
int
i
=
0
;
i
<
nSend
;
i
++
)
{
sendbuf
[
i
*
2
]
=
sendMatrixEntry
[
it
->
first
][
i
].
first
;
sendbuf
[
i
*
2
+
1
]
=
sendMatrixEntry
[
it
->
first
][
i
].
second
;
}
sendBuffers
.
push_back
(
sendbuf
);
}
else
{
sendBuffers
.
push_back
(
NULL
);
}
}
std
::
vector
<
int
>
recvSize
;
recvSize
.
reserve
(
sendDofs
.
size
());
int
i
=
0
;
for
(
RankToDofContainer
::
iterator
it
=
sendDofs
.
begin
();
it
!=
sendDofs
.
end
();
++
it
)
request
[
requestCounter
++
]
=
mpiComm
.
Irecv
(
&
(
recvSize
[
i
++
]),
1
,
MPI_INT
,
it
->
first
,
0
);
MPI
::
Request
::
Waitall
(
requestCounter
,
request
);
requestCounter
=
0
;
i
=
0
;
for
(
RankToDofContainer
::
iterator
it
=
recvDofs
.
begin
();
it
!=
recvDofs
.
end
();
++
it
)
{
int
nSend
=
sendMatrixEntry
[
it
->
first
].
size
();
if
(
nSend
>
0
)
request
[
requestCounter
++
]
=
mpiComm
.
Isend
(
sendBuffers
[
i
],
nSend
*
2
,
MPI_INT
,
it
->
first
,
0
);
i
++
;
}
// if (isRankDof[*cursor]) ... else ...
}
// for each row in mat[i][j]
}
// if mat[i][j]
}
}
std
::
vector
<
int
*>
recvBuffers
;
recvBuffers
.
reserve
(
sendDofs
.
size
());
i
=
0
;
for
(
RankToDofContainer
::
iterator
it
=
sendDofs
.
begin
();
it
!=
sendDofs
.
end
();
++
it
)
{
if
(
recvSize
[
i
]
>
0
)
{
int
*
recvbuf
=
new
int
[
recvSize
[
i
]
*
2
];
recvBuffers
.
push_back
(
recvbuf
);
// === Send and recv the nnz row structure to/from other ranks. ===
request
[
requestCounter
++
]
=
mpiComm
.
Irecv
(
recvbuf
,
recvSize
[
i
]
*
2
,
MPI_INT
,
it
->
first
,
0
);
}
else
{
recvBuffers
.
push_back
(
NULL
);
}
i
++
;
}
MPI
::
Request
::
Waitall
(
requestCounter
,
request
);
StdMpi
<
MatrixNnzEntry
,
MatrixNnzEntry
>
stdMpi
(
mpiComm
);
stdMpi
.
prepareCommunication
(
true
);
stdMpi
.
send
(
sendMatrixEntry
);
stdMpi
.
recv
(
sendDofs
);
stdMpi
.
startCommunication
<
int
>
(
MPI_INT
);
for
(
int
j
=
0
;
j
<
static_cast
<
int
>
(
sendBuffers
.
size
());
j
++
)
if
(
sendBuffers
[
j
])
delete
[]
sendBuffers
[
j
];
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
// === the PETSc nnz data structure. ===
i
=
0
;
for
(
RankToDofContainer
::
iterator
it
=
sendDofs
.
begin
();
it
!=
sendDofs
.
end
();
++
it
)
{
if
(
recvSize
[
i
]
>
0
)
{
for
(
int
j
=
0
;
j
<
recvSize
[
i
];
j
++
)
{
int
r
=
recvBuffers
[
i
][
j
*
2
];
int
c
=
recvBuffers
[
i
][
j
*
2
+
1
];
for
(
std
::
map
<
int
,
MatrixNnzEntry
>::
iterator
it
=
stdMpi
.
getRecvData
().
begin
();
it
!=
stdMpi
.
getRecvData
().
end
();
++
it
)
{
if
(
it
->
second
.
size
()
>
0
)
{
for
(
unsigned
int
i
=
0
;
i
<
it
->
second
.
size
();
i
++
)
{
int
r
=
it
->
second
[
i
].
first
;
int
c
=
it
->
second
[
i
].
second
;
r
-
=
rstart
*
nComponents
;
int
localRowIdx
=
r
-
rstart
*
nComponents
;
TEST_EXIT_DBG
(
r
>=
0
&&
r
<
nRankRows
)(
"Should not happen!
\n
"
);
TEST_EXIT_DBG
(
localRowIdx
>=
0
&&
localRowIdx
<
nRankRows
)
(
"Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!
\n
"
,
r
,
localRowIdx
,
nRankRows
,
it
->
first
);
if
(
c
<
rstart
*
nComponents
||
c
>=
rstart
*
nComponents
+
nRankRows
)
o_nnz
[
r
]
++
;
o_nnz
[
localRowIdx
]
++
;
else
d_nnz
[
r
]
++
;
d_nnz
[
localRowIdx
]
++
;
}
delete
[]
recvBuffers
[
i
];
}
}
i
++
;
}
// === Create PETSc matrix with the computed nnz data structure. ===
MatCreateMPIAIJ
(
PETSC_COMM_WORLD
,
nRankRows
,
nRankRows
,
nOverallRows
,
nOverallRows
,
0
,
d_nnz
,
0
,
o_nnz
,
&
petscMatrix
);
...
...
@@ -576,9 +547,7 @@ namespace AMDiS {
TEST_EXIT
(
b
==
rstart
*
nComponents
+
nRankRows
)(
"Wrong matrix ownership range!
\n
"
);
#endif
using
mtl
::
tag
::
major
;
using
mtl
::
tag
::
nz
;
using
mtl
::
begin
;
using
mtl
::
end
;
namespace
traits
=
mtl
::
traits
;
typedef
DOFMatrix
::
base_matrix_type
Matrix
;
// === Transfer values from DOF matrices to the PETSc matrix. ===
for
(
int
i
=
0
;
i
<
nComponents
;
i
++
)
for
(
int
j
=
0
;
j
<
nComponents
;
j