Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Backofen, Rainer
amdis
Commits
f9a324f2
Commit
f9a324f2
authored
Jan 18, 2011
by
Thomas Witkowski
Browse files
Some code refactoring of the last changes. Added some more comments to parallel code.
parent
8d3f6ec7
Changes
7
Hide whitespace changes
Inline
Side-by-side
AMDiS/src/parallel/MeshDistributor.cc
View file @
f9a324f2
...
...
@@ -70,8 +70,8 @@ namespace AMDiS {
writeSerializationFile
(
false
),
repartitioningAllowed
(
false
),
repartitionIthChange
(
20
),
n
Timestep
sAfterLastRepartitioning
(
0
),
repartCounter
(
0
),
n
MeshChange
sAfterLastRepartitioning
(
0
),
repart
itioning
Counter
(
0
),
debugOutputDir
(
""
),
lastMeshChangeIndex
(
0
)
{
...
...
@@ -168,9 +168,7 @@ namespace AMDiS {
if
(
writePartMesh
>
0
)
{
debug
::
writeElementIndexMesh
(
mesh
,
debugOutputDir
+
"elementIndex.vtu"
);
writePartitioningMesh
(
debugOutputDir
+
"part.vtu"
);
}
else
{
MSG
(
"Skip write part mesh!
\n
"
);
ParallelDebug
::
writePartitioning
(
*
this
,
debugOutputDir
+
"part.vtu"
);
}
}
#endif
...
...
@@ -607,7 +605,7 @@ namespace AMDiS {
// === Check on all ranks if at least one rank's mesh has changed. ===
int
sendValue
=
static_cast
<
int
>
(
!
meshChanged
);
int
sendValue
=
static_cast
<
int
>
(
meshChanged
);
recvAllValues
=
0
;
mpiComm
.
Allreduce
(
&
sendValue
,
&
recvAllValues
,
1
,
MPI_INT
,
MPI_SUM
);
...
...
@@ -635,13 +633,12 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
n
Timestep
sAfterLastRepartitioning
++
;
n
MeshChange
sAfterLastRepartitioning
++
;
if
(
repartitioningAllowed
)
{
if
(
nTimestepsAfterLastRepartitioning
>=
repartitionIthChange
)
{
repartitionMesh
();
nTimestepsAfterLastRepartitioning
=
0
;
}
if
(
repartitioningAllowed
&&
nMeshChangesAfterLastRepartitioning
>=
repartitionIthChange
)
{
repartitionMesh
();
nMeshChangesAfterLastRepartitioning
=
0
;
}
}
...
...
@@ -671,7 +668,7 @@ namespace AMDiS {
// === Compare received mesh structure codes. ===
bool
mesh
FitTogether
=
tru
e
;
bool
mesh
Changed
=
fals
e
;
for
(
RankToBoundMap
::
iterator
it
=
allBound
.
begin
();
it
!=
allBound
.
end
();
++
it
)
{
...
...
@@ -679,7 +676,7 @@ namespace AMDiS {
int
i
=
0
;
for
(
vector
<
AtomicBoundary
>::
iterator
boundIt
=
it
->
second
.
begin
();
boundIt
!=
it
->
second
.
end
();
++
boundIt
)
{
boundIt
!=
it
->
second
.
end
();
++
boundIt
,
i
++
)
{
MeshStructure
elCode
;
elCode
.
init
(
boundIt
->
rankObj
);
...
...
@@ -688,23 +685,18 @@ namespace AMDiS {
TEST_EXIT_DBG
(
refineManager
)(
"Refinement manager is not set correctly!
\n
"
);
MeshManipulation
meshManipulation
(
feSpace
);
bool
b
=
meshChanged
|=
meshManipulation
.
startFitElementToMeshCode
(
recvCodes
[
i
],
boundIt
->
rankObj
.
el
,
boundIt
->
rankObj
.
subObj
,
boundIt
->
rankObj
.
ithObj
,
boundIt
->
rankObj
.
elType
,
boundIt
->
rankObj
.
reverseMode
);
if
(
b
)
meshFitTogether
=
false
;
}
i
++
;
}
}
return
mesh
FitTogether
;
return
mesh
Changed
;
}
...
...
@@ -809,10 +801,15 @@ namespace AMDiS {
TEST_EXIT
(
mesh
->
getNumberOfDOFAdmin
()
==
1
)
(
"Only meshes with one DOFAdmin are supported!
\n
"
);
int
nDofs
=
mesh
->
getDofAdmin
(
0
).
getUsedDofs
();
// === First we check if the rank with the maximum number of DOFs has at ===
// === least 20% more DOFs than the rank with the minimum number of DOFs. ===
// === In this case, the mesh will be repartition. ===
int
repartitioning
=
0
;
vector
<
int
>
nDofsInRank
(
mpiSize
);
int
nDofs
=
mesh
->
getDofAdmin
(
0
).
getUsedDofs
();
mpiComm
.
Gather
(
&
nDofs
,
1
,
MPI_INT
,
&
(
nDofsInRank
[
0
]),
1
,
MPI_INT
,
0
);
if
(
mpiRank
==
0
)
{
...
...
@@ -828,7 +825,7 @@ namespace AMDiS {
MSG
(
"Overall DOFs: %d Min DOFs: %d Max DOFs: %d
\n
"
,
nOverallDofs
,
minDofs
,
maxDofs
);
if
(
static_cast
<
double
>
(
maxDofs
)
/
static_cast
<
double
>
(
minDofs
))
if
(
static_cast
<
double
>
(
maxDofs
)
/
static_cast
<
double
>
(
minDofs
)
>
1.2
)
repartitioning
=
1
;
mpiComm
.
Bcast
(
&
repartitioning
,
1
,
MPI_INT
,
0
);
...
...
@@ -841,25 +838,19 @@ namespace AMDiS {
return
;
double
timePoint
=
MPI
::
Wtime
();
repartitioningCounter
++
;
#if (DEBUG != 0)
ParallelDebug
::
testDoubleDofs
(
mesh
);
if
(
repartCounter
==
0
)
{
stringstream
oss
;
oss
<<
debugOutputDir
<<
"partitioning-"
<<
repartCounter
<<
".vtu"
;
MSG
(
"Write partitioning to %s
\n
"
,
oss
.
str
().
c_str
());
DOFVector
<
double
>
tmpa
(
feSpace
,
"tmp"
);
tmpa
.
set
(
mpiRank
);
VtkWriter
::
writeFile
(
&
tmpa
,
oss
.
str
());
repartCounter
++
;
}
if
(
repartitioningCounter
==
0
)
ParallelDebug
::
writePartitioningFile
(
debugOutputDir
+
"partitioning"
,
repartitioningCounter
,
feSpace
);
#endif
// === Create new element weights. ===
elemWeights
.
clear
();
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh
,
-
1
,
Mesh
::
CALL_LEAF_EL
);
while
(
elInfo
)
{
...
...
@@ -867,15 +858,16 @@ namespace AMDiS {
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
// === Run ParMETiS to calculate a new mesh partitioning. ===
partitioner
->
useLocalGlobalDofMap
(
&
mapLocalGlobalDofs
);
bool
partitioningSucceed
=
partitioner
->
partition
(
elemWeights
,
ADAPTIVE_REPART
,
1000.0
);
if
(
!
partitioningSucceed
)
{
MSG
(
"ParMETIS created empty partition!
\n
"
);
return
;
}
oldPartitionVec
=
partitionVec
;
...
...
@@ -1045,32 +1037,17 @@ namespace AMDiS {
mesh
->
dofCompress
();
#if (DEBUG != 0)
stringstream
oss
;
oss
<<
debugOutputDir
<<
"partitioning-"
<<
repartCounter
<<
".vtu"
;
DOFVector
<
double
>
tmpa
(
feSpace
,
"tmp"
);
tmpa
.
set
(
mpiRank
);
VtkWriter
::
writeFile
(
&
tmpa
,
oss
.
str
());
repartCounter
++
;
ParallelDebug
::
testAllElements
(
*
this
);
ParallelDebug
::
testDoubleDofs
(
mesh
);
#endif
partitioner
->
fillCoarsePartitionVec
(
&
partitionVec
);
updateInteriorBoundaryInfo
();
#if (DEBUG != 0)
ParallelDebug
::
printBoundaryInfo
(
*
this
);
#endif
updateLocalGlobalNumbering
();
#if (DEBUG != 0)
MSG
(
"AMDiS runs in debug mode, so make some test ...
\n
"
);
ParallelDebug
::
writePartitioningFile
(
debugOutputDir
+
"partitioning"
,
repartitioningCounter
,
feSpace
);
ParallelDebug
::
testAllElements
(
*
this
);
ParallelDebug
::
testDoubleDofs
(
mesh
);
ParallelDebug
::
testInteriorBoundary
(
*
this
);
ParallelDebug
::
printBoundaryInfo
(
*
this
);
...
...
@@ -1096,6 +1073,10 @@ namespace AMDiS {
elObjects
.
createRankData
(
partitionVec
);
createBoundaryData
();
#if (DEBUG != 0)
ParallelDebug
::
printBoundaryInfo
(
*
this
);
#endif
}
...
...
@@ -1872,8 +1853,8 @@ namespace AMDiS {
for
(
int
i
=
0
;
i
<
nSize
;
i
++
)
allMacroElements
[
i
]
->
serialize
(
out
);
SerUtil
::
serialize
(
out
,
n
Timestep
sAfterLastRepartitioning
);
SerUtil
::
serialize
(
out
,
repartCounter
);
SerUtil
::
serialize
(
out
,
n
MeshChange
sAfterLastRepartitioning
);
SerUtil
::
serialize
(
out
,
repart
itioning
Counter
);
}
...
...
@@ -1939,8 +1920,8 @@ namespace AMDiS {
allMacroElements
[
i
]
->
getElement
()
->
setMesh
(
mesh
);
}
SerUtil
::
deserialize
(
in
,
n
Timestep
sAfterLastRepartitioning
);
SerUtil
::
deserialize
(
in
,
repartCounter
);
SerUtil
::
deserialize
(
in
,
n
MeshChange
sAfterLastRepartitioning
);
SerUtil
::
deserialize
(
in
,
repart
itioning
Counter
);
deserialized
=
true
;
}
...
...
@@ -2014,23 +1995,4 @@ namespace AMDiS {
}
}
void
MeshDistributor
::
writePartitioningMesh
(
string
filename
)
{
FUNCNAME
(
"MeshDistributor::writePartitioningMesh()"
);
map
<
int
,
double
>
vec
;
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh
,
-
1
,
Mesh
::
CALL_LEAF_EL
|
Mesh
::
FILL_COORDS
);
while
(
elInfo
)
{
int
index
=
elInfo
->
getElement
()
->
getIndex
();
vec
[
index
]
=
partitionVec
[
index
];
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
ElementFileWriter
::
writeFile
(
vec
,
mesh
,
filename
);
}
}
AMDiS/src/parallel/MeshDistributor.h
View file @
f9a324f2
...
...
@@ -303,11 +303,13 @@ namespace AMDiS {
* refinement of a certain element may forces the refinement of other elements,
* it is not guaranteed that all rank's meshes fit together after this function
* terminates. Hence, it must be called until a stable mesh refinement is reached.
* If the mesh has been changed by this function, it returns true. Otherwise, it
* returns false, i.e., the given interior boundaries fit together on both sides.
*
* \param[in] allBound Defines a map from rank to interior boundaries which
* should be checked.
*
* \return If the mesh has been changed by this function, it returns true.
* Otherwise, it returns false, i.e., the given interior boundaries
* fit together on both sides.
*/
bool
checkAndAdaptBoundary
(
RankToBoundMap
&
allBound
);
...
...
@@ -318,13 +320,6 @@ namespace AMDiS {
*/
void
repartitionMesh
();
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of. This function can be used for
* debugging.
*/
void
writePartitioningMesh
(
string
filename
);
/// Sets \ref isRankDof to all matrices and rhs vectors in all stationary problems.
void
setRankDofs
();
...
...
@@ -550,11 +545,15 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool
repartitioningAllowed
;
/// Stores the number of mesh changes that must lie in between to repartitionings.
int
repartitionIthChange
;
int
nTimestepsAfterLastRepartitioning
;
/// Counts the number of mesh changes after the last mesh repartitioning was done.
int
nMeshChangesAfterLastRepartitioning
;
int
repartCounter
;
/// Countes the number of mesh repartitions that were done. Till now, this
/// variable is used only for debug outputs.
int
repartitioningCounter
;
/// Directory name where all debug output files should be written to.
string
debugOutputDir
;
...
...
AMDiS/src/parallel/MeshManipulation.cc
View file @
f9a324f2
...
...
@@ -247,8 +247,7 @@ namespace AMDiS {
TEST_EXIT_DBG
(
elInfo
->
getElement
()
==
el
)(
"This should not happen!
\n
"
);
meshChanged
=
fitElementToMeshCode
(
code
,
stack
,
subObj
,
ithObj
,
reverseMode
);
return
meshChanged
;
return
fitElementToMeshCode
(
code
,
stack
,
subObj
,
ithObj
,
reverseMode
);
}
...
...
AMDiS/src/parallel/MeshManipulation.h
View file @
f9a324f2
...
...
@@ -54,6 +54,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool
startFitElementToMeshCode
(
MeshStructure
&
code
,
Element
*
el
,
...
...
@@ -77,6 +80,9 @@ namespace AMDiS {
* \param[in] reverseMode Defines, whether the mesh structure code is given
* in reverse mode, i.e., left and right children where
* changed when the code was created.
*
* \return Returns true, if the mesh was changed, i.e. refined, to fit the
* element to the structure code.
*/
bool
fitElementToMeshCode
(
MeshStructure
&
code
,
TraverseStack
&
stack
,
...
...
AMDiS/src/parallel/ParallelDebug.cc
View file @
f9a324f2
...
...
@@ -18,6 +18,7 @@
#include
"StdMpi.h"
#include
"Debug.h"
#include
"MpiHelper.h"
#include
"io/VtkWriter.h"
namespace
AMDiS
{
...
...
@@ -806,4 +807,39 @@ namespace AMDiS {
file
.
close
();
}
void
ParallelDebug
::
writePartitioning
(
MeshDistributor
&
pdb
,
string
filename
)
{
FUNCNAME
(
"ParallelDebug::writeParitioning()"
);
map
<
int
,
double
>
vec
;
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
pdb
.
mesh
,
-
1
,
Mesh
::
CALL_LEAF_EL
|
Mesh
::
FILL_COORDS
);
while
(
elInfo
)
{
int
index
=
elInfo
->
getElement
()
->
getIndex
();
vec
[
index
]
=
pdb
.
partitionVec
[
index
];
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
ElementFileWriter
::
writeFile
(
vec
,
pdb
.
mesh
,
filename
);
}
void
ParallelDebug
::
writePartitioningFile
(
std
::
string
filename
,
int
counter
,
FiniteElemSpace
*
feSpace
)
{
FUNCNAME
(
"ParallelDebug::writePartitioningFile()"
);
stringstream
oss
;
oss
<<
filename
;
if
(
counter
>=
0
)
oss
<<
"-"
<<
counter
;
oss
<<
".vtu"
;
DOFVector
<
double
>
tmpa
(
feSpace
,
"tmp"
);
tmpa
.
set
(
MPI
::
COMM_WORLD
.
Get_rank
());
VtkWriter
::
writeFile
(
&
tmpa
,
oss
.
str
());
}
}
AMDiS/src/parallel/ParallelDebug.h
View file @
f9a324f2
...
...
@@ -142,6 +142,25 @@ namespace AMDiS {
static
void
writeDebugFile
(
MeshDistributor
&
pdb
,
std
::
string
prefix
,
std
::
string
postfix
);
/** \brief
* This functions create a Paraview file with the macro mesh where the elements
* are colored by the partition they are part of.
*/
static
void
writePartitioning
(
MeshDistributor
&
pdb
,
string
filename
);
/** \brief
* The mesh is written to a value and all values are assigned by the rank number
* where the vertex is contained in.
*
* \param[in] filename Name of the output file without extension (.vtu).
* \param[in] counter Counter index. If not negative, this number is added
* to the filename.
* \param[in] feSpace
*/
static
void
writePartitioningFile
(
std
::
string
filename
,
int
counter
,
FiniteElemSpace
*
feSpace
);
};
}
// namespace AMDiS
...
...
AMDiS/src/parallel/ParallelProblemStatBase.cc
View file @
f9a324f2
...
...
@@ -30,12 +30,12 @@ namespace AMDiS {
vm
/=
1024.0
;
rss
/=
1024.0
;
MSG
(
"My memory usage is VM = %f MB RSS = %f MB
\n
"
,
vm
,
rss
);
MSG
(
"My memory usage is VM = %
.1
f MB RSS = %
.1
f MB
\n
"
,
vm
,
rss
);
mpi
::
globalAdd
(
vm
);
mpi
::
globalAdd
(
rss
);
MSG
(
"Overall memory usage is VM = %f MB RSS = %f MB
\n
"
,
vm
,
rss
);
MSG
(
"Overall memory usage is VM = %
.1
f MB RSS = %
.1
f MB
\n
"
,
vm
,
rss
);
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment