Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Backofen, Rainer
amdis
Commits
8c23f60f
Commit
8c23f60f
authored
Jan 27, 2011
by
Thomas Witkowski
Browse files
Small improvements in parallel code.
parent
41220723
Changes
12
Hide whitespace changes
Inline
Side-by-side
AMDiS/src/DualTraverse.cc
View file @
8c23f60f
...
...
@@ -208,14 +208,9 @@ namespace AMDiS {
if
(
largeElInfo
->
getLevel
()
==
smallElInfo
->
getLevel
())
return
largeFace
;
int
refinementPathLength
=
smallElInfo
->
getRefinementPathLength
();
unsigned
long
refinementPath
=
smallElInfo
->
getRefinementPath
();
TEST_EXIT_DBG
(
refinementPathLength
!=
0
)
TEST_EXIT_DBG
(
smallElInfo
->
getRefinementPathLength
()
!=
0
)
(
"Refinement path is empty. This should not happen!
\n
"
);
std
::
cout
<<
"REFINEMENTPATHLENGTH = "
<<
refinementPathLength
<<
std
::
endl
;
return
-
1
;
}
}
AMDiS/src/Traverse.cc
View file @
8c23f60f
...
...
@@ -39,7 +39,7 @@ namespace AMDiS {
traverse_fill_flag
=
fill_flag
;
TEST_EXIT_DBG
(
mesh
)(
"No mesh!
\n
"
);
TEST_EXIT
_DBG
(
traverse_mesh
->
getMacroElements
().
size
()
>
0
)(
"Mesh is empty!
\n
"
);
TEST_EXIT
(
traverse_mesh
->
getMacroElements
().
size
()
>
0
)(
"Mesh is empty!
\n
"
);
if
(
stack_size
<
1
)
enlargeTraverseStack
();
...
...
AMDiS/src/parallel/ElementObjectData.cc
View file @
8c23f60f
...
...
@@ -422,10 +422,14 @@ namespace AMDiS {
{
FUNCNAME
(
"ElementObjects::createReverseModeData()"
);
// In 2D, all reverse modes are always true!
// === In 2D, all reverse modes are always true! ===
if
(
mesh
->
getDim
()
==
2
)
return
;
// === First, create reverse modes for all "directly" neighbouring elements. ===
for
(
map
<
DofEdge
,
vector
<
ElementObjectData
>
>::
iterator
edgeIt
=
edgeElements
.
begin
();
edgeIt
!=
edgeElements
.
end
();
++
edgeIt
)
{
...
...
@@ -473,6 +477,54 @@ namespace AMDiS {
}
}
}
// === And create reverse modes for periodic neighbouring elements. ===
for
(
PerBoundMap
<
DofEdge
>::
iterator
edgeIt
=
periodicEdges
.
begin
();
edgeIt
!=
periodicEdges
.
end
();
++
edgeIt
)
{
vector
<
ElementObjectData
>
&
edges0
=
edgeElements
[
edgeIt
->
first
.
first
];
vector
<
ElementObjectData
>
&
edges1
=
edgeElements
[
edgeIt
->
first
.
second
];
for
(
unsigned
int
i
=
0
;
i
<
edges0
.
size
();
i
++
)
{
BoundaryObject
obj0
(
elIndexMap
[
edges0
[
i
].
elIndex
],
elIndexTypeMap
[
edges0
[
i
].
elIndex
],
EDGE
,
edges0
[
i
].
ithObject
);
for
(
unsigned
int
j
=
0
;
j
<
edges1
.
size
();
j
++
)
{
BoundaryObject
obj1
(
elIndexMap
[
edges1
[
j
].
elIndex
],
elIndexTypeMap
[
edges1
[
j
].
elIndex
],
EDGE
,
edges1
[
j
].
ithObject
);
bool
reverseMode
=
BoundaryObject
::
computeReverseMode
(
obj0
,
obj1
,
feSpace
,
edgeIt
->
second
);
edgeReverseMode
[
make_pair
(
edges0
[
i
],
edges1
[
j
])]
=
reverseMode
;
edgeReverseMode
[
make_pair
(
edges1
[
j
],
edges0
[
i
])]
=
reverseMode
;
}
}
}
for
(
PerBoundMap
<
DofFace
>::
iterator
faceIt
=
periodicFaces
.
begin
();
faceIt
!=
periodicFaces
.
end
();
++
faceIt
)
{
vector
<
ElementObjectData
>
&
faces0
=
faceElements
[
faceIt
->
first
.
first
];
vector
<
ElementObjectData
>
&
faces1
=
faceElements
[
faceIt
->
first
.
second
];
TEST_EXIT_DBG
(
faces0
.
size
()
==
faces1
.
size
()
==
1
)(
"Should not happen!
\n
"
);
BoundaryObject
obj0
(
elIndexMap
[
faces0
[
0
].
elIndex
],
elIndexTypeMap
[
faces0
[
0
].
elIndex
],
FACE
,
faces0
[
0
].
ithObject
);
BoundaryObject
obj1
(
elIndexMap
[
faces1
[
0
].
elIndex
],
elIndexTypeMap
[
faces1
[
0
].
elIndex
],
FACE
,
faces1
[
0
].
ithObject
);
bool
reverseMode
=
BoundaryObject
::
computeReverseMode
(
obj0
,
obj1
,
feSpace
,
faceIt
->
second
);
faceReverseMode
[
make_pair
(
faces0
[
0
],
faces1
[
0
])]
=
reverseMode
;
faceReverseMode
[
make_pair
(
faces1
[
0
],
faces0
[
0
])]
=
reverseMode
;
}
}
...
...
AMDiS/src/parallel/InteriorBoundary.cc
View file @
8c23f60f
...
...
@@ -92,16 +92,6 @@ namespace AMDiS {
}
void
BoundaryObject
::
setReverseMode
(
BoundaryObject
&
otherBound
,
FiniteElemSpace
*
feSpace
,
BoundaryType
boundary
)
{
FUNCNAME
(
"BoundaryObject::setReverseMode()"
);
otherBound
.
reverseMode
=
computeReverseMode
(
*
this
,
otherBound
,
feSpace
,
boundary
);
}
bool
BoundaryObject
::
operator
==
(
const
BoundaryObject
&
other
)
const
{
return
(
other
.
elIndex
==
elIndex
&&
...
...
AMDiS/src/parallel/InteriorBoundary.h
View file @
8c23f60f
...
...
@@ -59,10 +59,6 @@ namespace AMDiS {
FiniteElemSpace
*
feSpace
,
BoundaryType
boundary
);
void
setReverseMode
(
BoundaryObject
&
otherBound
,
FiniteElemSpace
*
feSpace
,
BoundaryType
boundary
);
bool
operator
==
(
const
BoundaryObject
&
other
)
const
;
bool
operator
!=
(
const
BoundaryObject
&
other
)
const
;
...
...
AMDiS/src/parallel/MeshDistributor.cc
View file @
8c23f60f
...
...
@@ -119,9 +119,9 @@ namespace AMDiS {
TEST_EXIT
(
feSpace
)(
"No FE space has been defined for the mesh distributor!
\n
"
);
TEST_EXIT
(
mesh
)(
"No mesh has been defined for the mesh distributor!
\n
"
);
int
a
;
char
*
b
;
float
zoltanVersion
;
int
a
=
0
;
char
*
b
=
NULL
;
float
zoltanVersion
=
0.0
;
Zoltan_Initialize
(
a
,
&
b
,
&
zoltanVersion
);
elObjects
.
setFeSpace
(
feSpace
);
...
...
@@ -165,12 +165,9 @@ namespace AMDiS {
setInitialElementWeights
();
// and now partition the mesh
oldPartitionMap
=
partitioner
->
getPartitionMap
();
bool
partitioningSucceed
=
partitioner
->
partition
(
elemWeights
,
INITIAL
);
TEST_EXIT
(
partitioningSucceed
)(
"Initial partitioning does not work!
\n
"
);
partitionMap
=
partitioner
->
getPartitionMap
();
partitioner
->
createPartitionMap
(
partitionMap
);
#if (DEBUG != 0)
...
...
@@ -742,6 +739,25 @@ namespace AMDiS {
repartitionMesh
();
nMeshChangesAfterLastRepartitioning
=
0
;
}
// === Print imbalance factor. ===
vector
<
int
>
nDofsInRank
(
mpiSize
);
int
nDofs
=
mesh
->
getDofAdmin
(
0
).
getUsedDofs
();
mpiComm
.
Gather
(
&
nDofs
,
1
,
MPI_INT
,
&
(
nDofsInRank
[
0
]),
1
,
MPI_INT
,
0
);
if
(
mpiRank
==
0
)
{
int
nOverallDofs
=
0
;
int
maxDofs
=
numeric_limits
<
int
>::
min
();
for
(
int
i
=
0
;
i
<
mpiSize
;
i
++
)
{
nOverallDofs
+=
nDofsInRank
[
i
];
maxDofs
=
std
::
max
(
maxDofs
,
nDofsInRank
[
i
]);
}
int
avrgDofs
=
nOverallDofs
/
mpiSize
;
double
imbalance
=
(
static_cast
<
double
>
(
maxDofs
-
avrgDofs
)
/
avrgDofs
)
*
100.0
;
MSG
(
"Imbalancing factor: %.1f\%
\n
"
,
imbalance
);
}
}
...
...
@@ -935,7 +951,6 @@ namespace AMDiS {
return
;
double
timePoint
=
MPI
::
Wtime
();
repartitioningCounter
++
;
#if (DEBUG != 0)
ParallelDebug
::
testDoubleDofs
(
mesh
);
...
...
@@ -944,6 +959,7 @@ namespace AMDiS {
ParallelDebug
::
writePartitioningFile
(
debugOutputDir
+
"partitioning"
,
repartitioningCounter
,
feSpace
);
#endif
repartitioningCounter
++
;
// === Create new element weights. ===
...
...
@@ -964,8 +980,6 @@ namespace AMDiS {
MSG
(
"Mesh partitioner created empty partition!
\n
"
);
return
;
}
oldPartitionMap
=
partitionMap
;
// === Create map that maps macro element indices to pointers to the ===
// === macro elements. ===
...
...
@@ -1051,6 +1065,9 @@ namespace AMDiS {
stdMpi
.
recv
(
it
->
first
);
stdMpi
.
startCommunication
();
if
(
interchangeVectors
.
size
()
==
0
)
WARNING
(
"There are no interchange vectors defined!
\n
"
);
StdMpi
<
vector
<
vector
<
double
>
>
>
stdMpi2
(
mpiComm
,
true
);
stdMpi2
.
send
(
sendValues
);
for
(
std
::
map
<
int
,
std
::
vector
<
int
>
>::
iterator
it
=
partitioner
->
getRecvElements
().
begin
();
...
...
@@ -1058,15 +1075,16 @@ namespace AMDiS {
stdMpi2
.
recv
(
it
->
first
);
stdMpi2
.
startCommunication
();
// === Adapte the new macro elements due to the received mesh structure codes. ===
for
(
map
<
int
,
vector
<
int
>
>::
iterator
it
=
partitioner
->
getRecvElements
().
begin
();
it
!=
partitioner
->
getRecvElements
().
end
();
++
it
)
{
MeshCodeVec
&
recvCodes
=
stdMpi
.
getRecvData
()[
it
->
first
];
vector
<
vector
<
double
>
>
&
recvValues
=
stdMpi2
.
getRecvData
()[
it
->
first
];
int
i
=
0
;
int
j
=
0
;
int
i
=
0
;
int
j
=
0
;
TEST_EXIT_DBG
(
recvCodes
.
size
()
==
it
->
second
.
size
())
(
"Should not happen!
\n
"
);
...
...
@@ -1132,17 +1150,15 @@ namespace AMDiS {
// update the number of elements, vertices, etc. of the mesh.
mesh
->
removeMacroElements
(
deleteMacroElements
,
feSpace
);
// === Remove double DOFs. ===
MeshManipulation
meshManipulation
(
feSpace
);
meshManipulation
.
deleteDoubleDofs
(
newMacroEl
,
elObjects
);
mesh
->
dofCompress
();
partition
Map
=
partitioner
->
getP
artitionMap
(
);
partition
er
->
createPartitionMap
(
p
artitionMap
);
updateInteriorBoundaryInfo
();
updateLocalGlobalNumbering
();
#if (DEBUG != 0)
...
...
@@ -1215,11 +1231,11 @@ namespace AMDiS {
}
elObjects
.
createReverseModeData
(
macroElIndexMap
,
macroElIndexTypeMap
);
// === Create periodic data, if there are periodic boundary conditions. ===
elObjects
.
createPeriodicData
();
// === Create data about the reverse modes of neighbouring elements. ===
elObjects
.
createReverseModeData
(
macroElIndexMap
,
macroElIndexTypeMap
);
// === Create mesh element data for this rank. ===
elObjects
.
createRankData
(
partitionMap
);
...
...
@@ -1283,7 +1299,6 @@ namespace AMDiS {
AtomicBoundary
&
b
=
myIntBoundary
.
getNewAtomic
(
it2
->
first
);
b
=
bound
;
// b.rankObj.setReverseMode(b.neighObj, feSpace, bound.type);
if
(
geoIndex
==
EDGE
)
b
.
neighObj
.
reverseMode
=
elObjects
.
getEdgeReverseMode
(
rankBoundEl
,
it2
->
second
);
if
(
geoIndex
==
FACE
)
...
...
@@ -1305,10 +1320,7 @@ namespace AMDiS {
bound
.
type
=
INTERIOR
;
AtomicBoundary
&
b
=
otherIntBoundary
.
getNewAtomic
(
owner
);
b
=
bound
;
// b.neighObj.setReverseMode(b.rankObj, feSpace, bound.type);
b
=
bound
;
if
(
geoIndex
==
EDGE
)
b
.
rankObj
.
reverseMode
=
elObjects
.
getEdgeReverseMode
(
rankBoundEl
,
ownerBoundEl
);
if
(
geoIndex
==
FACE
)
...
...
@@ -1391,9 +1403,9 @@ namespace AMDiS {
b
=
bound
;
if
(
mpiRank
>
otherElementRank
)
b
.
rankObj
.
setReverseMode
(
b
.
neighObj
,
feSpace
,
bound
.
type
);
b
.
neighObj
.
reverseMode
=
elObjects
.
getEdgeReverseMode
(
perEdgeEl0
,
perEdgeEl1
);
else
b
.
neighObj
.
setReverseMode
(
b
.
rankObj
,
feSpace
,
bound
.
type
);
b
.
rankObj
.
reverseMode
=
elObjects
.
getEdgeReverseMode
(
perEdgeEl0
,
perEdgeEl1
);
}
}
...
...
@@ -1408,26 +1420,26 @@ namespace AMDiS {
TEST_EXIT_DBG
(
elObjects
.
getElements
(
it
->
first
.
second
).
size
()
==
1
)
(
"Should not happen!
\n
"
);
ElementObjectData
&
per
Edg
eEl0
=
elObjects
.
getElementsInRank
(
it
->
first
.
first
)[
mpiRank
];
ElementObjectData
&
per
Fac
eEl0
=
elObjects
.
getElementsInRank
(
it
->
first
.
first
)[
mpiRank
];
for
(
map
<
int
,
ElementObjectData
>::
iterator
elIt
=
elObjects
.
getElementsInRank
(
it
->
first
.
second
).
begin
();
elIt
!=
elObjects
.
getElementsInRank
(
it
->
first
.
second
).
end
();
++
elIt
)
{
int
otherElementRank
=
elIt
->
first
;
ElementObjectData
&
per
Edg
eEl1
=
elIt
->
second
;
ElementObjectData
&
per
Fac
eEl1
=
elIt
->
second
;
AtomicBoundary
bound
;
bound
.
rankObj
.
el
=
macroElIndexMap
[
per
Edg
eEl0
.
elIndex
];
bound
.
rankObj
.
elIndex
=
per
Edg
eEl0
.
elIndex
;
bound
.
rankObj
.
elType
=
macroElIndexTypeMap
[
per
Edg
eEl0
.
elIndex
];
bound
.
rankObj
.
el
=
macroElIndexMap
[
per
Fac
eEl0
.
elIndex
];
bound
.
rankObj
.
elIndex
=
per
Fac
eEl0
.
elIndex
;
bound
.
rankObj
.
elType
=
macroElIndexTypeMap
[
per
Fac
eEl0
.
elIndex
];
bound
.
rankObj
.
subObj
=
FACE
;
bound
.
rankObj
.
ithObj
=
per
Edg
eEl0
.
ithObject
;
bound
.
rankObj
.
ithObj
=
per
Fac
eEl0
.
ithObject
;
bound
.
neighObj
.
el
=
macroElIndexMap
[
per
Edg
eEl1
.
elIndex
];
bound
.
neighObj
.
elIndex
=
per
Edg
eEl1
.
elIndex
;
bound
.
neighObj
.
elType
=
macroElIndexTypeMap
[
per
Edg
eEl1
.
elIndex
];
bound
.
neighObj
.
el
=
macroElIndexMap
[
per
Fac
eEl1
.
elIndex
];
bound
.
neighObj
.
elIndex
=
per
Fac
eEl1
.
elIndex
;
bound
.
neighObj
.
elType
=
macroElIndexTypeMap
[
per
Fac
eEl1
.
elIndex
];
bound
.
neighObj
.
subObj
=
FACE
;
bound
.
neighObj
.
ithObj
=
per
Edg
eEl1
.
ithObject
;
bound
.
neighObj
.
ithObj
=
per
Fac
eEl1
.
ithObject
;
bound
.
type
=
it
->
second
;
...
...
@@ -1435,9 +1447,9 @@ namespace AMDiS {
b
=
bound
;
if
(
mpiRank
>
otherElementRank
)
b
.
rankObj
.
setReverseMode
(
b
.
neighObj
,
feSpace
,
bound
.
type
);
b
.
neighObj
.
reverseMode
=
elObjects
.
getFaceReverseMode
(
perFaceEl0
,
perFaceEl1
);
else
b
.
neighObj
.
setReverseMode
(
b
.
rankObj
,
feSpace
,
bound
.
type
);
b
.
rankObj
.
reverseMode
=
elObjects
.
getFaceReverseMode
(
perFaceEl0
,
perFaceEl1
);
}
}
...
...
@@ -1952,7 +1964,6 @@ namespace AMDiS {
SerUtil
::
serialize
(
out
,
elemWeights
);
SerUtil
::
serialize
(
out
,
partitionMap
);
SerUtil
::
serialize
(
out
,
oldPartitionMap
);
SerUtil
::
serialize
(
out
,
nRankDofs
);
SerUtil
::
serialize
(
out
,
nOverallDofs
);
...
...
@@ -1993,7 +2004,6 @@ namespace AMDiS {
SerUtil
::
deserialize
(
in
,
elemWeights
);
SerUtil
::
deserialize
(
in
,
partitionMap
);
SerUtil
::
deserialize
(
in
,
oldPartitionMap
);
SerUtil
::
deserialize
(
in
,
nRankDofs
);
SerUtil
::
deserialize
(
in
,
nOverallDofs
);
...
...
AMDiS/src/parallel/MeshDistributor.h
View file @
8c23f60f
...
...
@@ -31,6 +31,7 @@
#include "parallel/MeshPartitioner.h"
#include "parallel/InteriorBoundary.h"
#include "AMDiS_fwd.h"
#include "Global.h"
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
...
...
@@ -38,7 +39,7 @@
#include "Serializer.h"
#include "BoundaryManager.h"
#include "ElementObjectData.h"
#include "
AMDiS_fwd
.h"
#include "
SystemVector
.h"
namespace
AMDiS
{
...
...
@@ -94,6 +95,13 @@ namespace AMDiS {
interchangeVectors
.
push_back
(
vec
);
}
/// Adds all DOFVectors of a SystemVector to \ref interchangeVecs.
void
addInterchangeVector
(
SystemVector
*
vec
)
{
for
(
int
i
=
0
;
i
<
vec
->
getSize
();
i
++
)
interchangeVectors
.
push_back
(
vec
->
getDOFVector
(
i
));
}
/** \brief
* This function checks if the mesh has changed on at least on rank. In this case,
* the interior boundaries are adapted on all ranks such that they fit together on
...
...
@@ -448,12 +456,6 @@ namespace AMDiS {
*/
map
<
int
,
int
>
partitionMap
;
/** \brief
* Stores an old partitioning of elements. To every macro element index the
* number of the rank it corresponds to is stored.
*/
map
<
int
,
int
>
oldPartitionMap
;
/// Number of DOFs in the rank mesh.
int
nRankDofs
;
...
...
AMDiS/src/parallel/MeshPartitioner.h
View file @
8c23f60f
...
...
@@ -58,6 +58,8 @@ namespace AMDiS {
virtual
bool
partition
(
map
<
int
,
double
>
&
elemWeights
,
PartitionMode
mode
=
INITIAL
)
=
0
;
virtual
void
createPartitionMap
(
map
<
int
,
int
>&
partitionMap
)
=
0
;
/// Write partitioner state to disk.
void
serialize
(
ostream
&
out
);
...
...
@@ -84,11 +86,6 @@ namespace AMDiS {
return
elementInRank
;
}
map
<
int
,
int
>&
getPartitionMap
()
{
return
partitionMap
;
}
map
<
int
,
vector
<
int
>
>&
getRecvElements
()
{
return
recvElements
;
...
...
AMDiS/src/parallel/ParMetisPartitioner.cc
View file @
8c23f60f
...
...
@@ -421,16 +421,11 @@ namespace AMDiS {
// === Distribute new partition data. ===
bool
b
=
distributePartitioning
(
&
(
part
[
0
]));
if
(
b
)
createPartitionMap
();
return
b
;
return
distributePartitioning
(
&
(
part
[
0
]));
}
void
ParMetisPartitioner
::
createPartitionMap
()
void
ParMetisPartitioner
::
createPartitionMap
(
map
<
int
,
int
>&
pMap
)
{
FUNCNAME
(
"ParMetisPartitioner::createPartitionMap()"
);
...
...
@@ -468,6 +463,8 @@ namespace AMDiS {
for
(
int
i
=
0
;
i
<
mpiSize
;
i
++
)
for
(
int
j
=
0
;
j
<
nPartitionElements
[
i
];
j
++
)
partitionMap
[
partitionElements
[
elmdist
[
i
]
+
j
]]
=
i
;
pMap
=
partitionMap
;
}
...
...
AMDiS/src/parallel/ParMetisPartitioner.h
View file @
8c23f60f
...
...
@@ -179,6 +179,8 @@ namespace AMDiS {
bool
partition
(
map
<
int
,
double
>
&
elemWeights
,
PartitionMode
mode
=
INITIAL
);
void
createPartitionMap
(
map
<
int
,
int
>&
partitionMap
);
void
setItr
(
float
value
)
{
itr
=
value
;
...
...
@@ -189,8 +191,6 @@ namespace AMDiS {
// case, i.e., because there are empty partitions, the function returns false.
bool
distributePartitioning
(
int
*
part
);
void
createPartitionMap
();
protected:
ParMetisMesh
*
parMetisMesh
;
...
...
AMDiS/src/parallel/ZoltanPartitioner.cc
View file @
8c23f60f
...
...
@@ -54,9 +54,13 @@ namespace AMDiS {
int
*
export_procs
;
int
*
export_to_part
;
zoltan
.
Set_Param
(
"LB_METHOD"
,
"RCB"
);
// zoltan.Set_Param("LB_METHOD", "GRAPH");
// zoltan.Set_Param("LB_APPROACH", "PARTITION");
if
(
mode
==
INITIAL
)
{
zoltan
.
Set_Param
(
"LB_APPROACH"
,
"PARTITION"
);
}
else
{
zoltan
.
Set_Param
(
"LB_APPROACH"
,
"REPARTITION"
);
zoltan
.
Set_Param
(
"LB_METHOD"
,
"GRAPH"
);
}
zoltan
.
Set_Param
(
"OBJ_WEIGHT_DIM"
,
"1"
);
#if (DEBUG != 0)
...
...
AMDiS/src/parallel/ZoltanPartitioner.h
View file @
8c23f60f
...
...
@@ -42,6 +42,11 @@ namespace AMDiS {
/// \ref MeshPartitioner::partition
bool
partition
(
map
<
int
,
double
>
&
weights
,
PartitionMode
mode
=
INITIAL
);
void
createPartitionMap
(
map
<
int
,
int
>
&
pMap
)
{
pMap
=
partitionMap
;
}
protected:
void
createPartitionMap
();
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment