Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Backofen, Rainer
amdis
Commits
b97f84ba
Commit
b97f84ba
authored
Jan 28, 2011
by
Thomas Witkowski
Browse files
Added box partitioning.
parent
8c23f60f
Changes
5
Show whitespace changes
Inline
Side-by-side
AMDiS/src/parallel/MeshDistributor.cc
View file @
b97f84ba
...
...
@@ -105,6 +105,10 @@ namespace AMDiS {
if
(
partStr
==
"zoltan"
)
partitioner
=
new
ZoltanPartitioner
(
&
mpiComm
);
tmp
=
0
;
GET_PARAMETER
(
0
,
name
+
"->box partitioning"
,
"%d"
,
&
tmp
);
partitioner
->
setBoxPartitioning
(
static_cast
<
bool
>
(
tmp
));
TEST_EXIT
(
partitioner
)(
"Could not create partitioner
\"
%s
\"
!
\n
"
,
partStr
.
c_str
());
}
...
...
AMDiS/src/parallel/MeshPartitioner.cc
View file @
b97f84ba
...
...
@@ -29,40 +29,99 @@ namespace AMDiS {
// === Create initial partitioning of the AMDiS mesh. ===
elementInRank
.
clear
();
map
<
DofEdge
,
set
<
int
>
>
vertexElements
;
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh
,
0
,
Mesh
::
CALL_EL_LEVEL
|
Mesh
::
FILL_NEIGH
|
Mesh
::
FILL_BOUND
);
Mesh
::
CALL_EL_LEVEL
|
Mesh
::
FILL_NEIGH
|
Mesh
::
FILL_COORDS
|
Mesh
::
FILL_BOUND
);
while
(
elInfo
)
{
int
elIndex
=
elInfo
->
getElement
()
->
getIndex
();
int
elInRank
=
std
::
min
(
elIndex
/
elPerRank
,
mpiSize
-
1
);
if
(
elInRank
==
mpiRank
)
elementInRank
[
elIndex
]
=
true
;
else
elementInRank
[
elIndex
]
=
false
;
partitionMap
[
elIndex
]
=
elInRank
;
Element
*
el
=
elInfo
->
getElement
();
int
elIndex
=
el
->
getIndex
();
for
(
int
i
=
0
;
i
<
mesh
->
getGeo
(
NEIGH
);
i
++
)
if
(
elInfo
->
getNeighbour
(
i
)
&&
elInfo
->
getBoundary
(
i
)
==
INTERIOR
)
elNeighbours
[
elIndex
].
push_back
(
elInfo
->
getNeighbour
(
i
)
->
getIndex
());
if
(
boxPartitioning
)
{
vertexElements
[
el
->
getEdge
(
0
)].
insert
(
elIndex
);
}
else
{
int
elInRank
=
std
::
min
(
elIndex
/
elPerRank
,
mpiSize
-
1
);
elementInRank
[
elIndex
]
=
(
elInRank
==
mpiRank
);
partitionMap
[
elIndex
]
=
elInRank
;
}
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
if
(
boxPartitioning
)
{
TEST_EXIT
(
mesh
->
getDim
()
==
3
)(
"Box partitioning only implemented for 3D!
\n
"
);
int
boxCounter
=
0
;
for
(
map
<
DofEdge
,
set
<
int
>
>::
iterator
it
=
vertexElements
.
begin
();
it
!=
vertexElements
.
end
();
++
it
)
{
TEST_EXIT_DBG
(
it
->
second
.
size
()
==
6
)(
"Should not happen!
\n
"
);
boxSplitting
[
boxCounter
]
=
it
->
second
;
for
(
set
<
int
>::
iterator
elIt
=
it
->
second
.
begin
();
elIt
!=
it
->
second
.
end
();
++
elIt
)
elInBox
[
*
elIt
]
=
boxCounter
;
boxCounter
++
;
}
for
(
map
<
int
,
int
>::
iterator
it
=
elInBox
.
begin
();
it
!=
elInBox
.
end
();
++
it
)
{
int
elBoxNo
=
it
->
second
;
for
(
vector
<
int
>::
iterator
neighIt
=
elNeighbours
[
it
->
first
].
begin
();
neighIt
!=
elNeighbours
[
it
->
first
].
end
();
++
neighIt
)
{
int
neighBoxNo
=
elInBox
[
*
neighIt
];
if
(
elBoxNo
!=
neighBoxNo
)
boxNeighbours
[
elBoxNo
].
insert
(
neighBoxNo
);
}
}
MSG
(
"Box partitioning with %d boxes enabled!
\n
"
,
boxCounter
);
int
boxPerRank
=
boxCounter
/
mpiSize
;
for
(
map
<
int
,
set
<
int
>
>::
iterator
it
=
boxSplitting
.
begin
();
it
!=
boxSplitting
.
end
();
++
it
)
{
int
boxInRank
=
std
::
min
(
it
->
first
/
boxPerRank
,
mpiSize
-
1
);
for
(
set
<
int
>::
iterator
elIt
=
it
->
second
.
begin
();
elIt
!=
it
->
second
.
end
();
++
elIt
)
{
elementInRank
[
*
elIt
]
=
(
boxInRank
==
mpiRank
);
partitionMap
[
*
elIt
]
=
boxInRank
;
}
}
}
}
void
MeshPartitioner
::
serialize
(
std
::
ostream
&
out
)
{
SerUtil
::
serialize
(
out
,
elementInRank
);
SerUtil
::
serialize
(
out
,
boxPartitioning
);
SerUtil
::
serialize
(
out
,
boxSplitting
);
SerUtil
::
serialize
(
out
,
boxNeighbours
);
SerUtil
::
serialize
(
out
,
elInBox
);
}
void
MeshPartitioner
::
deserialize
(
std
::
istream
&
in
)
{
SerUtil
::
deserialize
(
in
,
elementInRank
);
SerUtil
::
deserialize
(
in
,
boxPartitioning
);
SerUtil
::
deserialize
(
in
,
boxSplitting
);
SerUtil
::
deserialize
(
in
,
boxNeighbours
);
SerUtil
::
deserialize
(
in
,
elInBox
);
}
}
AMDiS/src/parallel/MeshPartitioner.h
View file @
b97f84ba
...
...
@@ -24,6 +24,7 @@
#define AMDIS_MESH_PARTITIONER_H
#include
<map>
#include
<set>
#include
"AMDiS_fwd.h"
#include
"Mesh.h"
...
...
@@ -47,6 +48,7 @@ namespace AMDiS {
MeshPartitioner
(
MPI
::
Intracomm
*
comm
)
:
mpiComm
(
comm
),
mesh
(
NULL
),
boxPartitioning
(
false
),
mapLocalGlobal
(
NULL
)
{}
...
...
@@ -71,6 +73,11 @@ namespace AMDiS {
mesh
=
m
;
}
void
setBoxPartitioning
(
bool
b
)
{
boxPartitioning
=
b
;
}
void
setLocalGlobalDofMap
(
map
<
DegreeOfFreedom
,
DegreeOfFreedom
>
*
m
)
{
mapLocalGlobal
=
m
;
...
...
@@ -101,6 +108,14 @@ namespace AMDiS {
Mesh
*
mesh
;
bool
boxPartitioning
;
map
<
int
,
std
::
set
<
int
>
>
boxSplitting
;
map
<
int
,
std
::
set
<
int
>
>
boxNeighbours
;
map
<
int
,
int
>
elInBox
;
map
<
DegreeOfFreedom
,
DegreeOfFreedom
>
*
mapLocalGlobal
;
map
<
int
,
vector
<
int
>
>
elNeighbours
;
...
...
AMDiS/src/parallel/ZoltanPartitioner.cc
View file @
b97f84ba
...
...
@@ -21,21 +21,30 @@ namespace AMDiS {
:
MeshPartitioner
(
comm
),
zoltan
(
*
comm
),
elWeights
(
NULL
)
{}
bool
ZoltanPartitioner
::
partition
(
map
<
int
,
double
>
&
weights
,
PartitionMode
mode
)
{
FUNCNAME
(
"ZoltanPartitioner::partition()"
);
if
(
!
boxPartitioning
)
{
zoltan
.
Set_Num_Obj_Fn
(
ZoltanFunctions
::
getNumObj
,
this
);
zoltan
.
Set_Obj_List_Fn
(
ZoltanFunctions
::
getObjectList
,
this
);
zoltan
.
Set_Num_Geom_Fn
(
ZoltanFunctions
::
getNumGeom
,
this
);
zoltan
.
Set_Geom_Multi_Fn
(
ZoltanFunctions
::
getGeomMulti
,
this
);
zoltan
.
Set_Num_Edges_Multi_Fn
(
ZoltanFunctions
::
getNumEdgeMulti
,
this
);
zoltan
.
Set_Edge_List_Multi_Fn
(
ZoltanFunctions
::
getEdgeListMulti
,
this
);
}
else
{
zoltan
.
Set_Num_Obj_Fn
(
ZoltanFunctions
::
box_getNumObj
,
this
);
zoltan
.
Set_Obj_List_Fn
(
ZoltanFunctions
::
box_getObjectList
,
this
);
zoltan
.
Set_Num_Geom_Fn
(
ZoltanFunctions
::
box_getNumGeom
,
this
);
zoltan
.
Set_Geom_Multi_Fn
(
ZoltanFunctions
::
box_getGeomMulti
,
this
);
zoltan
.
Set_Num_Edges_Multi_Fn
(
ZoltanFunctions
::
box_getNumEdgeMulti
,
this
);
zoltan
.
Set_Edge_List_Multi_Fn
(
ZoltanFunctions
::
box_getEdgeListMulti
,
this
);
}
bool
ZoltanPartitioner
::
partition
(
map
<
int
,
double
>
&
weights
,
PartitionMode
mode
)
{
FUNCNAME
(
"ZoltanPartitioner::partition()"
);
if
(
mode
!=
INITIAL
)
elWeights
=
&
weights
;
...
...
@@ -56,6 +65,8 @@ namespace AMDiS {
if
(
mode
==
INITIAL
)
{
zoltan
.
Set_Param
(
"LB_APPROACH"
,
"PARTITION"
);
if
(
boxPartitioning
)
zoltan
.
Set_Param
(
"LB_METHOD"
,
"GRAPH"
);
}
else
{
zoltan
.
Set_Param
(
"LB_APPROACH"
,
"REPARTITION"
);
zoltan
.
Set_Param
(
"LB_METHOD"
,
"GRAPH"
);
...
...
@@ -89,8 +100,17 @@ namespace AMDiS {
for
(
int
i
=
0
;
i
<
nImportEls
;
i
++
)
{
int
recvElIndex
=
import_global_ids
[
i
];
if
(
!
boxPartitioning
)
{
elementInRank
[
recvElIndex
]
=
true
;
recvElements
[
import_procs
[
i
]].
push_back
(
recvElIndex
);
}
else
{
set
<
int
>&
box
=
boxSplitting
[
recvElIndex
];
for
(
set
<
int
>::
iterator
it
=
box
.
begin
();
it
!=
box
.
end
();
++
it
)
{
elementInRank
[
*
it
]
=
true
;
recvElements
[
import_procs
[
i
]].
push_back
(
*
it
);
}
}
}
}
...
...
@@ -98,8 +118,17 @@ namespace AMDiS {
for
(
int
i
=
0
;
i
<
nExportEls
;
i
++
)
{
int
sendElIndex
=
export_global_ids
[
i
];
if
(
!
boxPartitioning
)
{
elementInRank
[
sendElIndex
]
=
false
;
sendElements
[
export_procs
[
i
]].
push_back
(
sendElIndex
);
}
else
{
set
<
int
>&
box
=
boxSplitting
[
sendElIndex
];
for
(
set
<
int
>::
iterator
it
=
box
.
begin
();
it
!=
box
.
end
();
++
it
)
{
elementInRank
[
*
it
]
=
false
;
sendElements
[
export_procs
[
i
]].
push_back
(
*
it
);
}
}
}
}
...
...
@@ -169,6 +198,9 @@ namespace AMDiS {
// === Zoltan query function for "normal" element partitioning. ===
int
ZoltanFunctions
::
getNumObj
(
void
*
data
,
int
*
ierr
)
{
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
...
...
@@ -340,4 +372,166 @@ namespace AMDiS {
*
ierr
=
ZOLTAN_OK
;
}
// === Zoltan query functions for box partitioning. ===
int
ZoltanFunctions
::
box_getNumObj
(
void
*
data
,
int
*
ierr
)
{
FUNCNAME
(
"ZoltanFunctions::box_getNumObj()"
);
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
map
<
int
,
bool
>&
elInRank
=
zoltan
->
getElementInRank
();
int
nObjects
=
0
;
for
(
map
<
int
,
bool
>::
iterator
it
=
elInRank
.
begin
();
it
!=
elInRank
.
end
();
++
it
)
if
(
it
->
second
)
nObjects
++
;
TEST_EXIT_DBG
(
nObjects
%
6
==
0
)(
"Should not happen!
\n
"
);
nObjects
=
nObjects
/
6
;
*
ierr
=
ZOLTAN_OK
;
return
nObjects
;
}
void
ZoltanFunctions
::
box_getObjectList
(
void
*
data
,
int
sizeGid
,
int
sizeLid
,
ZOLTAN_ID_PTR
globalId
,
ZOLTAN_ID_PTR
localId
,
int
wgt_dim
,
float
*
obj_wgts
,
int
*
ierr
)
{
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
map
<
int
,
bool
>&
elInRank
=
zoltan
->
getElementInRank
();
set
<
int
>
boxes
;
map
<
int
,
double
>
boxWeight
;
for
(
map
<
int
,
bool
>::
iterator
it
=
elInRank
.
begin
();
it
!=
elInRank
.
end
();
++
it
)
if
(
it
->
second
)
{
int
boxId
=
zoltan
->
elInBox
[
it
->
first
];
boxes
.
insert
(
boxId
);
if
(
zoltan
->
elWeights
)
{
TEST_EXIT_DBG
(
zoltan
->
elWeights
->
count
(
it
->
first
))
(
"No element weight for element index %d
\n
"
,
it
->
first
);
boxWeight
[
boxId
]
+=
(
*
zoltan
->
elWeights
)[
it
->
first
];
}
}
int
localCounter
=
0
;
for
(
set
<
int
>::
iterator
it
=
boxes
.
begin
();
it
!=
boxes
.
end
();
++
it
)
{
globalId
[
localCounter
]
=
*
it
;
localId
[
localCounter
]
=
localCounter
;
if
(
wgt_dim
>
0
)
{
TEST_EXIT_DBG
(
wgt_dim
==
1
)(
"Should not happen!
\n
"
);
if
(
zoltan
->
elWeights
)
obj_wgts
[
localCounter
]
=
static_cast
<
float
>
(
boxWeight
[
*
it
]);
else
obj_wgts
[
localCounter
]
=
1.0
;
}
localCounter
++
;
}
*
ierr
=
ZOLTAN_OK
;
}
int
ZoltanFunctions
::
box_getNumGeom
(
void
*
data
,
int
*
ierr
)
{
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
*
ierr
=
ZOLTAN_OK
;
return
zoltan
->
getMesh
()
->
getGeo
(
WORLD
);
}
void
ZoltanFunctions
::
box_getGeomMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
num_dim
,
double
*
geom_vec
,
int
*
ierr
)
{
FUNCNAME
(
"ZoltanFunctions::box_getGeomMulti()"
);
ERROR_EXIT
(
"Not yet supported!
\n
"
);
}
void
ZoltanFunctions
::
box_getNumEdgeMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
*
num_edges
,
int
*
ierr
)
{
FUNCNAME
(
"ZoltanFunctions::box_getNumEdgeMulti()"
);
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
for
(
int
i
=
0
;
i
<
num_obj
;
i
++
)
{
TEST_EXIT_DBG
(
zoltan
->
boxNeighbours
.
count
(
global_ids
[
i
]))(
"Should not happen!
\n
"
);
num_edges
[
i
]
=
zoltan
->
boxNeighbours
[
global_ids
[
i
]].
size
();
}
*
ierr
=
ZOLTAN_OK
;
}
void
ZoltanFunctions
::
box_getEdgeListMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
*
num_edges
,
ZOLTAN_ID_PTR
nbor_global_id
,
int
*
nbor_procs
,
int
wgt_dim
,
float
*
ewgts
,
int
*
ierr
)
{
FUNCNAME
(
"ZoltanFunctions::box_getEdgeListMulti()"
);
ZoltanPartitioner
*
zoltan
=
(
ZoltanPartitioner
*
)
data
;
int
c
=
0
;
for
(
int
i
=
0
;
i
<
num_obj
;
i
++
)
{
TEST_EXIT_DBG
(
static_cast
<
int
>
(
zoltan
->
boxNeighbours
[
global_ids
[
i
]].
size
())
==
num_edges
[
i
])
(
"Wrong sizes for global el index %d: %d %d
\n
"
,
global_ids
[
i
],
zoltan
->
boxNeighbours
[
global_ids
[
i
]].
size
(),
num_edges
[
i
]);
set
<
int
>
&
neigh
=
zoltan
->
boxNeighbours
[
global_ids
[
i
]];
for
(
set
<
int
>::
iterator
it
=
neigh
.
begin
();
it
!=
neigh
.
end
();
++
it
)
{
nbor_global_id
[
c
]
=
*
it
;
nbor_procs
[
c
]
=
zoltan
->
partitionMap
[
*
(
zoltan
->
boxSplitting
[
*
it
].
begin
())];
c
++
;
}
}
TEST_EXIT_DBG
(
wgt_dim
==
0
)(
"Edge weights are not handled yet!
\n
"
);
*
ierr
=
ZOLTAN_OK
;
}
}
AMDiS/src/parallel/ZoltanPartitioner.h
View file @
b97f84ba
...
...
@@ -64,6 +64,8 @@ namespace AMDiS {
class
ZoltanFunctions
{
public:
// === Zoltan query function for "normal" element partitioning. ===
static
int
getNumObj
(
void
*
data
,
int
*
ierr
);
static
void
getObjectList
(
void
*
data
,
...
...
@@ -108,6 +110,55 @@ namespace AMDiS {
int
wgt_dim
,
float
*
ewgts
,
int
*
ierr
);
// === Zoltan query functions for box partitioning. ===
static
int
box_getNumObj
(
void
*
data
,
int
*
ierr
);
static
void
box_getObjectList
(
void
*
data
,
int
sizeGid
,
int
sizeLid
,
ZOLTAN_ID_PTR
globalId
,
ZOLTAN_ID_PTR
localId
,
int
wgt_dim
,
float
*
obj_wgts
,
int
*
ierr
);
static
int
box_getNumGeom
(
void
*
data
,
int
*
ierr
);
static
void
box_getGeomMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
num_dim
,
double
*
geom_vec
,
int
*
ierr
);
static
void
box_getNumEdgeMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
*
num_edges
,
int
*
ierr
);
static
void
box_getEdgeListMulti
(
void
*
data
,
int
num_gid_entries
,
int
num_lid_entries
,
int
num_obj
,
ZOLTAN_ID_PTR
global_ids
,
ZOLTAN_ID_PTR
local_ids
,
int
*
num_edges
,
ZOLTAN_ID_PTR
nbor_global_id
,
int
*
nbor_procs
,
int
wgt_dim
,
float
*
ewgts
,
int
*
ierr
);
};
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment