Commit b97f84ba authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Added box partitioning.

parent 8c23f60f
......@@ -105,6 +105,10 @@ namespace AMDiS {
if (partStr == "zoltan")
partitioner = new ZoltanPartitioner(&mpiComm);
tmp = 0;
GET_PARAMETER(0, name + "->box partitioning", "%d", &tmp);
partitioner->setBoxPartitioning(static_cast<bool>(tmp));
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
}
......
......@@ -29,40 +29,99 @@ namespace AMDiS {
// === Create initial partitioning of the AMDiS mesh. ===
elementInRank.clear();
map<DofEdge, set<int> > vertexElements;
TraverseStack stack;
ElInfo *elInfo =
stack.traverseFirst(mesh, 0,
Mesh::CALL_EL_LEVEL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
Mesh::CALL_EL_LEVEL | Mesh::FILL_NEIGH | Mesh::FILL_COORDS | Mesh::FILL_BOUND);
while (elInfo) {
int elIndex = elInfo->getElement()->getIndex();
int elInRank = std::min(elIndex / elPerRank, mpiSize - 1);
if (elInRank == mpiRank)
elementInRank[elIndex] = true;
else
elementInRank[elIndex] = false;
partitionMap[elIndex] = elInRank;
Element *el = elInfo->getElement();
int elIndex = el->getIndex();
for (int i = 0; i < mesh->getGeo(NEIGH); i++)
if (elInfo->getNeighbour(i) && elInfo->getBoundary(i) == INTERIOR)
elNeighbours[elIndex].push_back(elInfo->getNeighbour(i)->getIndex());
if (boxPartitioning) {
vertexElements[el->getEdge(0)].insert(elIndex);
} else {
int elInRank = std::min(elIndex / elPerRank, mpiSize - 1);
elementInRank[elIndex] = (elInRank == mpiRank);
partitionMap[elIndex] = elInRank;
}
elInfo = stack.traverseNext(elInfo);
}
if (boxPartitioning) {
TEST_EXIT(mesh->getDim() == 3)("Box partitioning only implemented for 3D!\n");
int boxCounter = 0;
for (map<DofEdge, set<int> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
TEST_EXIT_DBG(it->second.size() == 6)("Should not happen!\n");
boxSplitting[boxCounter] = it->second;
for (set<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt)
elInBox[*elIt] = boxCounter;
boxCounter++;
}
for (map<int, int>::iterator it = elInBox.begin(); it != elInBox.end(); ++it) {
int elBoxNo = it->second;
for (vector<int>::iterator neighIt = elNeighbours[it->first].begin();
neighIt != elNeighbours[it->first].end(); ++neighIt) {
int neighBoxNo = elInBox[*neighIt];
if (elBoxNo != neighBoxNo)
boxNeighbours[elBoxNo].insert(neighBoxNo);
}
}
MSG("Box partitioning with %d boxes enabled!\n", boxCounter);
int boxPerRank = boxCounter / mpiSize;
for (map<int, set<int> >::iterator it = boxSplitting.begin();
it != boxSplitting.end(); ++it) {
int boxInRank = std::min(it->first / boxPerRank, mpiSize - 1);
for (set<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt) {
elementInRank[*elIt] = (boxInRank == mpiRank);
partitionMap[*elIt] = boxInRank;
}
}
}
}
void MeshPartitioner::serialize(std::ostream &out)
{
SerUtil::serialize(out, elementInRank);
SerUtil::serialize(out, boxPartitioning);
SerUtil::serialize(out, boxSplitting);
SerUtil::serialize(out, boxNeighbours);
SerUtil::serialize(out, elInBox);
}
void MeshPartitioner::deserialize(std::istream &in)
{
SerUtil::deserialize(in, elementInRank);
SerUtil::deserialize(in, boxPartitioning);
SerUtil::deserialize(in, boxSplitting);
SerUtil::deserialize(in, boxNeighbours);
SerUtil::deserialize(in, elInBox);
}
}
......@@ -24,6 +24,7 @@
#define AMDIS_MESH_PARTITIONER_H
#include <map>
#include <set>
#include "AMDiS_fwd.h"
#include "Mesh.h"
......@@ -47,6 +48,7 @@ namespace AMDiS {
MeshPartitioner(MPI::Intracomm *comm)
: mpiComm(comm),
mesh(NULL),
boxPartitioning(false),
mapLocalGlobal(NULL)
{}
......@@ -71,6 +73,11 @@ namespace AMDiS {
mesh = m;
}
void setBoxPartitioning(bool b)
{
boxPartitioning = b;
}
void setLocalGlobalDofMap(map<DegreeOfFreedom, DegreeOfFreedom> *m)
{
mapLocalGlobal = m;
......@@ -101,6 +108,14 @@ namespace AMDiS {
Mesh *mesh;
bool boxPartitioning;
map<int, std::set<int> > boxSplitting;
map<int, std::set<int> > boxNeighbours;
map<int, int> elInBox;
map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal;
map<int, vector<int> > elNeighbours;
......
......@@ -21,21 +21,30 @@ namespace AMDiS {
: MeshPartitioner(comm),
zoltan(*comm),
elWeights(NULL)
{}
bool ZoltanPartitioner::partition(map<int, double> &weights,
PartitionMode mode)
{
FUNCNAME("ZoltanPartitioner::partition()");
if (!boxPartitioning) {
zoltan.Set_Num_Obj_Fn(ZoltanFunctions::getNumObj, this);
zoltan.Set_Obj_List_Fn(ZoltanFunctions::getObjectList, this);
zoltan.Set_Num_Geom_Fn(ZoltanFunctions::getNumGeom, this);
zoltan.Set_Geom_Multi_Fn(ZoltanFunctions::getGeomMulti, this);
zoltan.Set_Num_Edges_Multi_Fn(ZoltanFunctions::getNumEdgeMulti, this);
zoltan.Set_Edge_List_Multi_Fn(ZoltanFunctions::getEdgeListMulti, this);
} else {
zoltan.Set_Num_Obj_Fn(ZoltanFunctions::box_getNumObj, this);
zoltan.Set_Obj_List_Fn(ZoltanFunctions::box_getObjectList, this);
zoltan.Set_Num_Geom_Fn(ZoltanFunctions::box_getNumGeom, this);
zoltan.Set_Geom_Multi_Fn(ZoltanFunctions::box_getGeomMulti, this);
zoltan.Set_Num_Edges_Multi_Fn(ZoltanFunctions::box_getNumEdgeMulti, this);
zoltan.Set_Edge_List_Multi_Fn(ZoltanFunctions::box_getEdgeListMulti, this);
}
bool ZoltanPartitioner::partition(map<int, double> &weights,
PartitionMode mode)
{
FUNCNAME("ZoltanPartitioner::partition()");
if (mode != INITIAL)
elWeights = &weights;
......@@ -56,6 +65,8 @@ namespace AMDiS {
if (mode == INITIAL) {
zoltan.Set_Param("LB_APPROACH", "PARTITION");
if (boxPartitioning)
zoltan.Set_Param("LB_METHOD", "GRAPH");
} else {
zoltan.Set_Param("LB_APPROACH", "REPARTITION");
zoltan.Set_Param("LB_METHOD", "GRAPH");
......@@ -89,8 +100,17 @@ namespace AMDiS {
for (int i = 0; i < nImportEls; i++) {
int recvElIndex = import_global_ids[i];
if (!boxPartitioning) {
elementInRank[recvElIndex] = true;
recvElements[import_procs[i]].push_back(recvElIndex);
} else {
set<int>& box = boxSplitting[recvElIndex];
for (set<int>::iterator it = box.begin(); it != box.end(); ++it) {
elementInRank[*it] = true;
recvElements[import_procs[i]].push_back(*it);
}
}
}
}
......@@ -98,8 +118,17 @@ namespace AMDiS {
for (int i = 0; i < nExportEls; i++) {
int sendElIndex = export_global_ids[i];
if (!boxPartitioning) {
elementInRank[sendElIndex] = false;
sendElements[export_procs[i]].push_back(sendElIndex);
} else {
set<int>& box = boxSplitting[sendElIndex];
for (set<int>::iterator it = box.begin(); it != box.end(); ++it) {
elementInRank[*it] = false;
sendElements[export_procs[i]].push_back(*it);
}
}
}
}
......@@ -169,6 +198,9 @@ namespace AMDiS {
// === Zoltan query function for "normal" element partitioning. ===
int ZoltanFunctions::getNumObj(void *data, int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
......@@ -340,4 +372,166 @@ namespace AMDiS {
*ierr = ZOLTAN_OK;
}
// === Zoltan query functions for box partitioning. ===
int ZoltanFunctions::box_getNumObj(void *data, int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getNumObj()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
map<int, bool>& elInRank = zoltan->getElementInRank();
int nObjects = 0;
for (map<int, bool>::iterator it = elInRank.begin(); it != elInRank.end(); ++it)
if (it->second)
nObjects++;
TEST_EXIT_DBG(nObjects % 6 == 0)("Should not happen!\n");
nObjects = nObjects / 6;
*ierr = ZOLTAN_OK;
return nObjects;
}
void ZoltanFunctions::box_getObjectList(void *data,
int sizeGid,
int sizeLid,
ZOLTAN_ID_PTR globalId,
ZOLTAN_ID_PTR localId,
int wgt_dim,
float *obj_wgts,
int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
map<int, bool>& elInRank = zoltan->getElementInRank();
set<int> boxes;
map<int, double> boxWeight;
for (map<int, bool>::iterator it = elInRank.begin(); it != elInRank.end(); ++it)
if (it->second) {
int boxId = zoltan->elInBox[it->first];
boxes.insert(boxId);
if (zoltan->elWeights) {
TEST_EXIT_DBG(zoltan->elWeights->count(it->first))
("No element weight for element index %d\n", it->first);
boxWeight[boxId] += (*zoltan->elWeights)[it->first];
}
}
int localCounter = 0;
for (set<int>::iterator it = boxes.begin(); it != boxes.end(); ++it) {
globalId[localCounter] = *it;
localId[localCounter] = localCounter;
if (wgt_dim > 0) {
TEST_EXIT_DBG(wgt_dim == 1)("Should not happen!\n");
if (zoltan->elWeights)
obj_wgts[localCounter] = static_cast<float>(boxWeight[*it]);
else
obj_wgts[localCounter] = 1.0;
}
localCounter++;
}
*ierr = ZOLTAN_OK;
}
int ZoltanFunctions::box_getNumGeom(void *data, int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
*ierr = ZOLTAN_OK;
return zoltan->getMesh()->getGeo(WORLD);
}
void ZoltanFunctions::box_getGeomMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int num_dim,
double *geom_vec,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getGeomMulti()");
ERROR_EXIT("Not yet supported!\n");
}
void ZoltanFunctions::box_getNumEdgeMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getNumEdgeMulti()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
for (int i = 0; i < num_obj; i++) {
TEST_EXIT_DBG(zoltan->boxNeighbours.count(global_ids[i]))("Should not happen!\n");
num_edges[i] = zoltan->boxNeighbours[global_ids[i]].size();
}
*ierr = ZOLTAN_OK;
}
void ZoltanFunctions::box_getEdgeListMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
ZOLTAN_ID_PTR nbor_global_id,
int *nbor_procs,
int wgt_dim,
float *ewgts,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getEdgeListMulti()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
int c = 0;
for (int i = 0; i < num_obj; i++) {
TEST_EXIT_DBG(static_cast<int>(zoltan->boxNeighbours[global_ids[i]].size()) ==
num_edges[i])
("Wrong sizes for global el index %d: %d %d\n",
global_ids[i],
zoltan->boxNeighbours[global_ids[i]].size(),
num_edges[i]);
set<int> &neigh = zoltan->boxNeighbours[global_ids[i]];
for (set<int>::iterator it = neigh.begin(); it != neigh.end(); ++it) {
nbor_global_id[c] = *it;
nbor_procs[c] = zoltan->partitionMap[*(zoltan->boxSplitting[*it].begin())];
c++;
}
}
TEST_EXIT_DBG(wgt_dim == 0)("Edge weights are not handled yet!\n");
*ierr = ZOLTAN_OK;
}
}
......@@ -64,6 +64,8 @@ namespace AMDiS {
class ZoltanFunctions
{
public:
// === Zoltan query function for "normal" element partitioning. ===
static int getNumObj(void *data, int *ierr);
static void getObjectList(void *data,
......@@ -108,6 +110,55 @@ namespace AMDiS {
int wgt_dim,
float *ewgts,
int *ierr);
// === Zoltan query functions for box partitioning. ===
static int box_getNumObj(void *data, int *ierr);
static void box_getObjectList(void *data,
int sizeGid,
int sizeLid,
ZOLTAN_ID_PTR globalId,
ZOLTAN_ID_PTR localId,
int wgt_dim,
float *obj_wgts,
int *ierr);
static int box_getNumGeom(void *data, int *ierr);
static void box_getGeomMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int num_dim,
double *geom_vec,
int *ierr);
static void box_getNumEdgeMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
int *ierr);
static void box_getEdgeListMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
ZOLTAN_ID_PTR nbor_global_id,
int *nbor_procs,
int wgt_dim,
float *ewgts,
int *ierr);
};
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment