Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit b97f84ba authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Added box partitioning.

parent 8c23f60f
......@@ -105,6 +105,10 @@ namespace AMDiS {
if (partStr == "zoltan")
partitioner = new ZoltanPartitioner(&mpiComm);
tmp = 0;
GET_PARAMETER(0, name + "->box partitioning", "%d", &tmp);
partitioner->setBoxPartitioning(static_cast<bool>(tmp));
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
}
......
......@@ -29,40 +29,99 @@ namespace AMDiS {
// === Create initial partitioning of the AMDiS mesh. ===
elementInRank.clear();
map<DofEdge, set<int> > vertexElements;
TraverseStack stack;
ElInfo *elInfo =
stack.traverseFirst(mesh, 0,
Mesh::CALL_EL_LEVEL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
Mesh::CALL_EL_LEVEL | Mesh::FILL_NEIGH | Mesh::FILL_COORDS | Mesh::FILL_BOUND);
while (elInfo) {
int elIndex = elInfo->getElement()->getIndex();
int elInRank = std::min(elIndex / elPerRank, mpiSize - 1);
Element *el = elInfo->getElement();
int elIndex = el->getIndex();
if (elInRank == mpiRank)
elementInRank[elIndex] = true;
else
elementInRank[elIndex] = false;
partitionMap[elIndex] = elInRank;
for (int i = 0; i < mesh->getGeo(NEIGH); i++)
if (elInfo->getNeighbour(i) && elInfo->getBoundary(i) == INTERIOR)
elNeighbours[elIndex].push_back(elInfo->getNeighbour(i)->getIndex());
if (boxPartitioning) {
vertexElements[el->getEdge(0)].insert(elIndex);
} else {
int elInRank = std::min(elIndex / elPerRank, mpiSize - 1);
elementInRank[elIndex] = (elInRank == mpiRank);
partitionMap[elIndex] = elInRank;
}
elInfo = stack.traverseNext(elInfo);
}
if (boxPartitioning) {
TEST_EXIT(mesh->getDim() == 3)("Box partitioning only implemented for 3D!\n");
int boxCounter = 0;
for (map<DofEdge, set<int> >::iterator it = vertexElements.begin();
it != vertexElements.end(); ++it) {
TEST_EXIT_DBG(it->second.size() == 6)("Should not happen!\n");
boxSplitting[boxCounter] = it->second;
for (set<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt)
elInBox[*elIt] = boxCounter;
boxCounter++;
}
for (map<int, int>::iterator it = elInBox.begin(); it != elInBox.end(); ++it) {
int elBoxNo = it->second;
for (vector<int>::iterator neighIt = elNeighbours[it->first].begin();
neighIt != elNeighbours[it->first].end(); ++neighIt) {
int neighBoxNo = elInBox[*neighIt];
if (elBoxNo != neighBoxNo)
boxNeighbours[elBoxNo].insert(neighBoxNo);
}
}
MSG("Box partitioning with %d boxes enabled!\n", boxCounter);
int boxPerRank = boxCounter / mpiSize;
for (map<int, set<int> >::iterator it = boxSplitting.begin();
it != boxSplitting.end(); ++it) {
int boxInRank = std::min(it->first / boxPerRank, mpiSize - 1);
for (set<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt) {
elementInRank[*elIt] = (boxInRank == mpiRank);
partitionMap[*elIt] = boxInRank;
}
}
}
}
void MeshPartitioner::serialize(std::ostream &out)
{
SerUtil::serialize(out, elementInRank);
SerUtil::serialize(out, boxPartitioning);
SerUtil::serialize(out, boxSplitting);
SerUtil::serialize(out, boxNeighbours);
SerUtil::serialize(out, elInBox);
}
void MeshPartitioner::deserialize(std::istream &in)
{
SerUtil::deserialize(in, elementInRank);
SerUtil::deserialize(in, boxPartitioning);
SerUtil::deserialize(in, boxSplitting);
SerUtil::deserialize(in, boxNeighbours);
SerUtil::deserialize(in, elInBox);
}
}
......@@ -24,6 +24,7 @@
#define AMDIS_MESH_PARTITIONER_H
#include <map>
#include <set>
#include "AMDiS_fwd.h"
#include "Mesh.h"
......@@ -47,6 +48,7 @@ namespace AMDiS {
MeshPartitioner(MPI::Intracomm *comm)
: mpiComm(comm),
mesh(NULL),
boxPartitioning(false),
mapLocalGlobal(NULL)
{}
......@@ -71,6 +73,11 @@ namespace AMDiS {
mesh = m;
}
void setBoxPartitioning(bool b)
{
boxPartitioning = b;
}
void setLocalGlobalDofMap(map<DegreeOfFreedom, DegreeOfFreedom> *m)
{
mapLocalGlobal = m;
......@@ -101,6 +108,14 @@ namespace AMDiS {
Mesh *mesh;
bool boxPartitioning;
map<int, std::set<int> > boxSplitting;
map<int, std::set<int> > boxNeighbours;
map<int, int> elInBox;
map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal;
map<int, vector<int> > elNeighbours;
......
......@@ -21,14 +21,7 @@ namespace AMDiS {
: MeshPartitioner(comm),
zoltan(*comm),
elWeights(NULL)
{
zoltan.Set_Num_Obj_Fn(ZoltanFunctions::getNumObj, this);
zoltan.Set_Obj_List_Fn(ZoltanFunctions::getObjectList, this);
zoltan.Set_Num_Geom_Fn(ZoltanFunctions::getNumGeom, this);
zoltan.Set_Geom_Multi_Fn(ZoltanFunctions::getGeomMulti, this);
zoltan.Set_Num_Edges_Multi_Fn(ZoltanFunctions::getNumEdgeMulti, this);
zoltan.Set_Edge_List_Multi_Fn(ZoltanFunctions::getEdgeListMulti, this);
}
{}
bool ZoltanPartitioner::partition(map<int, double> &weights,
......@@ -36,6 +29,22 @@ namespace AMDiS {
{
FUNCNAME("ZoltanPartitioner::partition()");
if (!boxPartitioning) {
zoltan.Set_Num_Obj_Fn(ZoltanFunctions::getNumObj, this);
zoltan.Set_Obj_List_Fn(ZoltanFunctions::getObjectList, this);
zoltan.Set_Num_Geom_Fn(ZoltanFunctions::getNumGeom, this);
zoltan.Set_Geom_Multi_Fn(ZoltanFunctions::getGeomMulti, this);
zoltan.Set_Num_Edges_Multi_Fn(ZoltanFunctions::getNumEdgeMulti, this);
zoltan.Set_Edge_List_Multi_Fn(ZoltanFunctions::getEdgeListMulti, this);
} else {
zoltan.Set_Num_Obj_Fn(ZoltanFunctions::box_getNumObj, this);
zoltan.Set_Obj_List_Fn(ZoltanFunctions::box_getObjectList, this);
zoltan.Set_Num_Geom_Fn(ZoltanFunctions::box_getNumGeom, this);
zoltan.Set_Geom_Multi_Fn(ZoltanFunctions::box_getGeomMulti, this);
zoltan.Set_Num_Edges_Multi_Fn(ZoltanFunctions::box_getNumEdgeMulti, this);
zoltan.Set_Edge_List_Multi_Fn(ZoltanFunctions::box_getEdgeListMulti, this);
}
if (mode != INITIAL)
elWeights = &weights;
......@@ -56,6 +65,8 @@ namespace AMDiS {
if (mode == INITIAL) {
zoltan.Set_Param("LB_APPROACH", "PARTITION");
if (boxPartitioning)
zoltan.Set_Param("LB_METHOD", "GRAPH");
} else {
zoltan.Set_Param("LB_APPROACH", "REPARTITION");
zoltan.Set_Param("LB_METHOD", "GRAPH");
......@@ -89,8 +100,17 @@ namespace AMDiS {
for (int i = 0; i < nImportEls; i++) {
int recvElIndex = import_global_ids[i];
elementInRank[recvElIndex] = true;
recvElements[import_procs[i]].push_back(recvElIndex);
if (!boxPartitioning) {
elementInRank[recvElIndex] = true;
recvElements[import_procs[i]].push_back(recvElIndex);
} else {
set<int>& box = boxSplitting[recvElIndex];
for (set<int>::iterator it = box.begin(); it != box.end(); ++it) {
elementInRank[*it] = true;
recvElements[import_procs[i]].push_back(*it);
}
}
}
}
......@@ -98,8 +118,17 @@ namespace AMDiS {
for (int i = 0; i < nExportEls; i++) {
int sendElIndex = export_global_ids[i];
elementInRank[sendElIndex] = false;
sendElements[export_procs[i]].push_back(sendElIndex);
if (!boxPartitioning) {
elementInRank[sendElIndex] = false;
sendElements[export_procs[i]].push_back(sendElIndex);
} else {
set<int>& box = boxSplitting[sendElIndex];
for (set<int>::iterator it = box.begin(); it != box.end(); ++it) {
elementInRank[*it] = false;
sendElements[export_procs[i]].push_back(*it);
}
}
}
}
......@@ -167,8 +196,11 @@ namespace AMDiS {
partitionMap[partitionElements[elDist[i] + j]] = i;
}
// === Zoltan query function for "normal" element partitioning. ===
int ZoltanFunctions::getNumObj(void *data, int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
......@@ -340,4 +372,166 @@ namespace AMDiS {
*ierr = ZOLTAN_OK;
}
// === Zoltan query functions for box partitioning. ===
int ZoltanFunctions::box_getNumObj(void *data, int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getNumObj()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
map<int, bool>& elInRank = zoltan->getElementInRank();
int nObjects = 0;
for (map<int, bool>::iterator it = elInRank.begin(); it != elInRank.end(); ++it)
if (it->second)
nObjects++;
TEST_EXIT_DBG(nObjects % 6 == 0)("Should not happen!\n");
nObjects = nObjects / 6;
*ierr = ZOLTAN_OK;
return nObjects;
}
void ZoltanFunctions::box_getObjectList(void *data,
int sizeGid,
int sizeLid,
ZOLTAN_ID_PTR globalId,
ZOLTAN_ID_PTR localId,
int wgt_dim,
float *obj_wgts,
int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
map<int, bool>& elInRank = zoltan->getElementInRank();
set<int> boxes;
map<int, double> boxWeight;
for (map<int, bool>::iterator it = elInRank.begin(); it != elInRank.end(); ++it)
if (it->second) {
int boxId = zoltan->elInBox[it->first];
boxes.insert(boxId);
if (zoltan->elWeights) {
TEST_EXIT_DBG(zoltan->elWeights->count(it->first))
("No element weight for element index %d\n", it->first);
boxWeight[boxId] += (*zoltan->elWeights)[it->first];
}
}
int localCounter = 0;
for (set<int>::iterator it = boxes.begin(); it != boxes.end(); ++it) {
globalId[localCounter] = *it;
localId[localCounter] = localCounter;
if (wgt_dim > 0) {
TEST_EXIT_DBG(wgt_dim == 1)("Should not happen!\n");
if (zoltan->elWeights)
obj_wgts[localCounter] = static_cast<float>(boxWeight[*it]);
else
obj_wgts[localCounter] = 1.0;
}
localCounter++;
}
*ierr = ZOLTAN_OK;
}
int ZoltanFunctions::box_getNumGeom(void *data, int *ierr)
{
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
*ierr = ZOLTAN_OK;
return zoltan->getMesh()->getGeo(WORLD);
}
void ZoltanFunctions::box_getGeomMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int num_dim,
double *geom_vec,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getGeomMulti()");
ERROR_EXIT("Not yet supported!\n");
}
void ZoltanFunctions::box_getNumEdgeMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getNumEdgeMulti()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
for (int i = 0; i < num_obj; i++) {
TEST_EXIT_DBG(zoltan->boxNeighbours.count(global_ids[i]))("Should not happen!\n");
num_edges[i] = zoltan->boxNeighbours[global_ids[i]].size();
}
*ierr = ZOLTAN_OK;
}
void ZoltanFunctions::box_getEdgeListMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
ZOLTAN_ID_PTR nbor_global_id,
int *nbor_procs,
int wgt_dim,
float *ewgts,
int *ierr)
{
FUNCNAME("ZoltanFunctions::box_getEdgeListMulti()");
ZoltanPartitioner* zoltan = (ZoltanPartitioner*)data;
int c = 0;
for (int i = 0; i < num_obj; i++) {
TEST_EXIT_DBG(static_cast<int>(zoltan->boxNeighbours[global_ids[i]].size()) ==
num_edges[i])
("Wrong sizes for global el index %d: %d %d\n",
global_ids[i],
zoltan->boxNeighbours[global_ids[i]].size(),
num_edges[i]);
set<int> &neigh = zoltan->boxNeighbours[global_ids[i]];
for (set<int>::iterator it = neigh.begin(); it != neigh.end(); ++it) {
nbor_global_id[c] = *it;
nbor_procs[c] = zoltan->partitionMap[*(zoltan->boxSplitting[*it].begin())];
c++;
}
}
TEST_EXIT_DBG(wgt_dim == 0)("Edge weights are not handled yet!\n");
*ierr = ZOLTAN_OK;
}
}
......@@ -64,6 +64,8 @@ namespace AMDiS {
class ZoltanFunctions
{
public:
// === Zoltan query function for "normal" element partitioning. ===
static int getNumObj(void *data, int *ierr);
static void getObjectList(void *data,
......@@ -108,6 +110,55 @@ namespace AMDiS {
int wgt_dim,
float *ewgts,
int *ierr);
// === Zoltan query functions for box partitioning. ===
static int box_getNumObj(void *data, int *ierr);
static void box_getObjectList(void *data,
int sizeGid,
int sizeLid,
ZOLTAN_ID_PTR globalId,
ZOLTAN_ID_PTR localId,
int wgt_dim,
float *obj_wgts,
int *ierr);
static int box_getNumGeom(void *data, int *ierr);
static void box_getGeomMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int num_dim,
double *geom_vec,
int *ierr);
static void box_getNumEdgeMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
int *ierr);
static void box_getEdgeListMulti(void *data,
int num_gid_entries,
int num_lid_entries,
int num_obj,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR local_ids,
int *num_edges,
ZOLTAN_ID_PTR nbor_global_id,
int *nbor_procs,
int wgt_dim,
float *ewgts,
int *ierr);
};
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment