Commit 97538b56 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Zoltan and ParMETiS work now both together, use parameter in init file to switch between them.

parent f1665f9d
...@@ -96,8 +96,16 @@ namespace AMDiS { ...@@ -96,8 +96,16 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->log main rank", "%d", &tmp); GET_PARAMETER(0, name + "->log main rank", "%d", &tmp);
Msg::outputMainRank = (tmp > 0); Msg::outputMainRank = (tmp > 0);
// partitioner = new ParMetisPartitioner(&mpiComm); string partStr = "parmetis";
partitioner = new ZoltanPartitioner(&mpiComm); GET_PARAMETER(0, name + "->partitioner", &partStr);
if (partStr == "parmetis")
partitioner = new ParMetisPartitioner(&mpiComm);
if (partStr == "zoltan")
partitioner = new ZoltanPartitioner(&mpiComm);
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
} }
...@@ -256,11 +264,13 @@ namespace AMDiS { ...@@ -256,11 +264,13 @@ namespace AMDiS {
MSG("Debug mode tests finished!\n"); MSG("Debug mode tests finished!\n");
#endif #endif
// === Create periodic DOF mapping, if there are periodic boundaries. === // === Create periodic DOF mapping, if there are periodic boundaries. ===
createPeriodicMap(); createPeriodicMap();
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
#endif
// === Global refinements. === // === Global refinements. ===
...@@ -276,6 +286,10 @@ namespace AMDiS { ...@@ -276,6 +286,10 @@ namespace AMDiS {
// === Update periodic mapping, if there are periodic boundaries. === // === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(); createPeriodicMap();
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
#endif
} }
...@@ -700,21 +714,29 @@ namespace AMDiS { ...@@ -700,21 +714,29 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, debugOutputDir + "mesh"); debug::writeMesh(feSpace, -1, debugOutputDir + "mesh");
#endif #endif
// === Because the mesh has been changed, update the DOF numbering and mappings. === // === Because the mesh has been changed, update the DOF numbering and mappings. ===
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. === // === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(); createPeriodicMap();
INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n", #if (DEBUG != 0)
MPI::Wtime() - first); ParallelDebug::testPeriodicBoundary(*this);
#endif
// === The mesh has changed, so check if it is required to repartition the mesh. === // === The mesh has changed, so check if it is required to repartition the mesh. ===
nMeshChangesAfterLastRepartitioning++; nMeshChangesAfterLastRepartitioning++;
INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n",
MPI::Wtime() - first);
if (repartitioningAllowed && if (repartitioningAllowed &&
nMeshChangesAfterLastRepartitioning >= repartitionIthChange) { nMeshChangesAfterLastRepartitioning >= repartitionIthChange) {
repartitionMesh(); repartitionMesh();
...@@ -1781,8 +1803,8 @@ namespace AMDiS { ...@@ -1781,8 +1803,8 @@ namespace AMDiS {
} }
stdMpi.updateSendDataSize(); stdMpi.updateSendDataSize();
stdMpi.startCommunication();
stdMpi.startCommunication();
// === The rank has received the dofs from the rank on the other side of === // === The rank has received the dofs from the rank on the other side of ===
// === the boundary. Now it can use them to create the mapping between === // === the boundary. Now it can use them to create the mapping between ===
...@@ -1856,9 +1878,9 @@ namespace AMDiS { ...@@ -1856,9 +1878,9 @@ namespace AMDiS {
stdMpi2.startCommunication(); stdMpi2.startCommunication();
for (std::map<int, PeriodicDofMap>::iterator it = stdMpi2.getRecvData().begin(); for (std::map<int, PeriodicDofMap>::iterator it = stdMpi2.getRecvData().begin();
it != stdMpi2.getRecvData().end(); ++it) it != stdMpi2.getRecvData().end(); ++it) {
for (PeriodicDofMap::iterator perIt = it->second.begin(); for (PeriodicDofMap::iterator perIt = it->second.begin();
perIt != it->second.end(); ++perIt) perIt != it->second.end(); ++perIt) {
for (DofMapping::iterator dofIt = perIt->second.begin(); for (DofMapping::iterator dofIt = perIt->second.begin();
dofIt != perIt->second.end(); ++dofIt) { dofIt != perIt->second.end(); ++dofIt) {
TEST_EXIT_DBG(periodicDof[perIt->first].count(dofIt->second) == 0 || TEST_EXIT_DBG(periodicDof[perIt->first].count(dofIt->second) == 0 ||
...@@ -1867,10 +1889,8 @@ namespace AMDiS { ...@@ -1867,10 +1889,8 @@ namespace AMDiS {
periodicDof[perIt->first][dofIt->second] = dofIt->first; periodicDof[perIt->first][dofIt->second] = dofIt->first;
} }
}
#if (DEBUG != 0) }
ParallelDebug::testPeriodicBoundary(*this);
#endif
} }
......
...@@ -28,7 +28,7 @@ namespace AMDiS { ...@@ -28,7 +28,7 @@ namespace AMDiS {
ParMetisMesh::ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm, ParMetisMesh::ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm,
std::map<int, bool>& elementInRank, std::map<int, bool>& elementInRank,
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal) map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal)
: dim(mesh->getDim()), : dim(mesh->getDim()),
nElements(0), nElements(0),
mpiComm(comm) mpiComm(comm)
...@@ -184,7 +184,7 @@ namespace AMDiS { ...@@ -184,7 +184,7 @@ namespace AMDiS {
{ {
FUNCNAME("ParMetisGraph::print()"); FUNCNAME("ParMetisGraph::print()");
std::stringstream oss; stringstream oss;
for (int i = 0; i <= MPI::COMM_WORLD.Get_size(); i++) for (int i = 0; i <= MPI::COMM_WORLD.Get_size(); i++)
oss << parMetisMesh->getElementDist()[i] << " "; oss << parMetisMesh->getElementDist()[i] << " ";
...@@ -224,7 +224,7 @@ namespace AMDiS { ...@@ -224,7 +224,7 @@ namespace AMDiS {
} }
bool ParMetisPartitioner::partition(std::map<int, double> &elemWeights, bool ParMetisPartitioner::partition(map<int, double> &elemWeights,
PartitionMode mode) PartitionMode mode)
{ {
FUNCNAME("ParMetisPartitioner::partition()"); FUNCNAME("ParMetisPartitioner::partition()");
...@@ -246,8 +246,8 @@ namespace AMDiS { ...@@ -246,8 +246,8 @@ namespace AMDiS {
// === Create weight array === // === Create weight array ===
std::vector<int> wgts(nElements); vector<int> wgts(nElements);
std::vector<float> floatWgts(nElements); vector<float> floatWgts(nElements);
unsigned int floatWgtsPos = 0; unsigned int floatWgtsPos = 0;
float maxWgt = 0.0; float maxWgt = 0.0;
...@@ -287,11 +287,11 @@ namespace AMDiS { ...@@ -287,11 +287,11 @@ namespace AMDiS {
int ncon = 1; // one weight at each vertex! int ncon = 1; // one weight at each vertex!
int nparts = mpiSize; // number of partitions int nparts = mpiSize; // number of partitions
std::vector<float> tpwgts(mpiSize); vector<float> tpwgts(mpiSize);
float ubvec = 1.05; float ubvec = 1.05;
int options[4] = {0, 0, 15, 1}; // default options int options[4] = {0, 0, 15, 1}; // default options
int edgecut = -1; int edgecut = -1;
std::vector<int> part(nElements); vector<int> part(nElements);
// set tpwgts // set tpwgts
for (int i = 0; i < mpiSize; i++) for (int i = 0; i < mpiSize; i++)
...@@ -373,7 +373,7 @@ namespace AMDiS { ...@@ -373,7 +373,7 @@ namespace AMDiS {
break; break;
case ADAPTIVE_REPART: case ADAPTIVE_REPART:
{ {
std::vector<int> vsize(nElements); vector<int> vsize(nElements);
for (int i = 0; i < nElements; i++) for (int i = 0; i < nElements; i++)
vsize[i] = static_cast<int>(floatWgts[i]); vsize[i] = static_cast<int>(floatWgts[i]);
...@@ -425,7 +425,7 @@ namespace AMDiS { ...@@ -425,7 +425,7 @@ namespace AMDiS {
} }
void ParMetisPartitioner::getPartitionMap(std::map<int, int> &partitionMap) void ParMetisPartitioner::getPartitionMap(map<int, int> &partitionMap)
{ {
FUNCNAME("ParMetisPartitioner::getPartitionMap()"); FUNCNAME("ParMetisPartitioner::getPartitionMap()");
...@@ -437,7 +437,7 @@ namespace AMDiS { ...@@ -437,7 +437,7 @@ namespace AMDiS {
int mpiRank = mpiComm->Get_rank(); int mpiRank = mpiComm->Get_rank();
int mpiSize = mpiComm->Get_size(); int mpiSize = mpiComm->Get_size();
std::vector<int> nPartitionElements(mpiSize); vector<int> nPartitionElements(mpiSize);
int *elmdist = parMetisMesh->getElementDist(); int *elmdist = parMetisMesh->getElementDist();
for (int i = 0; i < mpiSize; i++) for (int i = 0; i < mpiSize; i++)
...@@ -448,7 +448,7 @@ namespace AMDiS { ...@@ -448,7 +448,7 @@ namespace AMDiS {
int localElements = parMetisMesh->getNumElements(); int localElements = parMetisMesh->getNumElements();
mpiComm->Allreduce(&localElements, &nElements, 1, MPI_INT, MPI_SUM); mpiComm->Allreduce(&localElements, &nElements, 1, MPI_INT, MPI_SUM);
std::vector<int> partitionElements(nElements); vector<int> partitionElements(nElements);
// distribute partition elements // distribute partition elements
mpiComm->Allgatherv(parMetisMesh->getAMDiSIndices(), mpiComm->Allgatherv(parMetisMesh->getAMDiSIndices(),
...@@ -459,7 +459,7 @@ namespace AMDiS { ...@@ -459,7 +459,7 @@ namespace AMDiS {
elmdist, elmdist,
MPI_INT); MPI_INT);
// fill partitionVec // fill partitionMap
for (int i = 0; i < mpiSize; i++) for (int i = 0; i < mpiSize; i++)
for (int j = 0; j < nPartitionElements[i]; j++) for (int j = 0; j < nPartitionElements[i]; j++)
partitionMap[partitionElements[elmdist[i] + j]] = i; partitionMap[partitionElements[elmdist[i] + j]] = i;
...@@ -540,7 +540,7 @@ namespace AMDiS { ...@@ -540,7 +540,7 @@ namespace AMDiS {
MPI_INT); MPI_INT);
TEST_EXIT(elementInRank.size() != 0)("Should not happen!\n"); TEST_EXIT(elementInRank.size() != 0)("Should not happen!\n");
for (std::map<int, bool>::iterator it = elementInRank.begin(); for (map<int, bool>::iterator it = elementInRank.begin();
it != elementInRank.end(); ++it) it != elementInRank.end(); ++it)
elementInRank[it->first] = false; elementInRank[it->first] = false;
......
...@@ -146,7 +146,7 @@ namespace AMDiS { ...@@ -146,7 +146,7 @@ namespace AMDiS {
// === 1. check: All periodic DOFs should have at least a correct number === // === 1. check: All periodic DOFs should have at least a correct number ===
// === of periodic associations. === // === of periodic associations. ===
for (map<int, std::set<BoundaryType> >::iterator it = pdb.periodicDofAssociations.begin(); for (map<int, std::set<BoundaryType> >::iterator it = pdb.periodicDofAssociations.begin();
it != pdb.periodicDofAssociations.end(); ++it) { it != pdb.periodicDofAssociations.end(); ++it) {
WorldVector<double> c; WorldVector<double> c;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment