Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 6dff19bd authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed bug for parallelization with non exp-2 number of processes.

parent 54114ba0
......@@ -116,6 +116,7 @@ namespace AMDiS {
}
}
ParMetisMesh::~ParMetisMesh()
{
if (eptr)
......@@ -134,6 +135,7 @@ namespace AMDiS {
delete [] elem_p2a;
}
ParMetisGraph::ParMetisGraph(ParMetisMesh *parMesh,
MPI::Intracomm *comm,
int ncommonnodes)
......@@ -156,6 +158,7 @@ namespace AMDiS {
&tmpComm);
}
ParMetisGraph::~ParMetisGraph()
{
free(xadj);
......@@ -174,8 +177,11 @@ namespace AMDiS {
}
}
void ParMetisPartitioner::createPartitionData()
{
FUNCNAME("ParMetrisPartitioner::createPartitionData()");
int mpiRank = mpiComm->Get_rank();
int mpiSize = mpiComm->Get_size();
int nLeaves = mesh->getNumberOfLeaves();
......@@ -195,8 +201,10 @@ namespace AMDiS {
new PartitionElementData(element->getElementData());
element->setElementData(elData);
if (element->getIndex() >= mpiRank * elPerRank &&
element->getIndex() < (mpiRank + 1) * elPerRank)
if ((element->getIndex() >= mpiRank * elPerRank &&
element->getIndex() < (mpiRank + 1) * elPerRank) ||
(element->getIndex() >= mpiSize * elPerRank &&
mpiRank == mpiSize - 1))
elData->setPartitionStatus(IN);
else
elData->setPartitionStatus(UNDEFINED);
......@@ -205,6 +213,7 @@ namespace AMDiS {
}
}
void ParMetisPartitioner::partition(std::map<int, double> *elemWeights,
PartitionMode mode,
float itr)
......@@ -362,6 +371,7 @@ namespace AMDiS {
delete [] part;
}
void ParMetisPartitioner::fillCoarsePartitionVec(std::map<int, int> *partitionVec)
{
TEST_EXIT(partitionVec)("no partition vector\n");
......@@ -405,6 +415,7 @@ namespace AMDiS {
delete [] nPartitionElements;
}
void ParMetisPartitioner::distributePartitioning(int *part)
{
FUNCNAME("ParMetisPartitioner::distributePartitioning()");
......@@ -510,6 +521,7 @@ namespace AMDiS {
delete [] recvBufferOffset;
}
void ParMetisPartitioner::descendPartitionData(Element *element)
{
FUNCNAME("ParMetisPartitioner::descendPartitionData()");
......
......@@ -35,6 +35,7 @@ namespace AMDiS {
return (*dof1 < *dof2);
}
ParallelDomainBase::ParallelDomainBase(ProblemVec *problemStat,
ProblemInstatVec *problemInstat)
: iterationIF(problemStat),
......@@ -126,6 +127,8 @@ namespace AMDiS {
else
MSG("Skip write part mesh!\n");
}
ParallelDomainDbg::testAllElements(*this);
#endif
// === Create interior boundary information. ===
......@@ -450,8 +453,6 @@ namespace AMDiS {
if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
// MSG("START WITH I = %d\n", i);
bool b = fitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el,
boundIt->rankObj.ithObj,
......@@ -496,9 +497,7 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
// MSG("------- START 0 ----------\n");
meshChanged = fitElementToMeshCode2(code, stack, elInfo, ithSide, elType);
meshChanged = fitElementToMeshCode2(code, stack, ithSide, elType);
return meshChanged;
}
......@@ -518,8 +517,6 @@ namespace AMDiS {
}
if (s1 != -1) {
// MSG("------- START 1 ----------\n");
TraverseStack stack;
ElInfo *elInfo =
stack.traverseFirst(el->getMesh(), -1, Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
......@@ -528,10 +525,8 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
meshChanged |= fitElementToMeshCode2(code, stack, elInfo, s1, el->getChildType(elType));
meshChanged |= fitElementToMeshCode2(code, stack, s1, el->getChildType(elType));
} else {
// MSG("------- START 2 --------- %d \n", el->getIndex());
TraverseStack stack;
ElInfo *elInfo =
stack.traverseFirst(el->getMesh(), -1, Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
......@@ -540,18 +535,15 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
meshChanged |= fitElementToMeshCode2(code, stack, elInfo, s2, el->getChildType(elType));
meshChanged |= fitElementToMeshCode2(code, stack, s2, el->getChildType(elType));
}
// MSG("-------- ENDE ---------\n");
return meshChanged;
}
bool ParallelDomainBase::fitElementToMeshCode2(MeshStructure &code,
TraverseStack &stack,
ElInfo *aelInfo,
int ithSide,
int elType)
{
......@@ -559,8 +551,6 @@ namespace AMDiS {
ElInfo *elInfo = stack.getElInfo();
// MSG("START EL WITH LEVEL = %d\n", elInfo->getLevel());
bool value = false;
if (!elInfo)
return value;
......@@ -573,9 +563,7 @@ namespace AMDiS {
do {
elInfo = stack.traverseNext(elInfo);
} while (elInfo && elInfo->getLevel() > level);
// MSG("RETURN CODE LEAF: %d\n", elInfo->getLevel());
return value;
}
......@@ -583,7 +571,6 @@ namespace AMDiS {
return value;
if (el->isLeaf()) {
// MSG("REFINE CODE NO LEAF!\n");
el->setMark(1);
refineManager->setMesh(el->getMesh());
refineManager->setStack(&stack);
......@@ -595,34 +582,28 @@ namespace AMDiS {
int s2 = el->getSideOfChild(1, ithSide, elType);
if (s1 != -1) {
// MSG("STEP LEFT 1\n");
stack.traverseNext(elInfo);
code.nextElement();
value |= fitElementToMeshCode2(code, stack, elInfo, s1, el->getChildType(elType));
value |= fitElementToMeshCode2(code, stack, s1, el->getChildType(elType));
elInfo = stack.getElInfo();
// MSG("STEP LEFT 2\n");
} else {
do {
elInfo = stack.traverseNext(elInfo);
} while (elInfo && elInfo->getElement() != el->getSecondChild());
// MSG("STEP LEFT OMMIT\n");
}
TEST_EXIT_DBG(elInfo->getElement() == el->getSecondChild())
("This should not happen!\n");
if (s2 != -1) {
// MSG("STEP RIGHT 1\n");
code.nextElement();
value |= fitElementToMeshCode2(code, stack, elInfo, s2, el->getChildType(elType));
// MSG("STEP RIGHT 2\n");
value |= fitElementToMeshCode2(code, stack, s2, el->getChildType(elType));
} else {
int level = elInfo->getLevel();
do {
elInfo = stack.traverseNext(elInfo);
} while (elInfo && elInfo->getLevel() > level);
// MSG("STEP RIGHT OMMIT\n");
}
return value;
......@@ -711,29 +692,13 @@ namespace AMDiS {
}
infile.close();
} else {
} else {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
Element *element = elInfo->getElement();
// get partition data
PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData && partitionData->getPartitionStatus() == IN) {
int elNum = -1;
if (partitionData->getLevel() == 0)
elNum = element->getIndex();
TEST_EXIT_DBG(elNum != -1)("invalid element number\n");
if (element->isLeaf()) {
elemWeights[elNum] += 1.0;
localWeightSum += 1.0;
}
}
elemWeights[elInfo->getElement()->getIndex()] = 1.0;
localWeightSum++;
elInfo = stack.traverseNext(elInfo);
}
}
......@@ -791,7 +756,7 @@ namespace AMDiS {
stdMpi.send(myIntBoundary.boundary);
stdMpi.recv(otherIntBoundary.boundary);
stdMpi.startCommunication<int>(MPI_INT);
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
......@@ -800,10 +765,11 @@ namespace AMDiS {
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. We now have to sort the corresponding list in this rank to ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
for (int j = 0; j < static_cast<int>(rankIt->second.size()); j++) {
// If the expected object is not at place, search for it.
BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj;
......@@ -897,7 +863,7 @@ namespace AMDiS {
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
// Check, if the element is within rank's partition.
if (partitionData->getPartitionStatus() == IN) {
......
......@@ -300,7 +300,6 @@ namespace AMDiS {
bool fitElementToMeshCode2(MeshStructure &code,
TraverseStack &stack,
ElInfo *elInfo,
int ithSide,
int elType);
......
#include "ParallelDomainDbg.h"
#include "ParallelDomainBase.h"
#include "PartitionElementData.h"
#include "ProblemVec.h"
#include "DOFVector.h"
#include "FixVec.h"
......@@ -208,6 +209,37 @@ namespace AMDiS {
}
void ParallelDomainDbg::testAllElements(ParallelDomainBase &pdb)
{
FUNCNAME("ParallelDomainDbg::testAllElements()");
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
int sendId = 0;
if (partitionData->getPartitionStatus() == IN)
sendId = 1;
int recvId = 0;
pdb.mpiComm.Allreduce(&sendId, &recvId, 1, MPI_INT, MPI_SUM);
if (recvId != 1 && pdb.mpiRank == 0) {
if (recvId == 0)
ERROR_EXIT("Element %d has no member partition!\n", element->getIndex());
if (recvId > 1)
ERROR_EXIT("Element %d is member of more than pne partition!\n",
element->getIndex());
}
elInfo = stack.traverseNext(elInfo);
}
}
void ParallelDomainDbg::printMapLocalGlobal(ParallelDomainBase &pdb, int rank)
{
if (rank == -1 || pdb.mpiRank == rank) {
......
......@@ -47,6 +47,13 @@ namespace AMDiS {
*/
static void testCommonDofs(ParallelDomainBase &pdb, bool printCoords = false);
/** \brief
* Tests if all elements in the macro mesh are memeber of exactly one rank.
*
* \param[in] pdb Parallel problem definition used for debugging.
*/
static void testAllElements(ParallelDomainBase &pdb);
/** \brief
* This function is used for debugging only. It prints all information from
* the local to global dof mapping, see \ref mapLocalGlobalDofs.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment