Skip to content

GitLab

Commit cd501be4 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

* Small code changes in ParallelProblem

parent 50cb43cc
Hide whitespace changes
Inline Side-by-side
Showing with 181 additions and 106 deletions
AMDiS/src/ParMetisPartitioner.h
View file @ cd501be4
......@@ -50,25 +50,17 @@ namespace AMDiS {
~ParMetisMesh();
inline void setParMetisIndex(int amdisIndex,
int parMetisIndex)
{
// if(MPI::COMM_WORLD.Get_rank() == 0)
// MSG("set %p %d %d\n", this, amdisIndex, parMetisIndex);
inline void setParMetisIndex(int amdisIndex, int parMetisIndex) {
elem_a2p_[amdisIndex] = parMetisIndex + 1;
};
inline int getParMetisIndex(int amdisIndex) {
int result = elem_a2p_[amdisIndex];
// if(MPI::COMM_WORLD.Get_rank() == 0)
// MSG("get %p %d %d\n", this, amdisIndex, result - 1);
TEST_EXIT(result > 0)("invalid index\n");
return result - 1;
};
inline void setAMDiSIndex(int parMetisIndex,
int amdisIndex)
{
inline void setAMDiSIndex(int parMetisIndex, int amdisIndex) {
elem_p2a_[parMetisIndex] = amdisIndex;
};
......@@ -80,12 +72,29 @@ namespace AMDiS {
return elem_p2a_;
};
inline int *getElementPtr() { return eptr_; };
inline int *getElementInd() { return eind_; };
inline int *getElementDist() { return elmdist_; };
inline int getDim() { return dim_; };
inline float *getXYZ() { return xyz_; };
inline int getNumElements() { return numElements_; };
inline int *getElementPtr() {
return eptr_;
};
inline int *getElementInd() {
return eind_;
};
inline int *getElementDist() {
return elmdist_;
};
inline int getDim() {
return dim_;
};
inline float *getXYZ() {
return xyz_;
};
inline int getNumElements() {
return numElements_;
};
protected:
int *eptr_;
......@@ -110,8 +119,13 @@ namespace AMDiS {
~ParMetisGraph();
inline int *getXAdj() { return xadj_; };
inline int *getAdjncy() { return adjncy_; };
inline int *getXAdj() {
return xadj_;
};
inline int *getAdjncy() {
return adjncy_;
};
protected:
ParMetisMesh *parMetisMesh_;
......
AMDiS/src/ParallelProblem.cc
View file @ cd501be4
......@@ -451,9 +451,8 @@ namespace AMDiS {
exchangeMeshStructureCodes(structures);
// merge codes
int rank;
for(rank = 0; rank < mpiSize_; rank++) {
if(rank != mpiRank_) {
for (int rank = 0; rank < mpiSize_; rank++) {
if (rank != mpiRank_) {
structures[mpiRank_].merge(&structures[rank]);
}
}
......@@ -511,7 +510,7 @@ namespace AMDiS {
const FiniteElemSpace *feSpace = rankSolutions[0]->getFESpace();
TEST_EXIT(feSpace->getMesh() == mesh_)("invalid mesh\n");
int i, dim = mesh_->getDim();
int dim = mesh_->getDim();
const BasisFunction *basFcts = feSpace->getBasisFcts();
int numFcts = basFcts->getNumber();
DegreeOfFreedom *coarseDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
......@@ -519,11 +518,6 @@ namespace AMDiS {
DOFAdmin *admin = feSpace->getAdmin();
int partition;
// std::vector<std::map<WorldVector<double>, DegreeOfFreedom> > sortedSendDOFs;
// std::vector<std::map<WorldVector<double>, DegreeOfFreedom> > sortedRecvDOFs;
// sortedSendDOFs.resize(mpiSize_);
// sortedRecvDOFs.resize(mpiSize_);
std::vector<std::vector<DegreeOfFreedom> > sendOrder;
std::vector<std::vector<DegreeOfFreedom> > recvOrder;
sendOrder.resize(mpiSize_);
......@@ -531,45 +525,36 @@ namespace AMDiS {
std::set<int>::iterator setIt, setBegin, setEnd;
//std::map<Element*, std::set<int> > elementPartitions;
elementPartitions_.clear();
// if(mpiRank_ == 0) {
// bool wait = true;
// while(wait) {}
// }
int elementPartition = -1;
Element *coarseElement = NULL;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh_, -1, Mesh::CALL_EVERY_EL_PREORDER);
while(elInfo) {
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if(partitionData) {
if(partitionData->getLevel() == 0) {
if (partitionData) {
if (partitionData->getLevel() == 0) {
elementPartition = partitionVec_[element->getIndex()];
}
PartitionStatus status = partitionData->getPartitionStatus();
if(status != OUT) {
if(partitionData->getLevel() == localCoarseGridLevel_) {
if (status != OUT) {
if (partitionData->getLevel() == localCoarseGridLevel_) {
basFcts->getLocalIndices(element,
admin,
coarseDOFs);
// collect other partitions element belongs to
for(i = 0; i < dim + 1; i++) {
for (int i = 0; i < dim + 1; i++) {
setBegin = vertexPartitions_[coarseDOFs[i]].begin();
setEnd = vertexPartitions_[coarseDOFs[i]].end();
// setBegin = vertexPartitions_[element->getDOF(i, 0)].begin();
// setEnd = vertexPartitions_[element->getDOF(i, 0)].end();
for(setIt = setBegin; setIt != setEnd; ++setIt) {
for (setIt = setBegin; setIt != setEnd; ++setIt) {
elementPartitions_[element].insert(*setIt/* - 1*/);
}
}
......@@ -578,25 +563,24 @@ namespace AMDiS {
}
if(element->isLeaf()) {
if (element->isLeaf()) {
basFcts->getLocalIndices(element,
admin,
fineDOFs);
for(i = 0; i < numFcts; i++) {
if(status == OVERLAP) {
for (int i = 0; i < numFcts; i++) {
if (status == OVERLAP) {
// send dofs
sendOrder[elementPartition].push_back(fineDOFs[i]);
}
if(status == IN) {
if (status == IN) {
// recv dofs
TEST_EXIT(elementPartition == mpiRank_)("???\n");
setBegin = elementPartitions_[coarseElement].begin();
setEnd = elementPartitions_[coarseElement].end();
for(setIt = setBegin; setIt != setEnd; ++setIt) {
if(*setIt != mpiRank_) {
for (setIt = setBegin; setIt != setEnd; ++setIt) {
if (*setIt != mpiRank_) {
recvOrder[*setIt].push_back(fineDOFs[i]);
//recvOrder[*setIt].push_back(element->getDOF(i, 0));
}
}
}
......@@ -661,12 +645,12 @@ namespace AMDiS {
WorldVector<double> worldCoords;
for(i = 0; i < numFcts; i++) {
elInfo2->coordToWorld(*(basFcts->getCoords(i)), &worldCoords);
if(status == OVERLAP) {
if (status == OVERLAP) {
// send dofs
//sortedSendDOFs[elementPartition][worldCoords] = fineDOFs[i];
sendOrder[elementPartition].push_back(fineDOFs[i]);
}
if(status == IN) {
if (status == IN) {
// recv dofs
TEST_EXIT(elementPartition == mpiRank_)("???\n");
setBegin = elementPartitions_[element1].begin();
......@@ -694,10 +678,8 @@ namespace AMDiS {
std::map<int, int> sendBufferSize;
std::map<int, int> recvBufferSize;
for(partition = 0; partition < mpiSize_; partition++) {
if(partition != mpiRank_) {
//int sendSize = static_cast<int>(sortedSendDOFs[partition].size());
//int recvSize = static_cast<int>(sortedRecvDOFs[partition].size());
for (partition = 0; partition < mpiSize_; partition++) {
if (partition != mpiRank_) {
int sendSize = static_cast<int>(sendOrder[partition].size());
int recvSize = static_cast<int>(recvOrder[partition].size());
......@@ -705,9 +687,7 @@ namespace AMDiS {
recvBufferSize[partition] = recvSize;
if(sendSize > 0) {
sendBuffer[partition] = GET_MEMORY(double, sendSize);
//std::map<WorldVector<double>, DegreeOfFreedom>::iterator dofIt;
std::vector<DegreeOfFreedom>::iterator dofIt;
//dofIt = sortedSendDOFs[partition].begin();
dofIt = sendOrder[partition].begin();
double *bufferIt, *bufferBegin, *bufferEnd;
bufferBegin = sendBuffer[partition];
......@@ -716,7 +696,6 @@ namespace AMDiS {
bufferIt < bufferEnd;
++bufferIt, ++dofIt)
{
//*bufferIt = (*solution)[dofIt->second];
*bufferIt = (*solution)[*dofIt];
}
}
......@@ -725,13 +704,6 @@ namespace AMDiS {
}
}
// for(partition = 0; partition < mpiSize_; partition++) {
// MSG("rank %d -> rank %d: %d",
// mpiRank_, partition, sendBufferSize[partition]);
// MSG("rank %d <- rank %d: %d",
// mpiRank_, partition, recvBufferSize[partition]);
// }
// non-blocking sends
for(partition = 0; partition < mpiSize_; partition++) {
if(partition != mpiRank_) {
......@@ -745,8 +717,6 @@ namespace AMDiS {
}
}
//MPI::COMM_WORLD.Barrier();
// blocking recieves
for(partition = 0; partition < mpiSize_; partition++) {
if(partition != mpiRank_) {
......@@ -760,29 +730,14 @@ namespace AMDiS {
}
}
// if(mpiRank_ == 1) {
// for(i = 0; i < recvBufferSize[0]; i++) {
// MSG("recv: %f", recvBuffer[0][i]);
// }
// }
// if(mpiRank_ == 0) {
// for(i = 0; i < sendBufferSize[1]; i++) {
// MSG("send: %f", sendBuffer[1][i]);
// }
// }
// wait for end of communication
MPI::COMM_WORLD.Barrier();
// copy values into rank solutions
for(partition = 0; partition < mpiSize_; partition++) {
if(partition != mpiRank_) {
//std::map<WorldVector<double>, DegreeOfFreedom>::iterator dofIt;
//dofIt = sortedRecvDOFs[partition].begin();
for (partition = 0; partition < mpiSize_; partition++) {
if (partition != mpiRank_) {
std::vector<DegreeOfFreedom>::iterator dofIt = recvOrder[partition].begin();
for(i = 0; i < recvBufferSize[partition]; i++) {
//(*(rankSolutions[partition]))[dofIt->second] =
for (i = 0; i < recvBufferSize[partition]; i++) {
(*(rankSolutions[partition]))[*dofIt] =
recvBuffer[partition][i];
++dofIt;
......@@ -791,9 +746,9 @@ namespace AMDiS {
}
// free send and recv buffers
for(partition = 0; partition < mpiSize_; partition++) {
if(partition != mpiRank_) {
if(sendBufferSize[partition] > 0)
for (partition = 0; partition < mpiSize_; partition++) {
if (partition != mpiRank_) {
if (sendBufferSize[partition] > 0)
FREE_MEMORY(sendBuffer[partition],
double,
sendBufferSize[partition]);
......@@ -1523,7 +1478,7 @@ namespace AMDiS {
bool openOverlap,
std::map<Element*, int> &overlapDistance)
{
int i, dim = mesh_->getDim();
int dim = mesh_->getDim();
TraverseStack stack;
ElInfo *elInfo;
......@@ -1534,25 +1489,25 @@ namespace AMDiS {
// first: partition elements ...
int index, partition;
elInfo = stack.traverseFirst(mesh_, -1, Mesh::CALL_EVERY_EL_PREORDER);
while(elInfo) {
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
if(partitionData) {
if(partitionData->getLevel() == 0) {
if (partitionData) {
if (partitionData->getLevel() == 0) {
index = element->getIndex();
partition = partitionVec_[index];
}
if(partitionData->getLevel() == level) {
for(i = 0; i < dim + 1; i++) {
vertexPartitions_[element->getDOF(i, 0)].insert(partition/* + 1*/);
if (partitionData->getLevel() == level) {
for (int i = 0; i < dim + 1; i++) {
vertexPartitions_[element->getDOF(i, 0)].insert(partition);
}
}
}
elInfo = stack.traverseNext(elInfo);
}
if(overlap > 1 || openOverlap == false) {
if (overlap > 1 || openOverlap == false) {
// exchange mesh structure codes
MeshStructure *structures = NEW MeshStructure[mpiSize_];
exchangeMeshStructureCodes(structures);
......@@ -2050,9 +2005,8 @@ namespace AMDiS {
std::vector<DOFVector<double>*> rankSol(mpiSize_);
int i, j;
for(i = 0; i < numComponents_; i++) {
for(j = 0; j < mpiSize_; j++) {
for (int i = 0; i < numComponents_; i++) {
for (int j = 0; j < mpiSize_; j++) {
rankSol[j] = rankSolution_[j]->getDOFVector(i);
}
......
AMDiS/src/ParallelProblem.h
View file @ cd501be4
......@@ -262,31 +262,130 @@ namespace AMDiS {
double errors2map(std::map<int, double> &errMap, int comp, bool add);
protected:
/** \brief
*
*/
std::string name_;
/** \brief
*
*/
Mesh *mesh_;
/** \brief
*
*/
RefinementManager *refinementManager_;
/** \brief
*
*/
CoarseningManager *coarseningManager_;
/** \brief
* Pointer to the paritioner which is used to devide a mesh into partitions.
*/
ParMetisPartitioner *partitioner_;
/** \brief
* Stores to every element index the number of the partition it corresponds to.
*/
std::map<int, int> partitionVec_;
/** \brief
* Stores an old partitioning of elements. To every element index the number of
* the parition it corresponds to is stored.
*/
std::map<int, int> oldPartitionVec_;
/** \brief
*
*/
std::map<int, double> elemWeights_;
/** \brief
*
*/
std::map<Element*, std::set<int> > elementPartitions_;
/** \brief
* Stores to every DOF the set of partitions it corresponds to.
*/
std::map<DegreeOfFreedom, std::set<int> > vertexPartitions_;
/** \brief
*
*/
int repartitionSteps_;
/** \brief
*
*/
bool puEveryTimestep_;
/** \brief
*
*/
std::vector<DOFVector<double>*> dofVectors_;
/** \brief
*
*/
double upperPartThreshold_;
/** \brief
*
*/
double lowerPartThreshold_;
/** \brief
*
*/
int globalCoarseGridLevel_;
/** \brief
*
*/
int localCoarseGridLevel_;
/** \brief
*
*/
int globalRefinements_;
/** \brief
*
*/
std::map<Element*, int> overlapDistance_;
/** \brief
*
*/
int adaptiveThresholds_;
/** \brief
*
*/
double thresholdIncFactor_;
/** \brief
*
*/
double thresholdDecFactor_;
/** \brief
*
*/
double repartTimeFactor_;
/** \brief
*
*/
double minUpperTH_;
/** \brief
*
*/
double maxLowerTH_;
};
......
AMDiS/src/PartitionElementData.h
View file @ cd501be4
......@@ -43,7 +43,7 @@ namespace AMDiS {
MEMORY_MANAGED(PartitionElementData);
inline bool isOfType(int typeID) const {
if(typeID == PARTITION_ED)
if (typeID == PARTITION_ED)
return true;
return false;
};
......@@ -85,9 +85,13 @@ namespace AMDiS {
return newObj;
};
inline std::string getTypeName() const { return "PartitionElementData"; };
inline std::string getTypeName() const {
return "PartitionElementData";
};
inline const int getTypeID() const { return PARTITION_ED; };
inline const int getTypeID() const {
return PARTITION_ED;
};
void serialize(std::ostream& out)
{
......@@ -109,17 +113,21 @@ namespace AMDiS {
status_ = status;
};
inline PartitionStatus getPartitionStatus() { return status_; };
inline PartitionStatus getPartitionStatus() {
return status_;
};
inline void setLevel(int level) {
level_ = level;
};
inline int getLevel() { return level_; };
inline int getLevel() {
return level_;
};
void descend(Element *element)
{
if(!element->isLeaf()) {
if (!element->isLeaf()) {
Element *child0 = element->getChild(0);
Element *child1 = element->getChild(1);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment