Commit 11b3b152 authored by Thomas Witkowski's avatar Thomas Witkowski

FETI-DP uses subdomains consisting of multiple ranks. Still cannot run a solve on these systems.

parent 45e59970
......@@ -46,7 +46,7 @@ namespace AMDiS {
{
FUNCNAME("DofComm::Iterator::setNextFeMap()");
if (dataIter != dofComm.data[0].end()) {
if (dataIter != dofComm.data[traverseLevel].end()) {
TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
feMapIter = dataIter->second.begin();
......
......@@ -78,19 +78,26 @@ namespace AMDiS {
const FiniteElemSpace *fe = NULL)
: dofComm(dc),
dofCounter(-1),
traverseFeSpace(fe)
traverseFeSpace(fe),
traverseLevel(0)
{
FUNCNAME("DofComm::Iterator::Iterator()");
dataIter = dofComm.data[0].begin();
goFirst();
}
while (setNextFeMap() == false)
++dataIter;
Iterator(DofComm &dc,
int level,
const FiniteElemSpace *fe = NULL)
: dofComm(dc),
dofCounter(-1),
traverseFeSpace(fe),
traverseLevel(level)
{
goFirst();
}
inline bool end()
{
return (dataIter == dofComm.data[0].end());
return (dataIter == dofComm.data[traverseLevel].end());
}
inline void nextRank()
......@@ -181,6 +188,14 @@ namespace AMDiS {
}
protected:
void goFirst()
{
dataIter = dofComm.data[traverseLevel].begin();
while (setNextFeMap() == false)
++dataIter;
}
bool setNextFeMap();
protected:
......@@ -195,6 +210,8 @@ namespace AMDiS {
int dofCounter;
const FiniteElemSpace *traverseFeSpace;
int traverseLevel;
};
......
......@@ -216,14 +216,18 @@ namespace AMDiS {
TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");
TEST_EXIT_DBG(level == 1)("Only 2-level method supported!\n");
int rankInLevel = levelData->mapRank(mapIt->first, level - 1, level);
MSG("rankInLevel %d\n", rankInLevel);
}
while (mapIt->second.size() == 0) {
++mapIt;
if (mapIt == bound.boundary.end())
return;
while (levelData->rankInSubdomain(mapIt->first, level) ||
mapIt->second.size() == 0) {
++mapIt;
if (mapIt == bound.boundary.end())
return;
}
} else {
while (mapIt->second.size() == 0) {
++mapIt;
if (mapIt == bound.boundary.end())
return;
}
}
}
......
......@@ -665,14 +665,15 @@ namespace AMDiS {
void MeshDistributor::getAllBoundaryDofs(const FiniteElemSpace *feSpace,
int level,
DofContainer& dofs)
{
FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
DofContainerSet dofSet;
for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
for (DofComm::Iterator it(sendDofs, level, feSpace); !it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end());
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
dofSet.insert(it.getDofs().begin(), it.getDofs().end());
dofs.clear();
......@@ -1036,23 +1037,6 @@ namespace AMDiS {
}
void MeshDistributor::createBoundaryDofs(const FiniteElemSpace *feSpace,
std::set<DegreeOfFreedom> &boundaryDofs)
{
FUNCNAME("MeshDistributor::createBoundaryDofs()");
boundaryDofs.clear();
for (DofComm::Iterator it(sendDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
boundaryDofs.insert(it.getDofIndex());
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
boundaryDofs.insert(it.getDofIndex());
}
void MeshDistributor::serialize(ostream &out, DofContainer &data)
{
int vecSize = data.size();
......
......@@ -373,11 +373,6 @@ namespace AMDiS {
return initialized;
}
/// Creates a set of all DOFs that are on interior boundaries of rank's
/// domain. Thus, it creates the union of \ref sendDofs and \ref recvDofs.
void createBoundaryDofs(const FiniteElemSpace *feSpace,
std::set<DegreeOfFreedom> &boundaryDofs);
// Writes all data of this object to an output stream.
void serialize(ostream &out);
......@@ -436,7 +431,8 @@ namespace AMDiS {
createBoundaryDofFlag = flag;
}
BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace, int level = 0)
BoundaryDofInfo& getBoundaryDofInfo(const FiniteElemSpace *feSpace,
int level = 0)
{
FUNCNAME("MeshDistributor::getBoundaryDofInfo()");
......@@ -447,7 +443,8 @@ namespace AMDiS {
return boundaryDofInfo[level][feSpace];
}
void getAllBoundaryDofs(const FiniteElemSpace *feSpace,
void getAllBoundaryDofs(const FiniteElemSpace *feSpace,
int level,
DofContainer& dofs);
const ElementObjectDatabase& getElementObjectDb()
......
......@@ -98,6 +98,13 @@ namespace AMDiS {
return toRank;
}
bool rankInSubdomain(int rank, int level)
{
TEST_EXIT_DBG(level < nLevel)("Should not happen!\n");
return static_cast<bool>(levelRanks[level].count(rank));
}
protected:
int nLevel;
......
......@@ -63,8 +63,8 @@ namespace AMDiS {
/** \brief
* This function is used for debugging only. It traverses all interior boundaries
* and compares the DOF indices on them with the dof indices of the boundarys
* neighbours. The function fails, when dof indices on an interior boundary do
* and compares the DOF indices on them with the DOF indices of the boundarys
* neighbours. The function fails, when DOF indices on an interior boundary do
* not fit together.
*
* \param[in] pdb Parallel problem definition used for debugging.
......
......@@ -10,6 +10,7 @@
// See also license.opensource.txt in the distribution.
#include "AMDiS.h"
#include "parallel/PetscSolverFeti.h"
#include "parallel/PetscSolverFetiStructs.h"
#include "parallel/StdMpi.h"
......@@ -315,7 +316,6 @@ namespace AMDiS {
DofIndexSet primals;
DofContainerSet& vertices =
meshDistributor->getBoundaryDofInfo(feSpace, meshLevel).geoDofs[VERTEX];
TEST_EXIT_DBG(vertices.size())("No primal vertices on this rank!\n");
for (DofContainerSet::iterator it = vertices.begin();
it != vertices.end(); ++it)
primals.insert(**it);
......@@ -339,7 +339,7 @@ namespace AMDiS {
// === Create global index of the dual nodes on each rank. ===
DofContainer allBoundaryDofs;
meshDistributor->getAllBoundaryDofs(feSpace, allBoundaryDofs);
meshDistributor->getAllBoundaryDofs(feSpace, meshLevel, allBoundaryDofs);
for (DofContainer::iterator it = allBoundaryDofs.begin();
it != allBoundaryDofs.end(); ++it)
......@@ -357,7 +357,7 @@ namespace AMDiS {
boundaryDofRanks[feSpace].clear();
for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
for (DofComm::Iterator it(meshDistributor->getSendDofs(), meshLevel, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) {
if (!isPrimal(feSpace, it.getDofIndex())) {
......@@ -372,20 +372,23 @@ namespace AMDiS {
StdMpi<vector<std::set<int> > > stdMpi(meshDistributor->getMpiComm());
for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
for (DofComm::Iterator it(meshDistributor->getSendDofs(), meshLevel, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (!isPrimal(feSpace, it.getDofIndex()))
if (!isPrimal(feSpace, it.getDofIndex())) {
MSG("SEND TO RANK %d\n", it.getRank());
stdMpi.getSendData(it.getRank()).push_back(boundaryDofRanks[feSpace][it.getDofIndex()]);
}
stdMpi.updateSendDataSize();
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), meshLevel, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (!isPrimal(feSpace, it.getDofIndex())) {
recvFromRank = true;
MSG("RECV FROM RANK %d\n", it.getRank());
break;
}
}
......@@ -396,7 +399,7 @@ namespace AMDiS {
stdMpi.startCommunication();
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), meshLevel, feSpace);
!it.end(); it.nextRank()) {
int i = 0;
for (; !it.endDofIter(); it.nextDof())
......@@ -405,7 +408,6 @@ namespace AMDiS {
stdMpi.getRecvData(it.getRank())[i++];
}
// === Reserve for each dual node, on the rank that owns this node, the ===
// === appropriate number of Lagrange constraints. ===
......
......@@ -190,7 +190,7 @@ namespace AMDiS {
/// in this map. Is used for the Dirichlet preconditioner only.
ParallelDofMapping interiorDofMap;
/// Stores to each dual boundary DOF in each finite elment space the set of
/// Stores to each dual boundary DOF in each FE space the set of
/// ranks in which the DOF is contained in.
map<const FiniteElemSpace*, DofIndexToPartitions> boundaryDofRanks;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment