Commit 7856d906 authored by Thomas Witkowski's avatar Thomas Witkowski

Periodic boundary conditions with mixed finite elements in parallel should work now.

parent 6bfde321
......@@ -57,7 +57,7 @@ namespace AMDiS {
FUNCNAME("DOFMatrix::DOFMatrix()");
TEST_EXIT(rowFeSpace)("No fe space for row!\n");
if (!colFeSpace)
colFeSpace = rowFeSpace;
......@@ -211,16 +211,18 @@ namespace AMDiS {
using namespace mtl;
#if 0
std::cout << "----- PRINT MAT " << rowElInfo->getElement()->getIndex() << "--------" << std::endl;
std::cout << elMat << std::endl;
std::cout << "rows: ";
for (int i = 0; i < rowIndices.size(); i++)
std::cout << rowIndices[i] << " ";
std::cout << std::endl;
std::cout << "cols: ";
for (int i = 0; i < colIndices.size(); i++)
std::cout << colIndices[i] << " ";
std::cout << std::endl;
if (MPI::COMM_WORLD.Get_rank() == 0) {
std::cout << "----- PRINT MAT " << rowElInfo->getElement()->getIndex() << "--------" << std::endl;
std::cout << elMat << std::endl;
std::cout << "rows: ";
for (int i = 0; i < rowIndices.size(); i++)
std::cout << rowIndices[i] << " ";
std::cout << std::endl;
std::cout << "cols: ";
for (int i = 0; i < colIndices.size(); i++)
std::cout << colIndices[i] << " ";
std::cout << std::endl;
}
#endif
for (int i = 0; i < nRow; i++) {
......@@ -229,7 +231,7 @@ namespace AMDiS {
BoundaryCondition *condition =
bound ? boundaryManager->getBoundaryCondition(bound[i]) : NULL;
if (condition && condition->isDirichlet()) {
if (condition && condition->isDirichlet()) {
if (condition->applyBoundaryCondition()) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if ((*rankDofs)[rowIndices[i]])
......@@ -241,8 +243,9 @@ namespace AMDiS {
} else {
for (int j = 0; j < nCol; j++) {
DegreeOfFreedom col = colIndices[j];
if (fabs(elMat[i][j]) > 1e-10)
if (fabs(elMat[i][j]) > 1e-10) {
ins[row][col] += elMat[i][j];
}
}
}
}
......@@ -259,7 +262,9 @@ namespace AMDiS {
{}
void DOFMatrix::assemble(double factor, ElInfo *elInfo, const BoundaryType *bound)
void DOFMatrix::assemble(double factor,
ElInfo *elInfo,
const BoundaryType *bound)
{
FUNCNAME("DOFMatrix::assemble()");
......@@ -278,7 +283,9 @@ namespace AMDiS {
}
void DOFMatrix::assemble(double factor, ElInfo *elInfo, const BoundaryType *bound,
void DOFMatrix::assemble(double factor,
ElInfo *elInfo,
const BoundaryType *bound,
Operator *op)
{
FUNCNAME("DOFMatrix::assemble()");
......
......@@ -1356,6 +1356,10 @@ namespace AMDiS {
{
FUNCNAME("ProblemStat::addMatrixOperator()");
TEST_EXIT(i < nComponents && j < nComponents)
("Cannot add matrix operator at position %d/%d. The stationary problem has only %d components!\n",
i, j, nComponents);
TEST_EXIT(!boundaryConditionSet)
("Do not add operators after boundary conditions were set!\n");
......@@ -1402,6 +1406,10 @@ namespace AMDiS {
{
FUNCNAME("ProblemStat::addVectorOperator()");
TEST_EXIT(i < nComponents)
("Cannot add vector operator at position %d. The stationary problem has only %d components!\n",
i, nComponents);
TEST_EXIT(!boundaryConditionSet)
("Do not add operators after boundary conditions were set!\n");
......
......@@ -64,17 +64,13 @@ namespace AMDiS {
/// Destructor
virtual ~SubAssembler() {}
/** \brief
* Calculates the element matrix for elInfo and adds it to mat. Memory
* for mat must be provided by the caller.
*/
/// Calculates the element matrix for elInfo and adds it to mat. Memory for
/// mat must be provided by the caller.
virtual void calculateElementMatrix(const ElInfo *elInfo,
ElementMatrix& mat) = 0;
/** \brief
* Calculates the element vector for elInfo and adds it to vec. Memory
* for vec must be provided by the caller.
*/
/// Calculates the element vector for elInfo and adds it to vec. Memory for
/// vec must be provided by the caller.
virtual void calculateElementVector(const ElInfo *elInfo,
ElementVector& vec) = 0;
......@@ -104,10 +100,8 @@ namespace AMDiS {
WorldVector<double>* getCoordsAtQPs(const ElInfo* elInfo,
Quadrature *quad = NULL);
/** \brief
* DOFVector dv evaluated at quadrature points.
* Used by \ref OperatorTerm::initElement().
*/
/// DOFVector dv evaluated at quadrature points.
/// Used by \ref OperatorTerm::initElement().
template<typename T>
void getVectorAtQPs(DOFVectorBase<T>* dv,
const ElInfo* elInfo,
......@@ -122,10 +116,8 @@ namespace AMDiS {
Quadrature *quad,
mtl::dense_vector<T>& vecAtQPs);
/** \brief
* Gradients of DOFVector dv evaluated at quadrature points.
* Used by \ref OperatorTerm::initElement().
*/
/// Gradients of DOFVector dv evaluated at quadrature points.
/// Used by \ref OperatorTerm::initElement().
template<typename T>
void getGradientsAtQPs( DOFVectorBase<T>* dv,
const ElInfo* elInfo,
......@@ -139,11 +131,10 @@ namespace AMDiS {
Quadrature *quad,
mtl::dense_vector<typename GradientType<T>::type>& grdAtQPs);
/** \brief
* The comp'th component of the derivative of DOFVector dv evaluated at quadrature points.
* Used by \ref OperatorTerm::initElement().
* Attention: not caching at the moment! Using cache if gradients for read but not for write
*/
/// The comp'th component of the derivative of DOFVector dv evaluated at
/// quadrature points. Used by \ref OperatorTerm::initElement().
/// Attention: not caching at the moment! Using cache if gradients for read
/// but not for write.
template<typename T>
void getDerivativeAtQPs(DOFVectorBase<T>* dv,
const ElInfo* elInfo,
......@@ -159,11 +150,9 @@ namespace AMDiS {
int comp,
mtl::dense_vector<T>& grdAtQPs);
/** \brief
* Called once for each ElInfo when \ref calculateElementMatrix() or
* \ref calculateElementVector() is called for the first time for this
* Element.
*/
/// Called once for each ElInfo when \ref calculateElementMatrix() or
/// \ref calculateElementVector() is called for the first time for this
/// Element.
virtual void initElement(const ElInfo *smallElInfo,
const ElInfo *largeElInfo = NULL,
Quadrature *quad = NULL);
......@@ -202,16 +191,12 @@ namespace AMDiS {
/// Column FiniteElemSpace.
const FiniteElemSpace *colFeSpace;
/** \brief
* Number of rows of the element matrix and length of the element
* vector. Is equal to the number of row basis functions
*/
/// Number of rows of the element matrix and length of the element
/// vector. Is equal to the number of row basis functions
int nRow;
/** \brief
* Number of columns of the element matrix. Is equal to the number
* of column basis functions
*/
/// Number of columns of the element matrix. Is equal to the number
/// of column basis functions
int nCol;
/// Used for \ref getVectorAtQPs() and \ref getGradientsAtQPs().
......@@ -230,16 +215,15 @@ namespace AMDiS {
std::map<const void*, ValuesAtQPs* > cachedValuesAtQPs;
std::map<const void*, ValuesAtQPs* > cachedGradientsAtQPs;
/** \brief
* Set and updated by \ref initElement() for each ElInfo.
* coordsAtQPs[i] points to the coordinates of the i-th quadrature point.
*/
/// Set and updated by \ref initElement() for each ElInfo.
/// coordsAtQPs[i] points to the coordinates of the i-th quadrature point.
WorldVector<double> *coordsAtQPs;
/// Used for \ref getCoordsAtQPs().
bool coordsValid;
/// Used for \ref getCoordsAtQP(). Stores the number of allocated WorldVectors.
/// Used for \ref getCoordsAtQP(). Stores the number of allocated
/// WorldVectors.
int coordsNumAllocated;
/// Quadrature object to be used for assembling.
......
......@@ -376,6 +376,48 @@ namespace AMDiS {
return;
break;
case EDGE:
{
// === Create boundary information objects for children elements. ===
BoundaryObject nextBound0 = bound;
prepareNextBound(nextBound0, 0);
BoundaryObject nextBound1 = bound;
prepareNextBound(nextBound1, 1);
// === Check for boundary on children elements. ===
if ((nextBound0.ithObj >= 0 || nextBound1.ithObj >= 0) && child[0]) {
// So, the edge is contained in at least on of the children and the
// element is also refined. Then we have go down further in refinement
// hierarchie.
if (bound.reverseMode) {
if (nextBound1.ithObj >= 0)
child[1]->getHigherOrderDofs(feSpace, nextBound1, dofs);
if (nextBound0.ithObj >= 0)
child[0]->getHigherOrderDofs(feSpace, nextBound0, dofs);
} else {
if (nextBound0.ithObj >= 0)
child[0]->getHigherOrderDofs(feSpace, nextBound0, dofs);
if (nextBound1.ithObj >= 0)
child[1]->getHigherOrderDofs(feSpace, nextBound1, dofs);
}
} else {
// Either the edge is not contained in further refined children, or
// the element is not refined further on this edge. Then we can get
// all the DOFs on this edge.
ElementDofIterator elDofIter(feSpace, true);
elDofIter.reset(this);
do {
if (elDofIter.getCurrentPos() == 1 &&
elDofIter.getCurrentElementPos() == bound.ithObj)
dofs.push_back(elDofIter.getDofPtr());
} while(elDofIter.next());
}
}
break;
case FACE:
{
......@@ -403,10 +445,8 @@ namespace AMDiS {
elDofIter.reset(this);
do {
if (elDofIter.getCurrentPos() == 2 &&
elDofIter.getCurrentElementPos() == bound.ithObj) {
ERROR_EXIT("Check this, if it will really work!\n");
elDofIter.getCurrentElementPos() == bound.ithObj)
dofs.push_back(elDofIter.getDofPtr());
}
} while(elDofIter.next());
}
}
......
......@@ -324,10 +324,10 @@ namespace AMDiS {
static const int sideOfChild[3][2][4];
/** \brief
* edgeOfChild[elType][i][j] is the local edge number of the j-th edge within the
* i-th children of an element of elType. If the value is -1, the edge is not
* included in the element's child. Note that the 0 edge is included in both
* children only by its half.
* edgeOfChild[elType][i][j] is the local edge number of the j-th edge within
* the i-th children of an element of elType. If the value is -1, the edge is
* not included in the element's child. Note that the 0 edge is included in
* both children only by its half.
*/
static const int edgeOfChild[3][2][6];
......
......@@ -169,7 +169,7 @@ namespace AMDiS {
break;
default:
ERROR_EXIT("Should never happen!\n");
}
}
}
......
......@@ -77,9 +77,9 @@ namespace AMDiS {
newAssembler = new StandardZOA(op, assembler, quad);
} else {
if (pwConst)
newAssembler = new PrecalcZOA(op, assembler, quad);
newAssembler = new PrecalcZOA(op, assembler, quad);
else
newAssembler = new FastQuadZOA(op, assembler, quad);
newAssembler = new FastQuadZOA(op, assembler, quad);
}
subAssemblers->push_back(newAssembler);
......
......@@ -85,9 +85,12 @@ namespace AMDiS {
periodicFaces[make_pair(face0, face1)] = elInfo->getBoundary(i);
/// Add all three vertices of the face to be periodic.
periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] = boundaryType;
periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] = boundaryType;
periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] = boundaryType;
periodicVertices[make_pair(face0.get<0>(), face1.get<0>())] =
boundaryType;
periodicVertices[make_pair(face0.get<1>(), face1.get<1>())] =
boundaryType;
periodicVertices[make_pair(face0.get<2>(), face1.get<2>())] =
boundaryType;
periodicDofAssoc[face0.get<0>()].insert(boundaryType);
periodicDofAssoc[face0.get<1>()].insert(boundaryType);
......@@ -131,8 +134,8 @@ namespace AMDiS {
TEST_EXIT_DBG(mesh)("Mesh not set!\n");
// === Return, if there are no periodic vertices, i.e., there are no no ===
// === periodic boundaries in the mesh. ===
// === Return, if there are no periodic vertices, i.e., there are no ===
// === periodic boundaries in the mesh. ===
if (periodicVertices.size() == 0)
return;
......@@ -141,9 +144,9 @@ namespace AMDiS {
// === Get all vertex DOFs that have multiple periodic associations. ===
// We group all vertices together, that have either two or three periodic
// associations. For rectangular domains in 2D, the four corner vertices have all
// two periodic associations. For box domains in 3D, the eight corner vertices
// have all three periodic associations.
// associations. For rectangular domains in 2D, the four corner vertices have
// all two periodic associations. For box domains in 3D, the eight corner
// vertices have all three periodic associations.
vector<DegreeOfFreedom> multPeriodicDof2, multPeriodicDof3;
for (map<DegreeOfFreedom, std::set<BoundaryType> >::iterator it = periodicDofAssoc.begin();
......
......@@ -126,11 +126,11 @@ namespace AMDiS {
/** \brief
* Creates final data of the periodic boundaries. Must be called after all
* elements of the mesh are added to the object database. Then this functions
* search for interectly connected vertices in periodic boundaries. This is only
* the case, if there are more than one boundary conditions. Then, e.g., in 2D,
* all edges of a square are iterectly connected. In 3D, if the macro mesh is a
* box, all eight vertex nodes and always four of the 12 edges are iterectly
* connected.
* search for indirectly connected vertices in periodic boundaries. This is
* only the case, if there are more than one boundary conditions. Then, e.g.,
* in 2D, all edges of a square are iterectly connected. In 3D, if the macro
* mesh is a box, all eight vertex nodes and always four of the 12 edges are
* indirectly connected.
*/
void createPeriodicData(const FiniteElemSpace *feSpace);
......
......@@ -289,17 +289,17 @@ namespace AMDiS {
// We have to remove the VertexVectors, which contain periodic assoiciations,
// because they are not valid anymore after some macro elements have been removed
// and the corresponding DOFs were deleted.
// because they are not valid anymore after some macro elements have been
// removed and the corresponding DOFs were deleted.
for (map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
it != mesh->getPeriodicAssociations().end(); ++it)
const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
updateLocalGlobalNumbering();
// === In 3D we have to make some test, if the resulting mesh is valid. If ===
// === it is not valid, there is no possiblity yet to fix this problem, just ===
// === exit with an error message. ===
// === In 3D we have to make some test, if the resulting mesh is valid. ===
// === If it is not valid, there is no possiblity yet to fix this ===
// === problem, just exit with an error message. ===
check3dValidMesh();
......@@ -320,7 +320,7 @@ namespace AMDiS {
// === Create periodic DOF mapping, if there are periodic boundaries. ===
createPeriodicMap(feSpaces[0]);
createPeriodicMap();
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
......@@ -335,25 +335,20 @@ namespace AMDiS {
refineManager->globalRefine(mesh, globalRefinement);
updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(feSpaces[0]);
createPeriodicMap();
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
#endif
}
/// === Set DOF rank information to all matrices and vectors. ===
// Set DOF rank information to all matrices and vectors.
setRankDofs();
// === Remove periodic boundary conditions in sequential problem definition. ===
// Remove periodic boundary conditions in sequential problem definition.
removePeriodicBoundaryConditions();
initialized = true;
......@@ -364,6 +359,9 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::addProblemStat()");
TEST_EXIT_DBG(probStat->getFeSpaces().size())
("No FE spaces in stationary problem!\n");
// === Add FE spaces from stationary problem to mesh distributor. ===
for (unsigned int i = 0; i < probStat->getFeSpaces().size(); i++) {
......@@ -845,21 +843,20 @@ namespace AMDiS {
#endif
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
// Because the mesh has been changed, update the DOF numbering and mappings.
updateLocalGlobalNumbering();
// Update periodic mapping, if there are periodic boundaries.
createPeriodicMap();
// === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(feSpaces[0]);
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
#endif
// === The mesh has changed, so check if it is required to repartition the mesh. ===
// === The mesh has changed, so check if it is required to repartition ===
// === the mesh. ===
nMeshChangesAfterLastRepartitioning++;
......@@ -900,7 +897,8 @@ namespace AMDiS {
maxDofs = std::max(maxDofs, nDofsInRank[i]);
}
int avrgDofs = nOverallDofs / mpiSize;
double imbalance = (static_cast<double>(maxDofs - avrgDofs) / avrgDofs) * 100.0;
double imbalance =
(static_cast<double>(maxDofs - avrgDofs) / avrgDofs) * 100.0;
MSG("Imbalancing factor: %.1f\%\n", imbalance);
}
......@@ -1202,7 +1200,8 @@ namespace AMDiS {
it != newMacroEl.end(); ++it) {
MacroElement *mel = *it;
// First, reset all neighbour relations. The correct neighbours will be set later.
// First, reset all neighbour relations. The correct neighbours will be
// set later.
for (int i = 0; i < mesh->getGeo(NEIGH); i++)
mel->setNeighbour(i, NULL);
......@@ -1259,7 +1258,8 @@ namespace AMDiS {
stdMpi2.startCommunication();
// === Adapte the new macro elements due to the received mesh structure codes. ===
// === Adapte the new macro elements due to the received mesh ===
// === structure codes. ===
for (map<int, vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it) {
......@@ -1348,7 +1348,7 @@ namespace AMDiS {
// === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(feSpaces[0]);
createPeriodicMap();
#if (DEBUG != 0)
......@@ -1435,25 +1435,21 @@ namespace AMDiS {
macroElIndexMap[el->getIndex()] = el;
macroElIndexTypeMap[el->getIndex()] = elInfo->getType();
// === Add all sub object of the element to the variable elObjects. ===
// Add all sub object of the element to the variable elObjects.
elObjects.addElement(elInfo);
elInfo = stack.traverseNext(elInfo);
}
// === Create periodic data, if there are periodic boundary conditions. ===
if (elObjects.hasPeriodicData()) {
TEST_EXIT(feSpaces.size() == 1)
("Sebastian: Na, dass funktioniert auch noch nicht mit mehreren FE spaces. Du weisst schon, wen du jetzt mobben kannst :)!\n");
}
// Create periodic data, if there are periodic boundary conditions.
elObjects.createPeriodicData(feSpaces[0]);
// === Create data about the reverse modes of neighbouring elements. ===
// Create data about the reverse modes of neighbouring elements.
elObjects.createReverseModeData(feSpaces[0], macroElIndexMap,
macroElIndexTypeMap);
// === Create mesh element data for this rank. ===
// Create mesh element data for this rank.
elObjects.createRankData(partitionMap);
}
......@@ -1482,7 +1478,8 @@ namespace AMDiS {
int owner = elObjects.getIterateOwner();
ElementObjectData& rankBoundEl = objData[mpiRank];
TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])("Should not happen!\n");
TEST_EXIT_DBG(macroElIndexMap[rankBoundEl.elIndex])
("Should not happen!\n");
AtomicBoundary bound;
bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex];
......@@ -1555,12 +1552,8 @@ namespace AMDiS {
if (elObjects.isInRank(it->first.first, mpiRank) == false)
continue;
TEST_EXIT(feSpaces.size() == 1)("Does not work for multiple fe spaces!\n");
WorldVector<double> c0, c1;
mesh->getDofIndexCoords(it->first.first, feSpaces[0], c0);
mesh->getDofIndexCoords(it->first.second, feSpaces[0], c1);
ElementObjectData& perDofEl0 = elObjects.getElementsInRank(it->first.first)[mpiRank];
ElementObjectData& perDofEl0 =
elObjects.getElementsInRank(it->first.first)[mpiRank];
for (map<int, ElementObjectData>::iterator elIt = elObjects.getElementsInRank(it->first.second).begin();
elIt != elObjects.getElementsInRank(it->first.second).end(); ++elIt) {
......@@ -1665,16 +1658,19 @@ namespace AMDiS {
b = bound;
if (mpiRank > otherElementRank)
b.neighObj.reverseMode = elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
b.neighObj.reverseMode =
elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
else
b.rankObj.reverseMode = elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
b.rankObj.reverseMode =
elObjects.getFaceReverseMode(perFaceEl0, perFaceEl1);
}
}
// === Once we have this information, we must care about the order of the atomic ===
// === bounds in the three boundary handling object. Eventually all the bound- ===
// === aries have to be in the same order on both ranks that share the bounday. ===
// === Once we have this information, we must care about the order of the ===
// === atomic bounds in the three boundary handling object. Eventually ===
// === all the boundaries have to be in the same order on both ranks that ===
// === share the bounday. ===
StdMpi<vector<AtomicBoundary> > stdMpi(mpiComm);
stdMpi.send(myIntBoundary.boundary);
......@@ -1682,16 +1678,17 @@ namespace AMDiS {
stdMpi.startCommunication();
// === The information about all neighbouring boundaries has been received. So ===
// === the rank tests if its own atomic boundaries are in the same order. If ===
// === not, the atomic boundaries are swaped to the correct order. ===
// === The information about all neighbouring boundaries has been ===
// === received. So the rank tests if its own atomic boundaries are in ===
// === the same order. If not, the atomic boundaries are swaped to the ===
// === correct order. ===
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
// === We have received from rank "rankIt->first" the ordered list of element ===
// === indices. Now, we have to sort the corresponding list in this rank to ===
// === get the same order. ===
// === We have received from rank "rankIt->first" the ordered list of ===
// === element indices. Now, we have to sort the corresponding list in ===
// === this rank to get the same order. ===
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
......@@ -1706,7 +1703,8 @@ namespace AMDiS {
if ((rankIt->second)[k].neighObj == recvedBound)
break;
// The element must always be found, because the list is just in another order.
// The element must always be found, because the list is just in
// another order.
TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n");
// Swap the current with the found element.
......@@ -1747,8 +1745,10 @@ namespace AMDiS {
continue;
for (unsigned int j = 0; j < rankIt->second.size(); j++) {
BoundaryObject &recvRankObj = stdMpi.getRecvData()[rankIt->first][j].rankObj;
BoundaryObject &recvNeighObj = stdMpi.getRecvData()[rankIt->first][j].neighObj;
BoundaryObject &recvRankObj =
stdMpi.getRecvData()[rankIt->first][j].rankObj;
BoundaryObject &recvNeighObj =
stdMpi.getRecvData()[rankIt->first][j].neighObj;
if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvRankObj ||
periodicBoundary.boundary[rankIt->first][j].rankObj != recvNeighObj) {
......@@ -1827,16 +1827,13 @@ namespace AMDiS {
}
}
} else {
for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it) {
for (InteriorBoundary::iterator it(myIntBoundary); !it.end(); ++it)
it->rankObj.el->getAllDofs(feSpace, it->rankObj,
sendDofs.getDofCont(it.getRank(), feSpace));
}
for (InteriorBoundary::iterator it(otherIntBoundary);