Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit a56007bd authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Bugfix in domain parallelization.

parent eb258668
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
namespace AMDiS { namespace AMDiS {
ElementFileWriter::ElementFileWriter(const std::string& name_, ElementFileWriter::ElementFileWriter(std::string name_,
const FiniteElemSpace *feSpace_, const FiniteElemSpace *feSpace_,
std::map<int, double> &mapvec) std::map<int, double> &mapvec)
: name(name_), : name(name_),
...@@ -95,14 +95,14 @@ namespace AMDiS { ...@@ -95,14 +95,14 @@ namespace AMDiS {
void ElementFileWriter::writeFile(std::map<int, double> &vec, void ElementFileWriter::writeFile(std::map<int, double> &vec,
const FiniteElemSpace *feSpace, const FiniteElemSpace *feSpace,
const std::string& filename) std::string filename)
{ {
ElementFileWriter efw("", feSpace, vec); ElementFileWriter efw("", feSpace, vec);
efw.writeVtkValues(filename); efw.writeVtkValues(filename);
} }
void ElementFileWriter::writeTecPlotValues(const std::string &filename) void ElementFileWriter::writeTecPlotValues(std::string filename)
{ {
FUNCNAME("ElementFileWriter::writeTecPlotValues()"); FUNCNAME("ElementFileWriter::writeTecPlotValues()");
std::ofstream fout(filename.c_str()); std::ofstream fout(filename.c_str());
...@@ -154,10 +154,9 @@ namespace AMDiS { ...@@ -154,10 +154,9 @@ namespace AMDiS {
// Write coordinates of all element vertices and element value. // Write coordinates of all element vertices and element value.
for (int i = 0; i <= dim; ++i) { for (int i = 0; i <= dim; ++i) {
for (int j = 0; j < dim; ++j)
for (int j = 0; j < dim; ++j) {
fout << elInfo->getCoord(i)[j] << " "; fout << elInfo->getCoord(i)[j] << " ";
}
fout << val << "\n"; fout << val << "\n";
} }
...@@ -182,7 +181,7 @@ namespace AMDiS { ...@@ -182,7 +181,7 @@ namespace AMDiS {
fout.close(); fout.close();
} }
void ElementFileWriter::writeMeshDatValues(const std::string &filename, double time) void ElementFileWriter::writeMeshDatValues(std::string filename, double time)
{ {
FUNCNAME("ElementFileWriter::writeMeshDatValues()"); FUNCNAME("ElementFileWriter::writeMeshDatValues()");
std::ofstream fout(filename.c_str()); std::ofstream fout(filename.c_str());
...@@ -281,7 +280,7 @@ namespace AMDiS { ...@@ -281,7 +280,7 @@ namespace AMDiS {
fout.close(); fout.close();
} }
void ElementFileWriter::writeVtkValues(const std::string &filename) void ElementFileWriter::writeVtkValues(std::string filename)
{ {
FUNCNAME("ElementFileWriter::writeVtkValues()"); FUNCNAME("ElementFileWriter::writeVtkValues()");
std::ofstream fout(filename.c_str()); std::ofstream fout(filename.c_str());
...@@ -289,6 +288,7 @@ namespace AMDiS { ...@@ -289,6 +288,7 @@ namespace AMDiS {
TEST_EXIT(fout)("Could not open file %s !\n", filename.c_str()); TEST_EXIT(fout)("Could not open file %s !\n", filename.c_str());
int dim = mesh->getDim(); int dim = mesh->getDim();
int dimOfWorld = Global::getGeo(WORLD);
int vertices = mesh->getGeo(VERTEX); int vertices = mesh->getGeo(VERTEX);
int nElements = mesh->getNumberOfLeaves(); int nElements = mesh->getNumberOfLeaves();
double val; double val;
...@@ -312,12 +312,12 @@ namespace AMDiS { ...@@ -312,12 +312,12 @@ namespace AMDiS {
while (elInfo) { while (elInfo) {
// Write coordinates of all element vertices. // Write coordinates of all element vertices.
for (int i = 0; i <= dim; i++) { for (int i = 0; i <= dim; i++) {
for (int j = 0; j < dim; j++) { for (int j = 0; j < dimOfWorld; j++)
fout << elInfo->getCoord(i)[j] << " "; fout << elInfo->getCoord(i)[j] << " ";
}
for (int j = dim; j < 3; j++) { for (int j = dimOfWorld; j < 3; j++)
fout << "0.0"; fout << "0.0";
}
fout << "\n"; fout << "\n";
} }
...@@ -329,9 +329,8 @@ namespace AMDiS { ...@@ -329,9 +329,8 @@ namespace AMDiS {
fout << " <Cells>" << std::endl; fout << " <Cells>" << std::endl;
fout << " <DataArray type=\"Int32\" Name=\"offsets\">" << std::endl; fout << " <DataArray type=\"Int32\" Name=\"offsets\">" << std::endl;
for (int i = 0; i < nElements; i++) { for (int i = 0; i < nElements; i++)
fout << " " << (i + 1) * vertices << std::endl; fout << " " << (i + 1) * vertices << std::endl;
}
fout << " </DataArray>" << std::endl; fout << " </DataArray>" << std::endl;
......
...@@ -21,7 +21,7 @@ namespace AMDiS { ...@@ -21,7 +21,7 @@ namespace AMDiS {
{ {
public: public:
/// Constructor. /// Constructor.
ElementFileWriter(const std::string& name, ElementFileWriter(std::string name,
const FiniteElemSpace *feSpace, const FiniteElemSpace *feSpace,
std::map<int, double> &vec); std::map<int, double> &vec);
...@@ -34,17 +34,17 @@ namespace AMDiS { ...@@ -34,17 +34,17 @@ namespace AMDiS {
/// Simple writing procedure for one element map. /// Simple writing procedure for one element map.
static void writeFile(std::map<int, double> &vec, static void writeFile(std::map<int, double> &vec,
const FiniteElemSpace *feSpace, const FiniteElemSpace *feSpace,
const std::string& filename); std::string filename);
protected: protected:
/// Writes element data in tecplot format. /// Writes element data in tecplot format.
void writeTecPlotValues(const std::string &filename); void writeTecPlotValues(std::string filename);
/// Writes element data in AMDiS format (1 file !). /// Writes element data in AMDiS format (1 file !).
void writeMeshDatValues(const std::string &filename, double time); void writeMeshDatValues(std::string filename, double time);
/// Writes element data in VTK format. /// Writes element data in VTK format.
void writeVtkValues(const std::string &filename); void writeVtkValues(std::string filename);
protected: protected:
/// Name. /// Name.
......
...@@ -35,6 +35,9 @@ namespace AMDiS { ...@@ -35,6 +35,9 @@ namespace AMDiS {
/// The macro element to which the boundary element corresponds to. /// The macro element to which the boundary element corresponds to.
Element* el; Element* el;
/// Index of the macro element.
int elIndex;
/** \brief /** \brief
* Defines the geometrical object at the boundary. It must be "a part" of the * Defines the geometrical object at the boundary. It must be "a part" of the
* macro element \ref el, i.e., either 1 (a vertex), 2 (an edge) or 3 (a face). * macro element \ref el, i.e., either 1 (a vertex), 2 (an edge) or 3 (a face).
......
...@@ -48,13 +48,12 @@ namespace AMDiS { ...@@ -48,13 +48,12 @@ namespace AMDiS {
Lagrange::~Lagrange() Lagrange::~Lagrange()
{ {
for (int i = 0; i < static_cast<int>(bary->size()); i++) { for (int i = 0; i < static_cast<int>(bary->size()); i++)
if ((*bary)[i]) { if ((*bary)[i]) {
delete (*bary)[i]; delete (*bary)[i];
(*bary)[i] = NULL; (*bary)[i] = NULL;
} }
} }
}
Lagrange* Lagrange::getLagrange(int dim, int degree) Lagrange* Lagrange::getLagrange(int dim, int degree)
{ {
...@@ -70,14 +69,13 @@ namespace AMDiS { ...@@ -70,14 +69,13 @@ namespace AMDiS {
void Lagrange::clear() void Lagrange::clear()
{ {
std::list<Lagrange*>::iterator it; for (std::list<Lagrange*>::iterator it = allBasFcts.begin();
for (it = allBasFcts.begin(); it != allBasFcts.end(); it++) { it != allBasFcts.end(); it++)
if (*it) { if (*it) {
delete *it; delete *it;
*it = NULL; *it = NULL;
} }
} }
}
void Lagrange::setFunctionPointer() void Lagrange::setFunctionPointer()
{ {
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include "ElementDofIterator.h" #include "ElementDofIterator.h"
#include "ProblemStatBase.h" #include "ProblemStatBase.h"
#include "StandardProblemIteration.h" #include "StandardProblemIteration.h"
#include "ElementFileWriter.h"
#include "petscksp.h" #include "petscksp.h"
...@@ -22,7 +23,7 @@ namespace AMDiS { ...@@ -22,7 +23,7 @@ namespace AMDiS {
PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *) PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
{ {
if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0) // if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0)
std::cout << " Iteration " << iter << ": " << rnorm << std::endl; std::cout << " Iteration " << iter << ": " << rnorm << std::endl;
return 0; return 0;
...@@ -66,6 +67,23 @@ namespace AMDiS { ...@@ -66,6 +67,23 @@ namespace AMDiS {
if (mpiSize <= 1) if (mpiSize <= 1)
return; return;
#if 0
if (mpiRank == 0) {
std::map<int, double> vec;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH);
while (elInfo) {
vec[elInfo->getElement()->getIndex()] = static_cast<double>(elInfo->getElement()->getIndex());
elInfo = stack.traverseNext(elInfo);
}
ElementFileWriter::writeFile(vec, feSpace, "test.vtu");
}
#endif
// Test, if the mesh is the macro mesh only! Paritioning of the mesh is supported // Test, if the mesh is the macro mesh only! Paritioning of the mesh is supported
// only for macro meshes, so it will not work yet if the mesh is already refined // only for macro meshes, so it will not work yet if the mesh is already refined
// in some way. // in some way.
...@@ -112,6 +130,41 @@ namespace AMDiS { ...@@ -112,6 +130,41 @@ namespace AMDiS {
updateDofAdmins(); updateDofAdmins();
#if 0
if (mpiRank == 0) {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
if (elInfo->getElement()->getIndex() == 4) {
WorldVector<double> x;
mesh->getDofIndexCoords(elInfo->getElement()->getDOF(0), feSpace, x);
std::cout << "FOUND!" << std::endl;
x.print();
}
elInfo = stack.traverseNext(elInfo);
}
}
if (mpiRank == 1) {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
if (elInfo->getElement()->getIndex() == 3) {
WorldVector<double> x;
mesh->getDofIndexCoords(elInfo->getElement()->getDOF(2), feSpace, x);
std::cout << "FOUND!" << std::endl;
x.print();
}
elInfo = stack.traverseNext(elInfo);
}
}
exit(0);
#endif
// === Global refinements. === // === Global refinements. ===
...@@ -137,7 +190,7 @@ namespace AMDiS { ...@@ -137,7 +190,7 @@ namespace AMDiS {
#if (DEBUG != 0) #if (DEBUG != 0)
DbgTestCommonDofs(); DbgTestCommonDofs(true);
#endif #endif
...@@ -293,7 +346,7 @@ namespace AMDiS { ...@@ -293,7 +346,7 @@ namespace AMDiS {
KSPSetOperators(ksp, petscMatrix, petscMatrix, DIFFERENT_NONZERO_PATTERN); KSPSetOperators(ksp, petscMatrix, petscMatrix, DIFFERENT_NONZERO_PATTERN);
KSPGetPC(ksp, &pc); KSPGetPC(ksp, &pc);
// PCSetType(pc, PCNONE); // PCSetType(pc, PCNONE);
PCSetType(pc, PCJACOBI); PCSetType(pc, PCILU);
KSPSetTolerances(ksp, 1.e-7, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT); KSPSetTolerances(ksp, 1.e-7, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT);
KSPSetType(ksp, KSPBCGS); KSPSetType(ksp, KSPBCGS);
//KSPSetType(ksp, KSPCG); //KSPSetType(ksp, KSPCG);
...@@ -555,6 +608,24 @@ namespace AMDiS { ...@@ -555,6 +608,24 @@ namespace AMDiS {
bool isRankDOF2 = (find(rankDOFs.begin(), rankDOFs.end(), boundDOF2) != rankDOFs.end()); bool isRankDOF2 = (find(rankDOFs.begin(), rankDOFs.end(), boundDOF2) != rankDOFs.end());
bool ranksBoundary = isRankDOF1 || isRankDOF2; bool ranksBoundary = isRankDOF1 || isRankDOF2;
#if 0
if (mpiRank == 3 && ranksBoundary &&
partitionVec[elInfo->getNeighbour(i)->getIndex()] == 2) {
std::cout << "ADD MY BOUND " << element->getIndex() << "/" << i
<< " with "
<< elInfo->getNeighbour(i)->getIndex() << "/"
<< elInfo->getSideOfNeighbour(i) << std::endl;
}
if (mpiRank == 2 && !ranksBoundary &&
partitionVec[elInfo->getNeighbour(i)->getIndex()] == 3) {
std::cout << "ADD OT BOUND " << element->getIndex() << "/" << i
<< " with "
<< elInfo->getNeighbour(i)->getIndex() << "/"
<< elInfo->getSideOfNeighbour(i) << std::endl;
}
#endif
// === And add the part of the interior boundary. === // === And add the part of the interior boundary. ===
AtomicBoundary& bound = AtomicBoundary& bound =
...@@ -563,11 +634,19 @@ namespace AMDiS { ...@@ -563,11 +634,19 @@ namespace AMDiS {
otherIntBoundary.getNewAtomicBoundary(partitionVec[elInfo->getNeighbour(i)->getIndex()])); otherIntBoundary.getNewAtomicBoundary(partitionVec[elInfo->getNeighbour(i)->getIndex()]));
bound.rankObject.el = element; bound.rankObject.el = element;
bound.rankObject.elIndex = element->getIndex();
bound.rankObject.subObjAtBoundary = EDGE; bound.rankObject.subObjAtBoundary = EDGE;
bound.rankObject.ithObjAtBoundary = i; bound.rankObject.ithObjAtBoundary = i;
bound.neighbourObject.el = elInfo->getNeighbour(i); // Do not set a pointer to the element, because if will be deleted from
// mesh after partitioning and the pointer would become invalid.
bound.neighbourObject.el = NULL;
bound.neighbourObject.elIndex = elInfo->getNeighbour(i)->getIndex();
bound.neighbourObject.subObjAtBoundary = EDGE; bound.neighbourObject.subObjAtBoundary = EDGE;
bound.neighbourObject.ithObjAtBoundary = -1; bound.neighbourObject.ithObjAtBoundary = elInfo->getSideOfNeighbour(i);
// i == 2 => getSideOfNeighbour(i) == 2
TEST_EXIT_DBG(i != 2 || elInfo->getSideOfNeighbour(i) == 2)
("Should not happen!\n");
} }
} }
} }
...@@ -589,23 +668,25 @@ namespace AMDiS { ...@@ -589,23 +668,25 @@ namespace AMDiS {
for (RankToBoundMap::iterator rankIt = myIntBoundary.boundary.begin(); for (RankToBoundMap::iterator rankIt = myIntBoundary.boundary.begin();
rankIt != myIntBoundary.boundary.end(); ++rankIt) { rankIt != myIntBoundary.boundary.end(); ++rankIt) {
int nSendInt = rankIt->second.size(); int nSendInt = rankIt->second.size();
int* buffer = new int[nSendInt]; int* buffer = new int[nSendInt * 2];
for (int i = 0; i < nSendInt; i++) for (int i = 0; i < nSendInt; i++) {
buffer[i] = (rankIt->second)[i].rankObject.el->getIndex(); buffer[i * 2] = (rankIt->second)[i].rankObject.elIndex;
buffer[i * 2 + 1] = (rankIt->second)[i].rankObject.ithObjAtBoundary;
}
sendBuffers.push_back(buffer); sendBuffers.push_back(buffer);
request[requestCounter++] = request[requestCounter++] =
mpiComm.Isend(buffer, nSendInt, MPI_INT, rankIt->first, 0); mpiComm.Isend(buffer, nSendInt * 2, MPI_INT, rankIt->first, 0);
} }
for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin(); for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
rankIt != otherIntBoundary.boundary.end(); ++rankIt) { rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
int nRecvInt = rankIt->second.size(); int nRecvInt = rankIt->second.size();
int *buffer = new int[nRecvInt]; int *buffer = new int[nRecvInt * 2];
recvBuffers.push_back(buffer); recvBuffers.push_back(buffer);
request[requestCounter++] = request[requestCounter++] =
mpiComm.Irecv(buffer, nRecvInt, MPI_INT, rankIt->first, 0); mpiComm.Irecv(buffer, nRecvInt * 2, MPI_INT, rankIt->first, 0);
} }
...@@ -622,10 +703,12 @@ namespace AMDiS { ...@@ -622,10 +703,12 @@ namespace AMDiS {
for (int j = 0; j < static_cast<int>(rankIt->second.size()); j++) { for (int j = 0; j < static_cast<int>(rankIt->second.size()); j++) {
// If the expected object is not at place, search for it. // If the expected object is not at place, search for it.
if ((rankIt->second)[j].neighbourObject.el->getIndex() != recvBuffers[i][j]) { if ((rankIt->second)[j].neighbourObject.elIndex != recvBuffers[i][j * 2] ||
(rankIt->second)[j].neighbourObject.ithObjAtBoundary != recvBuffers[i][j * 2 + 1]) {
int k = j + 1; int k = j + 1;
for (; k < static_cast<int>(rankIt->second.size()); k++) for (; k < static_cast<int>(rankIt->second.size()); k++)
if ((rankIt->second)[k].neighbourObject.el->getIndex() == recvBuffers[i][j]) if ((rankIt->second)[k].neighbourObject.elIndex == recvBuffers[i][j * 2] &&
(rankIt->second)[k].neighbourObject.ithObjAtBoundary == recvBuffers[i][j * 2 + 1])
break; break;
// The element must always be found, because the list is just in another order. // The element must always be found, because the list is just in another order.
...@@ -790,6 +873,11 @@ namespace AMDiS { ...@@ -790,6 +873,11 @@ namespace AMDiS {
sendBuffers[i][c++] = *(dofIt->first); sendBuffers[i][c++] = *(dofIt->first);
sendBuffers[i][c++] = dofIt->second; sendBuffers[i][c++] = dofIt->second;
#if 0
if (mpiRank == 3 && sendIt->first == 2)
std::cout << "SEND DOF: " << dofIt->first << std::endl;
#endif
sendDofs[sendIt->first].push_back(dofIt->first); sendDofs[sendIt->first].push_back(dofIt->first);
} }
...@@ -852,6 +940,11 @@ namespace AMDiS { ...@@ -852,6 +940,11 @@ namespace AMDiS {
if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) { if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true; dofChanged[dofIt->first] = true;
#if 0
if (mpiRank == 2 && recvIt->first == 3)
std::cout << "RECV DOF: " << dofIt->first << std::endl;
#endif
recvDofs[recvIt->first].push_back(dofIt->first); recvDofs[recvIt->first].push_back(dofIt->first);
rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof; rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof;
isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false; isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false;
...@@ -897,8 +990,15 @@ namespace AMDiS { ...@@ -897,8 +990,15 @@ namespace AMDiS {
} }
DofContainer rankAllDofs; DofContainer rankAllDofs;
for (DofSet::iterator dofIt = rankDOFSet.begin(); dofIt != rankDOFSet.end(); ++dofIt) for (DofSet::iterator dofIt = rankDOFSet.begin(); dofIt != rankDOFSet.end(); ++dofIt) {
/* if (mpiRank == 1) {
std::cout << "COORDs of dof = " << **dofIt << std::endl;
WorldVector<double> x;
mesh->getDofIndexCoords(*dofIt, feSpace, x);
x.print();
}*/
rankAllDofs.push_back(*dofIt); rankAllDofs.push_back(*dofIt);
}
sort(rankAllDofs.begin(), rankAllDofs.end(), cmpDofsByValue); sort(rankAllDofs.begin(), rankAllDofs.end(), cmpDofsByValue);
DofContainer rankDOFs = rankAllDofs; DofContainer rankDOFs = rankAllDofs;
...@@ -915,6 +1015,16 @@ namespace AMDiS { ...@@ -915,6 +1015,16 @@ namespace AMDiS {
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
#if 0
if (mpiRank == 3 && it->first == 2)
std::cout << "GO ON MY BOUND: " << boundIt->rankObject.elIndex
<< "/" << boundIt->rankObject.ithObjAtBoundary << " with "
<< boundIt->neighbourObject.elIndex << "/"
<< boundIt->neighbourObject.ithObjAtBoundary
<< std::endl;
#endif
DofContainer dofs; DofContainer dofs;
DofContainer &dofsToSend = sendDofs[it->first]; DofContainer &dofsToSend = sendDofs[it->first];
...@@ -935,40 +1045,70 @@ namespace AMDiS { ...@@ -935,40 +1045,70 @@ namespace AMDiS {
ERROR_EXIT("Should never happen!\n"); ERROR_EXIT("Should never happen!\n");
} }
for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt) { #if 0
if (mpiRank == 3 && it->first == 2) {
WorldVector<double> x;
mesh->getDofIndexCoords(dofs[0], feSpace, x);
x.print();
mesh->getDofIndexCoords(dofs[1], feSpace, x);
x.print();
}
#endif
for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt)
if (find(dofsToSend.begin(), dofsToSend.end(), *dofIt) == dofsToSend.end()) if (find(dofsToSend.begin(), dofsToSend.end(), *dofIt) == dofsToSend.end())
dofsToSend.push_back(*dofIt); dofsToSend.push_back(*dofIt);
}
dofs.clear(); dofs.clear();
addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary, addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofs); dofs);
addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary, addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofs); dofs);
for (int i = 0; i < static_cast<int>(dofs.size()); i++) { for (int i = 0; i < static_cast<int>(dofs.size()); i++)
dofsToSend.push_back(dofs[i]); dofsToSend.push_back(dofs[i]);
} }
}
}