Commit 30c3b864 authored by Thomas Witkowski's avatar Thomas Witkowski

Some code refactoring and fixed some small problems in mesh repartitioning.

parent 8a79f686
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=1572864
max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
......@@ -100,6 +100,7 @@
#include "io/MacroWriter.h"
#include "io/PngWriter.h"
#include "io/PovrayWriter.h"
#include "io/ValueReader.h"
#include "io/ValueWriter.h"
#include "io/VtkWriter.h"
......
......@@ -258,7 +258,7 @@ namespace AMDiS {
}
void DOFAdmin::compress(std::vector<DegreeOfFreedom> &new_dof)
void DOFAdmin::compress(std::vector<DegreeOfFreedom> &newDofIndex)
{
FUNCNAME("DOFAdmin::compress()");
......@@ -268,17 +268,17 @@ namespace AMDiS {
// vector to mark used dofs
for (int i = 0; i < size; i++)
new_dof[i] = -1;
newDofIndex[i] = -1;
// mark used dofs
DOFIteratorBase it(this, USED_DOFS);
for (it.reset(); !it.end(); ++it)
new_dof[it.getDOFIndex()] = 1;
newDofIndex[it.getDOFIndex()] = 1;
int n = 0, last = 0;
for (int i = 0; i < size; i++) { /* create a MONOTONE compress */
if (new_dof[i] == 1) {
new_dof[i] = n++;
if (newDofIndex[i] == 1) {
newDofIndex[i] = n++;
last = i;
}
}
......@@ -300,21 +300,19 @@ namespace AMDiS {
// get index of first changed dof
int first = last;
for (int i = 0; i < size; i++) {
if (new_dof[i] < i && new_dof[i] >= 0) {
if (newDofIndex[i] < i && newDofIndex[i] >= 0) {
first = i;
break;
}
}
for (std::list<DOFIndexedBase*>::iterator di = dofIndexedList.begin();
di != dofIndexedList.end(); ++di)
(*di)->compressDOFIndexed(first, last, newDofIndex);
std::list<DOFIndexedBase*>::iterator di;
std::list<DOFIndexedBase*>::iterator end = dofIndexedList.end();
for (di = dofIndexedList.begin(); di != end; ++di)
(*di)->compressDOFIndexed(first, last, new_dof);
std::list<DOFContainer*>::iterator dc;
std::list<DOFContainer*>::iterator endc = dofContainerList.end();
for (dc = dofContainerList.begin(); dc != endc; ++dc)
(*dc)->compressDofContainer(n, new_dof);
for (std::list<DOFContainer*>::iterator dc = dofContainerList.begin();
dc != dofContainerList.end(); ++dc)
(*dc)->compressDofContainer(n, newDofIndex);
}
......
......@@ -102,7 +102,7 @@ namespace AMDiS {
* compression. This method is usually called after mesh adaption involving
* higher order elements or coarsening.
*/
void compress(std::vector<DegreeOfFreedom> &new_dof);
void compress(std::vector<DegreeOfFreedom> &newDofIndex);
/// Returns an iterator to the begin of \ref dofIndexedList
std::list<DOFIndexedBase*>::iterator beginDOFIndexed()
......
......@@ -56,10 +56,11 @@ namespace AMDiS {
for (int i = 0; i < size; i++) {
int j = newDOF[operator[](i)];
if (j >= 0)
operator[](i) = j;
else
ERROR_EXIT("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
TEST_EXIT_DBG(j >= 0)
("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
operator[](i) = j;
}
}
......
......@@ -198,7 +198,7 @@ namespace AMDiS {
/* should be used only at the end of dof_compress()!!!!! */
/****************************************************************************/
void Element::newDofFct1(const DOFAdmin* admin)
void Element::newDofFct1(const DOFAdmin* admin, std::vector<int>& newDofIndex)
{
int n0, nd, nd0;
......@@ -207,7 +207,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(VERTEX);
n0 = admin->getMesh()->getNode(VERTEX);
for (int i = 0; i < vertices; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
if (mesh->getDim() > 1) {
......@@ -216,7 +216,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(EDGE);
n0 = admin->getMesh()->getNode(EDGE);
for (int i = 0; i < edges; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
......@@ -226,14 +226,14 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(FACE);
n0 = admin->getMesh()->getNode(FACE);
for (int i = 0; i < faces; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
if ((nd = admin->getNumberOfDofs(CENTER))) {
nd0 = admin->getNumberOfPreDOFs(CENTER);
n0 = admin->getMesh()->getNode(CENTER);
changeDofs1(admin, n0, nd0, nd, 0);
changeDofs1(admin, newDofIndex, n0, nd0, nd, 0);
}
}
......@@ -283,13 +283,14 @@ namespace AMDiS {
}
void Element::changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos)
void Element::changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
int n0, int nd0, int nd, int pos)
{
DegreeOfFreedom *ldof = dof[n0 + pos] + nd0;
for (int j = 0; j < nd; j++) {
int k = ldof[j];
if (k >= 0)
ldof[j] = -admin->getMesh()->newDOF[k] - 1;
ldof[j] = -newDofIndex[k] - 1;
}
}
......
......@@ -521,13 +521,14 @@ namespace AMDiS {
}
/// Used by friend class Mesh while dofCompress
void newDofFct1(const DOFAdmin*);
void newDofFct1(const DOFAdmin*, std::vector<int>&);
/// Used by friend class Mesh while dofCompress
void newDofFct2(const DOFAdmin*);
/// Changes old dofs to negative new dofs
void changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos);
void changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
int n0, int nd0, int nd, int pos);
/// Changes negative new dofs to positive
void changeDofs2(int n0, int nd0, int nd, int pos);
......
......@@ -160,7 +160,6 @@ namespace AMDiS {
nDof = m.nDof;
nNodeEl = m.nNodeEl;
node = m.node;
newDOF = m.newDOF;
elementIndex = m.elementIndex;
initialized = m.initialized;
final_lambda = m.final_lambda;
......@@ -271,7 +270,8 @@ namespace AMDiS {
TEST_EXIT(admin.size() == 1)("Not yet implemented for multiple admins!\n");
TEST_EXIT(admin[0])("There is something wrong!\n");
// === Determine to all dofs the macro elements where the dof is part of. ===
// === Determine to all DOFs in mesh the macro elements where the DOF ===
// === is part of. ===
// Map that stores for each dof pointer (which may have a list of dofs)
// all macro element indices that own this dof.
......@@ -463,9 +463,8 @@ namespace AMDiS {
compressAdmin->getHoleCount() < 1)
continue;
newDOF.resize(size);
compressAdmin->compress(newDOF);
std::vector<int> newDofIndex(size);
compressAdmin->compress(newDofIndex);
Flag fill_flag = (preserveCoarseDOFs ?
Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_NOTHING :
......@@ -473,7 +472,7 @@ namespace AMDiS {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, fill_flag);
while (elInfo) {
elInfo->getElement()->newDofFct1(compressAdmin);
elInfo->getElement()->newDofFct1(compressAdmin, newDofIndex);
elInfo = stack.traverseNext(elInfo);
}
......@@ -482,8 +481,6 @@ namespace AMDiS {
elInfo->getElement()->newDofFct2(compressAdmin);
elInfo = stack.traverseNext(elInfo);
}
newDOF.resize(0);
}
}
......
......@@ -808,9 +808,6 @@ namespace AMDiS {
/// List of all MacroElements of this Mesh
std::deque<MacroElement*> macroElements;
/// Needed during DOF compression (\ref DOFAdmin::compress).
std::vector<DegreeOfFreedom> newDOF;
/// Used by check functions
static std::vector<DegreeOfFreedom> dof_used;
......@@ -880,9 +877,6 @@ namespace AMDiS {
friend class MacroWriter;
friend class MacroElement;
friend class Element;
friend void Element::newDofFct1(const DOFAdmin*);
friend void Element::newDofFct2(const DOFAdmin*);
friend void Element::changeDofs1(const DOFAdmin*, int, int, int, int);
};
}
......
......@@ -56,8 +56,7 @@ namespace AMDiS {
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
lastMeshChangeIndex(0),
macroElementStructureConsisten(false)
lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
......@@ -139,36 +138,38 @@ namespace AMDiS {
#endif
// === Create new global and local DOF numbering. ===
// createLocalGlobalNumbering();
// === Remove all macro elements that are not part of the rank partition. ===
removeMacroElements();
macroElementStructureConsisten = true;
updateLocalGlobalNumbering(true);
// === Create new global and local DOF numbering. ===
// === Reset all DOFAdmins of the mesh. ===
// We have to remove the VertexVectors, which contain periodic assoiciations,
// because they are not valid anymore after some macro elements have been removed
// and the corresponding DOFs were deleted.
for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
it != mesh->getPeriodicAssociations().end(); ++it)
const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
updateDofAdmins();
updateLocalGlobalNumbering();
// === If in debug mode, make some tests. ===
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
MSG("Debug mode tests finished!\n");
debug::writeMesh(feSpace, -1, "macro_mesh");
MSG("Debug mode tests finished!\n");
#endif
......@@ -188,8 +189,7 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, "gr_mesh");
#endif
mesh->dofCompress();
updateLocalGlobalNumbering(false);
updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -305,30 +305,6 @@ namespace AMDiS {
{}
void MeshDistributor::updateDofAdmins()
{
FUNCNAME("MeshDistributor::updateDofAdmins()");
for (int i = 0; i < mesh->getNumberOfDOFAdmin(); i++) {
DOFAdmin& admin = const_cast<DOFAdmin&>(mesh->getDofAdmin(i));
// There must be always more allocated DOFs than used DOFs in DOFAdmin. Otherwise,
// it is not possible to define a first DOF hole.
if (static_cast<unsigned int>(admin.getSize()) == mapLocalGlobalDofs.size())
admin.enlargeDOFLists();
for (int j = 0; j < admin.getSize(); j++)
admin.setDOFFree(j, true);
for (unsigned int j = 0; j < mapLocalGlobalDofs.size(); j++)
admin.setDOFFree(j, false);
admin.setUsedSize(mapLocalGlobalDofs.size());
admin.setUsedCount(mapLocalGlobalDofs.size());
admin.setFirstHole(mapLocalGlobalDofs.size());
}
}
void MeshDistributor::testForMacroMesh()
{
FUNCNAME("MeshDistributor::testForMacroMesh()");
......@@ -475,9 +451,6 @@ namespace AMDiS {
}
// Remove periodic vertex associations
for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
it != mesh->getPeriodicAssociations().end(); ++it)
const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
mesh->getPeriodicAssociations().clear();
}
......@@ -564,8 +537,8 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
mesh->dofCompress();
updateLocalGlobalNumbering(false);
updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -1125,11 +1098,14 @@ namespace AMDiS {
mesh->removeMacroElements(deleteMacroElements, feSpace);
if (mpiRank == 0)
debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
// === Remove double DOFs. ===
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
mesh->dofCompress();
DOFVector<double> tmp(feSpace, "tmp");
......@@ -1137,7 +1113,6 @@ namespace AMDiS {
VtkWriter::writeFile(tmp, "after-repartition.vtu");
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
exit(0);
#if (DEBUG != 0)
ParallelDebug::testAllElements(*this);
......@@ -1146,6 +1121,24 @@ namespace AMDiS {
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
updateLocalGlobalNumbering();
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
debug::writeMesh(feSpace, -1, "macro_mesh");
MSG("Debug mode tests finished!\n");
#endif
MSG("DONE\n");
}
......@@ -1573,184 +1566,12 @@ namespace AMDiS {
}
void MeshDistributor::createLocalGlobalNumbering()
{
FUNCNAME("MeshDistributor::createLocalGlobalNumbering()");
// === Get rank information about DOFs. ===
// Stores to each DOF pointer the set of ranks the DOF is part of.
DofToPartitions partitionDofs;
DofContainer rankDofs, rankAllDofs;
DofToRank boundaryDofs;
createDofMemberInfo(partitionDofs, rankDofs, rankAllDofs, boundaryDofs);
nRankDofs = rankDofs.size();
nOverallDofs = partitionDofs.size();
// === Get starting position for global rank dof ordering. ====
rstart = 0;
mpiComm.Scan(&nRankDofs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDofs;
// === Create for all dofs in rank new indices. The new index must start at ===
// === index 0, must be continues and have the same order as the indices ===
// === had before. ===
// Do not change the indices now, but create a new indexing and store it here.
DofIndexMap rankDofsNewLocalIndex;
isRankDof.clear();
int i = 0;
for (DofContainer::iterator dofIt = rankAllDofs.begin();
dofIt != rankAllDofs.end(); ++dofIt) {
rankDofsNewLocalIndex[*dofIt] = i;
// First, we set all dofs in ranks partition to be owend by the rank. Later,
// the dofs in ranks partition that are owned by other rank are set to false.
isRankDof[i] = true;
i++;
}
// === Create for all rank owned dofs a new global indexing. ===
// Stores for dofs in rank a new global index.
DofIndexMap rankDofsNewGlobalIndex;
// Stores for all rank owned dofs a continues local index.
DofIndexMap rankOwnedDofsNewLocalIndex;
i = 0;
for (DofContainer::iterator dofIt = rankDofs.begin();
dofIt != rankDofs.end(); ++dofIt) {
rankDofsNewGlobalIndex[*dofIt] = i + rstart;
rankOwnedDofsNewLocalIndex[*dofIt] = i;
i++;
}
// === Create information which dof indices must be send and which must ===
// === be received to/from other ranks. ===
// Stores to each rank a map from DOF pointers (which are all owned by the rank
// and lie on an interior boundary) to new global DOF indices.
std::map<int, DofIndexMap> sendNewDofs;
// Maps to each rank the number of new DOF indices this rank will receive from.
// All these DOFs are at an interior boundary on this rank, but are owned by
// another rank.
std::map<int, int> recvNewDofs;
for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
if (it->second == mpiRank) {
// If the boundary dof is a rank dof, it must be send to other ranks.
// Search for all ranks that have this dof too.
for (std::set<int>::iterator itRanks = partitionDofs[it->first].begin();
itRanks != partitionDofs[it->first].end();
++itRanks) {
if (*itRanks != mpiRank) {
TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1)
("DOF Key not found!\n");
sendNewDofs[*itRanks][it->first] = rankDofsNewGlobalIndex[it->first];
}
}
} else {
// If the boundary dof is not a rank dof, its new dof index (and later
// also the dof values) must be received from another rank.
if (recvNewDofs.find(it->second) == recvNewDofs.end())
recvNewDofs[it->second] = 1;
else
recvNewDofs[it->second]++;
}
}
// === Send and receive the dof indices at boundary. ===
sendDofs.clear();
for (std::map<int, DofIndexMap>::iterator sendIt = sendNewDofs.begin();
sendIt != sendNewDofs.end(); ++sendIt)
for (DofIndexMap::iterator dofIt = sendIt->second.begin();
dofIt != sendIt->second.end(); ++dofIt)
sendDofs[sendIt->first].push_back(dofIt->first);
typedef std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > DofMapVec;
StdMpi<DofIndexMap, DofMapVec> stdMpi(mpiComm);
stdMpi.send(sendNewDofs);
for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt, i++)
stdMpi.recv(recvIt->first, recvIt->second * 2);
stdMpi.startCommunication<int>(MPI_INT);
std::map<int, DofMapVec> &dofMap = stdMpi.getRecvData();
// === Change global dof indices at boundary from other ranks. ===
// Within this small data structure we track which dof index was already changed.
// This is used to avoid the following situation: Assume, there are two dof indices
// a and b in boundaryDofs. Then we have to change index a to b and b to c. When
// the second rule applies, we have to avoid that not the first b, resulted from
// changing a to b, is set to c, but the second one. Therefore, after the first
// rule was applied, the dof pointer is set to false in this data structure and
// is not allowed to be changed anymore.
std::map<const DegreeOfFreedom*, bool> dofChanged;
recvDofs.clear();
for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
++dofIt)
dofChanged[dofIt->first] = false;
for (std::map<int, DofMapVec>::iterator rankIt = dofMap.begin();
rankIt != dofMap.end(); ++rankIt) {
for (DofMapVec::iterator recvDofIt = rankIt->second.begin();
recvDofIt != rankIt->second.end(); ++recvDofIt) {
DegreeOfFreedom oldDof = recvDofIt->first;
DegreeOfFreedom newGlobalDof = recvDofIt->second;
bool found = false;
// Iterate over all boundary dofs to find the dof which index we have to change.
for (DofToRank::iterator dofIt = boundaryDofs.begin();
dofIt != boundaryDofs.end(); ++dofIt) {
if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true;
recvDofs[rankIt->first].push_back(dofIt->first);
rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof;
isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false;
found = true;
break;