Skip to content

GitLab

A
amdis
Commit 30c3b864 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

Some code refactoring and fixed some small problems in mesh repartitioning.

parent 8a79f686
Hide whitespace changes
Inline Side-by-side
Showing with 103 additions and 460 deletions
AMDiS/libtool
View file @ 30c3b864
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=1572864
max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
AMDiS/src/AMDiS.h
View file @ 30c3b864
......@@ -100,6 +100,7 @@
#include "io/MacroWriter.h"
#include "io/PngWriter.h"
#include "io/PovrayWriter.h"
#include "io/ValueReader.h"
#include "io/ValueWriter.h"
#include "io/VtkWriter.h"
......
AMDiS/src/DOFAdmin.cc
View file @ 30c3b864
......@@ -258,7 +258,7 @@ namespace AMDiS {
}
void DOFAdmin::compress(std::vector<DegreeOfFreedom> &new_dof)
void DOFAdmin::compress(std::vector<DegreeOfFreedom> &newDofIndex)
{
FUNCNAME("DOFAdmin::compress()");
......@@ -268,17 +268,17 @@ namespace AMDiS {
// vector to mark used dofs
for (int i = 0; i < size; i++)
new_dof[i] = -1;
newDofIndex[i] = -1;
// mark used dofs
DOFIteratorBase it(this, USED_DOFS);
for (it.reset(); !it.end(); ++it)
new_dof[it.getDOFIndex()] = 1;
newDofIndex[it.getDOFIndex()] = 1;
int n = 0, last = 0;
for (int i = 0; i < size; i++) { /* create a MONOTONE compress */
if (new_dof[i] == 1) {
new_dof[i] = n++;
if (newDofIndex[i] == 1) {
newDofIndex[i] = n++;
last = i;
}
}
......@@ -300,21 +300,19 @@ namespace AMDiS {
// get index of first changed dof
int first = last;
for (int i = 0; i < size; i++) {
if (new_dof[i] < i && new_dof[i] >= 0) {
if (newDofIndex[i] < i && newDofIndex[i] >= 0) {
first = i;
break;
}
}
for (std::list<DOFIndexedBase*>::iterator di = dofIndexedList.begin();
di != dofIndexedList.end(); ++di)
(*di)->compressDOFIndexed(first, last, newDofIndex);
std::list<DOFIndexedBase*>::iterator di;
std::list<DOFIndexedBase*>::iterator end = dofIndexedList.end();
for (di = dofIndexedList.begin(); di != end; ++di)
(*di)->compressDOFIndexed(first, last, new_dof);
std::list<DOFContainer*>::iterator dc;
std::list<DOFContainer*>::iterator endc = dofContainerList.end();
for (dc = dofContainerList.begin(); dc != endc; ++dc)
(*dc)->compressDofContainer(n, new_dof);
for (std::list<DOFContainer*>::iterator dc = dofContainerList.begin();
dc != dofContainerList.end(); ++dc)
(*dc)->compressDofContainer(n, newDofIndex);
}
......
AMDiS/src/DOFAdmin.h
View file @ 30c3b864
......@@ -102,7 +102,7 @@ namespace AMDiS {
* compression. This method is usually called after mesh adaption involving
* higher order elements or coarsening.
*/
void compress(std::vector<DegreeOfFreedom> &new_dof);
void compress(std::vector<DegreeOfFreedom> &newDofIndex);
/// Returns an iterator to the begin of \ref dofIndexedList
std::list<DOFIndexedBase*>::iterator beginDOFIndexed()
......
AMDiS/src/DOFContainer.h
View file @ 30c3b864
......@@ -56,10 +56,11 @@ namespace AMDiS {
for (int i = 0; i < size; i++) {
int j = newDOF[operator[](i)];
if (j >= 0)
operator[](i) = j;
else
ERROR_EXIT("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
TEST_EXIT_DBG(j >= 0)
("Invalid DOF %d in DOF container! (%d %d)\n", j, i, operator[](i));
operator[](i) = j;
}
}
......
AMDiS/src/Element.cc
View file @ 30c3b864
......@@ -198,7 +198,7 @@ namespace AMDiS {
/* should be used only at the end of dof_compress()!!!!! */
/****************************************************************************/
void Element::newDofFct1(const DOFAdmin* admin)
void Element::newDofFct1(const DOFAdmin* admin, std::vector<int>& newDofIndex)
{
int n0, nd, nd0;
......@@ -207,7 +207,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(VERTEX);
n0 = admin->getMesh()->getNode(VERTEX);
for (int i = 0; i < vertices; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
if (mesh->getDim() > 1) {
......@@ -216,7 +216,7 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(EDGE);
n0 = admin->getMesh()->getNode(EDGE);
for (int i = 0; i < edges; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
......@@ -226,14 +226,14 @@ namespace AMDiS {
nd0 = admin->getNumberOfPreDOFs(FACE);
n0 = admin->getMesh()->getNode(FACE);
for (int i = 0; i < faces; i++)
changeDofs1(admin, n0, nd0, nd, i);
changeDofs1(admin, newDofIndex, n0, nd0, nd, i);
}
}
if ((nd = admin->getNumberOfDofs(CENTER))) {
nd0 = admin->getNumberOfPreDOFs(CENTER);
n0 = admin->getMesh()->getNode(CENTER);
changeDofs1(admin, n0, nd0, nd, 0);
changeDofs1(admin, newDofIndex, n0, nd0, nd, 0);
}
}
......@@ -283,13 +283,14 @@ namespace AMDiS {
}
void Element::changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos)
void Element::changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
int n0, int nd0, int nd, int pos)
{
DegreeOfFreedom *ldof = dof[n0 + pos] + nd0;
for (int j = 0; j < nd; j++) {
int k = ldof[j];
if (k >= 0)
ldof[j] = -admin->getMesh()->newDOF[k] - 1;
ldof[j] = -newDofIndex[k] - 1;
}
}
......
AMDiS/src/Element.h
View file @ 30c3b864
......@@ -521,13 +521,14 @@ namespace AMDiS {
}
/// Used by friend class Mesh while dofCompress
void newDofFct1(const DOFAdmin*);
void newDofFct1(const DOFAdmin*, std::vector<int>&);
/// Used by friend class Mesh while dofCompress
void newDofFct2(const DOFAdmin*);
/// Changes old dofs to negative new dofs
void changeDofs1(const DOFAdmin* admin, int n0, int nd0, int nd, int pos);
void changeDofs1(const DOFAdmin* admin, std::vector<int>& newDofIndex,
int n0, int nd0, int nd, int pos);
/// Changes negative new dofs to positive
void changeDofs2(int n0, int nd0, int nd, int pos);
......
AMDiS/src/Mesh.cc
View file @ 30c3b864
......@@ -160,7 +160,6 @@ namespace AMDiS {
nDof = m.nDof;
nNodeEl = m.nNodeEl;
node = m.node;
newDOF = m.newDOF;
elementIndex = m.elementIndex;
initialized = m.initialized;
final_lambda = m.final_lambda;
......@@ -271,7 +270,8 @@ namespace AMDiS {
TEST_EXIT(admin.size() == 1)("Not yet implemented for multiple admins!\n");
TEST_EXIT(admin[0])("There is something wrong!\n");
// === Determine to all dofs the macro elements where the dof is part of. ===
// === Determine to all DOFs in mesh the macro elements where the DOF ===
// === is part of. ===
// Map that stores for each dof pointer (which may have a list of dofs)
// all macro element indices that own this dof.
......@@ -463,9 +463,8 @@ namespace AMDiS {
compressAdmin->getHoleCount() < 1)
continue;
newDOF.resize(size);
compressAdmin->compress(newDOF);
std::vector<int> newDofIndex(size);
compressAdmin->compress(newDofIndex);
Flag fill_flag = (preserveCoarseDOFs ?
Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_NOTHING :
......@@ -473,7 +472,7 @@ namespace AMDiS {
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, fill_flag);
while (elInfo) {
elInfo->getElement()->newDofFct1(compressAdmin);
elInfo->getElement()->newDofFct1(compressAdmin, newDofIndex);
elInfo = stack.traverseNext(elInfo);
}
......@@ -482,8 +481,6 @@ namespace AMDiS {
elInfo->getElement()->newDofFct2(compressAdmin);
elInfo = stack.traverseNext(elInfo);
}
newDOF.resize(0);
}
}
......
AMDiS/src/Mesh.h
View file @ 30c3b864
......@@ -808,9 +808,6 @@ namespace AMDiS {
/// List of all MacroElements of this Mesh
std::deque<MacroElement*> macroElements;
/// Needed during DOF compression (\ref DOFAdmin::compress).
std::vector<DegreeOfFreedom> newDOF;
/// Used by check functions
static std::vector<DegreeOfFreedom> dof_used;
......@@ -880,9 +877,6 @@ namespace AMDiS {
friend class MacroWriter;
friend class MacroElement;
friend class Element;
friend void Element::newDofFct1(const DOFAdmin*);
friend void Element::newDofFct2(const DOFAdmin*);
friend void Element::changeDofs1(const DOFAdmin*, int, int, int, int);
};
}
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 30c3b864
......@@ -56,8 +56,7 @@ namespace AMDiS {
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
lastMeshChangeIndex(0),
macroElementStructureConsisten(false)
lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
......@@ -139,36 +138,38 @@ namespace AMDiS {
#endif
// === Create new global and local DOF numbering. ===
// createLocalGlobalNumbering();
// === Remove all macro elements that are not part of the rank partition. ===
removeMacroElements();
macroElementStructureConsisten = true;
updateLocalGlobalNumbering(true);
// === Create new global and local DOF numbering. ===
// === Reset all DOFAdmins of the mesh. ===
// We have to remove the VertexVectors, which contain periodic assoiciations,
// because they are not valid anymore after some macro elements have been removed
// and the corresponding DOFs were deleted.
for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
it != mesh->getPeriodicAssociations().end(); ++it)
const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
updateDofAdmins();
updateLocalGlobalNumbering();
// === If in debug mode, make some tests. ===
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
MSG("Debug mode tests finished!\n");
debug::writeMesh(feSpace, -1, "macro_mesh");
MSG("Debug mode tests finished!\n");
#endif
......@@ -188,8 +189,7 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, "gr_mesh");
#endif
mesh->dofCompress();
updateLocalGlobalNumbering(false);
updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -305,30 +305,6 @@ namespace AMDiS {
{}
void MeshDistributor::updateDofAdmins()
{
FUNCNAME("MeshDistributor::updateDofAdmins()");
for (int i = 0; i < mesh->getNumberOfDOFAdmin(); i++) {
DOFAdmin& admin = const_cast<DOFAdmin&>(mesh->getDofAdmin(i));
// There must be always more allocated DOFs than used DOFs in DOFAdmin. Otherwise,
// it is not possible to define a first DOF hole.
if (static_cast<unsigned int>(admin.getSize()) == mapLocalGlobalDofs.size())
admin.enlargeDOFLists();
for (int j = 0; j < admin.getSize(); j++)
admin.setDOFFree(j, true);
for (unsigned int j = 0; j < mapLocalGlobalDofs.size(); j++)
admin.setDOFFree(j, false);
admin.setUsedSize(mapLocalGlobalDofs.size());
admin.setUsedCount(mapLocalGlobalDofs.size());
admin.setFirstHole(mapLocalGlobalDofs.size());
}
}
void MeshDistributor::testForMacroMesh()
{
FUNCNAME("MeshDistributor::testForMacroMesh()");
......@@ -475,9 +451,6 @@ namespace AMDiS {
}
// Remove periodic vertex associations
for (std::map<BoundaryType, VertexVector*>::iterator it = mesh->getPeriodicAssociations().begin();
it != mesh->getPeriodicAssociations().end(); ++it)
const_cast<DOFAdmin&>(mesh->getDofAdmin(0)).removeDOFContainer(dynamic_cast<DOFContainer*>(it->second));
mesh->getPeriodicAssociations().clear();
}
......@@ -564,8 +537,8 @@ namespace AMDiS {
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
mesh->dofCompress();
updateLocalGlobalNumbering(false);
updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
......@@ -1125,11 +1098,14 @@ namespace AMDiS {
mesh->removeMacroElements(deleteMacroElements, feSpace);
if (mpiRank == 0)
debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
// === Remove double DOFs. ===
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
mesh->dofCompress();
DOFVector<double> tmp(feSpace, "tmp");
......@@ -1137,7 +1113,6 @@ namespace AMDiS {
VtkWriter::writeFile(tmp, "after-repartition.vtu");
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
exit(0);
#if (DEBUG != 0)
ParallelDebug::testAllElements(*this);
......@@ -1146,6 +1121,24 @@ namespace AMDiS {
partitioner->fillCoarsePartitionVec(&partitionVec);
updateInteriorBoundaryInfo();
updateLocalGlobalNumbering();
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
debug::writeMesh(feSpace, -1, "macro_mesh");
MSG("Debug mode tests finished!\n");
#endif
MSG("DONE\n");
}
......@@ -1573,184 +1566,12 @@ namespace AMDiS {
}
void MeshDistributor::createLocalGlobalNumbering()
{
FUNCNAME("MeshDistributor::createLocalGlobalNumbering()");
// === Get rank information about DOFs. ===
// Stores to each DOF pointer the set of ranks the DOF is part of.
DofToPartitions partitionDofs;
DofContainer rankDofs, rankAllDofs;
DofToRank boundaryDofs;
createDofMemberInfo(partitionDofs, rankDofs, rankAllDofs, boundaryDofs);
nRankDofs = rankDofs.size();
nOverallDofs = partitionDofs.size();
// === Get starting position for global rank dof ordering. ====
rstart = 0;
mpiComm.Scan(&nRankDofs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDofs;
// === Create for all dofs in rank new indices. The new index must start at ===
// === index 0, must be continues and have the same order as the indices ===
// === had before. ===
// Do not change the indices now, but create a new indexing and store it here.
DofIndexMap rankDofsNewLocalIndex;
isRankDof.clear();
int i = 0;
for (DofContainer::iterator dofIt = rankAllDofs.begin();
dofIt != rankAllDofs.end(); ++dofIt) {
rankDofsNewLocalIndex[*dofIt] = i;
// First, we set all dofs in ranks partition to be owend by the rank. Later,
// the dofs in ranks partition that are owned by other rank are set to false.
isRankDof[i] = true;
i++;
}
// === Create for all rank owned dofs a new global indexing. ===
// Stores for dofs in rank a new global index.
DofIndexMap rankDofsNewGlobalIndex;
// Stores for all rank owned dofs a continues local index.
DofIndexMap rankOwnedDofsNewLocalIndex;
i = 0;
for (DofContainer::iterator dofIt = rankDofs.begin();
dofIt != rankDofs.end(); ++dofIt) {
rankDofsNewGlobalIndex[*dofIt] = i + rstart;
rankOwnedDofsNewLocalIndex[*dofIt] = i;
i++;
}
// === Create information which dof indices must be send and which must ===
// === be received to/from other ranks. ===
// Stores to each rank a map from DOF pointers (which are all owned by the rank
// and lie on an interior boundary) to new global DOF indices.
std::map<int, DofIndexMap> sendNewDofs;
// Maps to each rank the number of new DOF indices this rank will receive from.
// All these DOFs are at an interior boundary on this rank, but are owned by
// another rank.
std::map<int, int> recvNewDofs;
for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
if (it->second == mpiRank) {
// If the boundary dof is a rank dof, it must be send to other ranks.
// Search for all ranks that have this dof too.
for (std::set<int>::iterator itRanks = partitionDofs[it->first].begin();
itRanks != partitionDofs[it->first].end();
++itRanks) {
if (*itRanks != mpiRank) {
TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1)
("DOF Key not found!\n");
sendNewDofs[*itRanks][it->first] = rankDofsNewGlobalIndex[it->first];
}
}
} else {
// If the boundary dof is not a rank dof, its new dof index (and later
// also the dof values) must be received from another rank.
if (recvNewDofs.find(it->second) == recvNewDofs.end())
recvNewDofs[it->second] = 1;
else
recvNewDofs[it->second]++;
}
}
// === Send and receive the dof indices at boundary. ===
sendDofs.clear();
for (std::map<int, DofIndexMap>::iterator sendIt = sendNewDofs.begin();
sendIt != sendNewDofs.end(); ++sendIt)
for (DofIndexMap::iterator dofIt = sendIt->second.begin();
dofIt != sendIt->second.end(); ++dofIt)
sendDofs[sendIt->first].push_back(dofIt->first);
typedef std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > DofMapVec;
StdMpi<DofIndexMap, DofMapVec> stdMpi(mpiComm);
stdMpi.send(sendNewDofs);
for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt, i++)
stdMpi.recv(recvIt->first, recvIt->second * 2);
stdMpi.startCommunication<int>(MPI_INT);
std::map<int, DofMapVec> &dofMap = stdMpi.getRecvData();
// === Change global dof indices at boundary from other ranks. ===
// Within this small data structure we track which dof index was already changed.
// This is used to avoid the following situation: Assume, there are two dof indices
// a and b in boundaryDofs. Then we have to change index a to b and b to c. When
// the second rule applies, we have to avoid that not the first b, resulted from
// changing a to b, is set to c, but the second one. Therefore, after the first
// rule was applied, the dof pointer is set to false in this data structure and
// is not allowed to be changed anymore.
std::map<const DegreeOfFreedom*, bool> dofChanged;
recvDofs.clear();
for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
++dofIt)
dofChanged[dofIt->first] = false;
for (std::map<int, DofMapVec>::iterator rankIt = dofMap.begin();
rankIt != dofMap.end(); ++rankIt) {
for (DofMapVec::iterator recvDofIt = rankIt->second.begin();
recvDofIt != rankIt->second.end(); ++recvDofIt) {
DegreeOfFreedom oldDof = recvDofIt->first;
DegreeOfFreedom newGlobalDof = recvDofIt->second;
bool found = false;
// Iterate over all boundary dofs to find the dof which index we have to change.
for (DofToRank::iterator dofIt = boundaryDofs.begin();
dofIt != boundaryDofs.end(); ++dofIt) {
if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true;
recvDofs[rankIt->first].push_back(dofIt->first);
rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof;
isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false;
found = true;
break;