Commit 9ea77d93 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

And try to generalize the new parallel data structure also to 3d.

parent 3da16af9
...@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B" ...@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s" LN_S="ln -s"
# What is the maximum length of a command? # What is the maximum length of a command?
max_cmd_len=1572864 max_cmd_len=98304
# Object file suffix (normally "o"). # Object file suffix (normally "o").
objext=o objext=o
...@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib" ...@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds="" old_postuninstall_cmds=""
# A C compiler. # A C compiler.
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
...@@ -233,10 +233,10 @@ finish_eval="" ...@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes hardcode_into_libs=yes
# Compile-time system search path for libraries. # Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib" sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries. # Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby " sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported. # Whether dlopen is supported.
dlopen_support=unknown dlopen_support=unknown
...@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64" ...@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib" old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler. # A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler? # Is the compiler the GNU compiler?
with_gcc=yes with_gcc=yes
...@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64" ...@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib" old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler. # A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler? # Is the compiler the GNU compiler?
with_gcc=yes with_gcc=yes
...@@ -9039,17 +9039,17 @@ file_list_spec="" ...@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate hardcode_action=immediate
# The directories searched by this compiler when creating a shared library. # The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.." compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to # Dependencies to place before and after the objects being linked to
# create a shared library. # create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o" predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o" postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps="" predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s" postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking # The library search path used internally by the compiler when linking
# a shared library. # a shared library.
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.." compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX # ### END LIBTOOL TAG CONFIG: CXX
...@@ -198,7 +198,10 @@ namespace AMDiS { ...@@ -198,7 +198,10 @@ namespace AMDiS {
virtual int getEdgeOfFace(int face, int edge) const = 0; virtual int getEdgeOfFace(int face, int edge) const = 0;
/// ///
virtual GlobalEdge getEdge(int localEdgeIndex) const = 0; virtual DofEdge getEdge(int localEdgeIndex) const = 0;
///
virtual DofFace getFace(int localFaceIndex) const = 0;
/// Returns the number of parts of type i in this element /// Returns the number of parts of type i in this element
virtual int getGeo(GeoIndex i) const = 0; virtual int getGeo(GeoIndex i) const = 0;
......
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
#include "mpi.h" #include "mpi.h"
#endif #endif
#include "boost/tuple/tuple.hpp"
#include "AMDiS_fwd.h" #include "AMDiS_fwd.h"
namespace AMDiS { namespace AMDiS {
...@@ -78,8 +79,11 @@ namespace AMDiS { ...@@ -78,8 +79,11 @@ namespace AMDiS {
/// Defines type for a vector of DOF pointers. /// Defines type for a vector of DOF pointers.
typedef std::vector<const DegreeOfFreedom*> DofContainer; typedef std::vector<const DegreeOfFreedom*> DofContainer;
/// Defines a type for global edge identification. /// Defines a type for global edge identification via its DOFs.
typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> GlobalEdge; typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> DofEdge;
/// Defines a tzpe for global face identiication via its DOFs.
typedef boost::tuple<DegreeOfFreedom, DegreeOfFreedom, DegreeOfFreedom> DofFace;
/// returns the GeoIndex of d for dimension dim. /// returns the GeoIndex of d for dimension dim.
#define INDEX_OF_DIM(d, dim) (static_cast<GeoIndex>((d == dim) ? CENTER : d + 1)) #define INDEX_OF_DIM(d, dim) (static_cast<GeoIndex>((d == dim) ? CENTER : d + 1))
......
...@@ -88,10 +88,16 @@ namespace AMDiS { ...@@ -88,10 +88,16 @@ namespace AMDiS {
return 0; return 0;
} }
GlobalEdge getEdge(int localEdgeIndex) const DofEdge getEdge(int localEdgeIndex) const
{ {
ERROR_EXIT("This does not work in 1D!\n"); ERROR_EXIT("This does not work in 1D!\n");
return GlobalEdge(); return DofEdge();
}
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
} }
/// implements Element::sortFaceIndices /// implements Element::sortFaceIndices
......
...@@ -266,18 +266,48 @@ namespace AMDiS { ...@@ -266,18 +266,48 @@ namespace AMDiS {
return edgeOfFace[face][edge]; return edgeOfFace[face][edge];
} }
GlobalEdge getEdge(int localEdgeIndex) const DofEdge getEdge(int localEdgeIndex) const
{ {
FUNCNAME("Tetrahedron::getEdge()"); FUNCNAME("Tetrahedron::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("invalid edge\n"); TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("Invalid edge!\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0]; DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0]; DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1)); DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge; return edge;
} }
DofFace getFace(int localFaceIndex) const
{
FUNCNAME("Tetrahedron::getFace()");
TEST_EXIT_DBG(localFaceIndex >= 0 && localFaceIndex < 4)("Invalid face!\n");
// Get the three DOFs of the face.
DegreeOfFreedom dof0 = dof[vertexOfFace[localFaceIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfFace[localFaceIndex][1]][0];
DegreeOfFreedom dof2 = dof[vertexOfFace[localFaceIndex][2]][0];
// Sort the three DOFs of the face with respect to their values.
DegreeOfFreedom dofMin0, dofMin1, dofMin2;
if (dof0 < dof1 && dof0 < dof2) {
dofMin0 = dof0;
dofMin1 = std::min(dof1, dof2);
dofMin2 = std::max(dof1, dof2);
} else if (dof1 < dof0 && dof1 < dof2) {
dofMin0 = dof1;
dofMin1 = std::min(dof0, dof2);
dofMin2 = std::max(dof0, dof2);
} else {
TEST_EXIT_DBG(dof2 < dof0 && dof2 < dof1)("Should not happen!\n");
dofMin0 = dof2;
dofMin1 = std::min(dof0, dof1);
dofMin2 = std::max(dof0, dof1);
}
return DofFace(dofMin0, dofMin1, dofMin2);
}
protected: protected:
/// vertexOfEdge[i][j] is the local number of the j-vertex of edge i /// vertexOfEdge[i][j] is the local number of the j-vertex of edge i
......
...@@ -170,18 +170,24 @@ namespace AMDiS { ...@@ -170,18 +170,24 @@ namespace AMDiS {
return edge; return edge;
} }
GlobalEdge getEdge(int localEdgeIndex) const DofEdge getEdge(int localEdgeIndex) const
{ {
FUNCNAME("Triangle::getEdge()"); FUNCNAME("Triangle::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 3)("invalid edge\n"); TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 3)("invalid edge\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0]; DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0]; DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1)); DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge; return edge;
} }
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
}
std::string getTypeName() const std::string getTypeName() const
{ {
return "Triangle"; return "Triangle";
......
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
#include <fstream> #include <fstream>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include "parallel/MeshDistributor.h" #include "parallel/MeshDistributor.h"
#include "parallel/ParallelDebug.h" #include "parallel/ParallelDebug.h"
...@@ -994,18 +996,22 @@ namespace AMDiS { ...@@ -994,18 +996,22 @@ namespace AMDiS {
// element index, the second int denotes the local vertex index within // element index, the second int denotes the local vertex index within
// the element. // the element.
std::map<DegreeOfFreedom, std::vector<ElementNeighbour> > vertexElements; std::map<DegreeOfFreedom, std::vector<ElementNeighbour> > vertexElements;
std::map<GlobalEdge, std::vector<ElementNeighbour> > edgeElements; std::map<DofEdge, std::vector<ElementNeighbour> > edgeElements;
std::map<DofFace, std::vector<ElementNeighbour> > faceElements;
std::map<DegreeOfFreedom, std::map<int, ElementNeighbour> > vertexInRank; std::map<DegreeOfFreedom, std::map<int, ElementNeighbour> > vertexInRank;
std::map<GlobalEdge, std::map<int, ElementNeighbour> > edgeInRank; std::map<DofEdge, std::map<int, ElementNeighbour> > edgeInRank;
std::map<DofFace, std::map<int, ElementNeighbour> > faceInRank;
std::map<int, Element*> elIndexMap; std::map<int, Element*> elIndexMap;
std::map<int, int> elIndexTypeMap; std::map<int, int> elIndexTypeMap;
std::map<DegreeOfFreedom, int> vertexOwner; std::map<DegreeOfFreedom, int> vertexOwner;
std::map<GlobalEdge, int> edgeOwner; std::map<DofEdge, int> edgeOwner;
std::map<DofFace, int> faceOwner;
std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType> periodicEdges; std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
// === PHASE 1 === // === PHASE 1 ===
...@@ -1022,32 +1028,44 @@ namespace AMDiS { ...@@ -1022,32 +1028,44 @@ namespace AMDiS {
for (int i = 0; i < el->getGeo(VERTEX); i++) { for (int i = 0; i < el->getGeo(VERTEX); i++) {
DegreeOfFreedom vertex = el->getDOF(i, 0); DegreeOfFreedom vertex = el->getDOF(i, 0);
vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i)); vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i));
if (vertexOwner.count(vertex))
vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]); vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]);
else
vertexOwner[vertex] = partitionVec[el->getIndex()];
} }
for (int i = 0; i < el->getGeo(EDGE); i++) { for (int i = 0; i < el->getGeo(EDGE); i++) {
GlobalEdge edge = el->getEdge(i); DofEdge edge = el->getEdge(i);
edgeElements[edge].push_back(ElementNeighbour(el->getIndex(), i)); edgeElements[edge].push_back(ElementNeighbour(el->getIndex(), i));
if (edgeOwner.count(edge))
edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]); edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]);
else }
edgeOwner[edge] = partitionVec[el->getIndex()];
for (int i = 0; i < el->getGeo(FACE); i++) {
DofFace face = el->getFace(i);
faceElements[face].push_back(ElementNeighbour(el->getIndex(), i));
faceOwner[face] = std::max(faceOwner[face], partitionVec[el->getIndex()]);
} }
switch (mesh->getDim()) {
case 2:
for (int i = 0; i < el->getGeo(EDGE); i++) { for (int i = 0; i < el->getGeo(EDGE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) { if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
GlobalEdge edge1 = el->getEdge(i); DofEdge edge1 = el->getEdge(i);
GlobalEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i)); DofEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i));
periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i); periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i);
} }
} }
break;
case 3:
for (int i = 0; i < el->getGeo(FACE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
DofFace face1 = el->getFace(i);
DofFace face2 = elInfo->getNeighbour(i)->getFace(elInfo->getOppVertex(i));
periodicFaces[std::make_pair(face1, face2)] = elInfo->getBoundary(i);
}
}
break;
}
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
...@@ -1060,24 +1078,31 @@ namespace AMDiS { ...@@ -1060,24 +1078,31 @@ namespace AMDiS {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex]; int elOwner = partitionVec[it2->elIndex];
if (vertexInRank[it->first].count(elOwner) == 0 || if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
vertexInRank[it->first][elOwner] = *it2; vertexInRank[it->first][elOwner] = *it2;
} }
} }
for (std::map<GlobalEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin(); for (std::map<DofEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) { it != edgeElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) { for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex]; int elOwner = partitionVec[it2->elIndex];
if (edgeInRank[it->first].count(elOwner) == 0 || if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
edgeInRank[it->first][elOwner] = *it2; edgeInRank[it->first][elOwner] = *it2;
} }
} }
for (std::map<DofFace, std::vector<ElementNeighbour> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex)
faceInRank[it->first][elOwner] = *it2;
}
}
// === PHASE 3 === // === PHASE 3 ===
...@@ -1141,7 +1166,7 @@ namespace AMDiS { ...@@ -1141,7 +1166,7 @@ namespace AMDiS {
} }
for (std::map<GlobalEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin(); for (std::map<DofEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin();
it != edgeInRank.end(); ++it) { it != edgeInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) { if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = edgeOwner[it->first]; int owner = edgeOwner[it->first];
...@@ -1201,9 +1226,68 @@ namespace AMDiS { ...@@ -1201,9 +1226,68 @@ namespace AMDiS {
} }
for (std::map<DofFace, std::map<int, ElementNeighbour> >::iterator it = faceInRank.begin();
it != faceInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = faceOwner[it->first];
ElementNeighbour& rankBoundEl = it->second[mpiRank];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = rankBoundEl.ith;
if (owner == mpiRank) {
for (std::map<int, ElementNeighbour>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = it2->second.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == it2->second.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = myIntBoundary.getNewAtomic(it2->first);
b = bound;
b.rankObj.setReverseMode(b.neighObj, feSpace);
}
} else {
TEST_EXIT_DBG(it->second.count(owner) == 1)
("Should not happen!\n");
ElementNeighbour& ownerBoundEl = it->second[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = ownerBoundEl.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == ownerBoundEl.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner);
b = bound;
b.neighObj.setReverseMode(b.rankObj, feSpace);
}
}
}
// === PHASE 4 === // === PHASE 4 ===
for (std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType>::iterator it = periodicEdges.begin(); for (std::map<std::pair<DofEdge, DofEdge>, BoundaryType>::iterator it = periodicEdges.begin();
it != periodicEdges.end(); ++it) { it != periodicEdges.end(); ++it) {
int perEdgeOwner0 = edgeOwner[it->first.first]; int perEdgeOwner0 = edgeOwner[it->first.first];
int perEdgeOwner1 = edgeOwner[it->first.second]; int perEdgeOwner1 = edgeOwner[it->first.second];
...@@ -1628,7 +1712,7 @@ namespace AMDiS { ...@@ -1628,7 +1712,7 @@ namespace AMDiS {
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
#if 1 #if 0
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
MSG("nRankDofs = %d\n", nRankDofs); MSG("nRankDofs = %d\n", nRankDofs);
MSG("nOverallDofs = %d\n", nOverallDofs); MSG("nOverallDofs = %d\n", nOverallDofs);
...@@ -1655,16 +1739,12 @@ namespace AMDiS { ...@@ -1655,16 +1739,12 @@ namespace AMDiS {
debug::getAllDofs(feSpace, testDofs); debug::getAllDofs(feSpace, testDofs);
for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin(); for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin();
it != testDofs.end(); ++it) { it != testDofs.end(); ++it) {
// WorldVector<double> coords;
// mesh->getDofIndexCoords(**it, feSpace, coords);
MSG("DOF %d: mapLocalGlobalDofs = %d vertexDof = %d isRankDof = %d\n", **it, MSG("DOF %d: mapLocalGlobalDofs = %d vertexDof = %d isRankDof = %d\n", **it,
mapLocalGlobalDofs[**it], mapLocalGlobalDofs[**it],
vertexDof[*it], vertexDof[*it],
isRankDof[**it]); isRankDof[**it]);
// MSG("coord = %f %f\n", coords[0], coords[1]);
} }
#endif #endif
#endif #endif
#endif #endif
} }
...@@ -1943,8 +2023,6 @@ namespace AMDiS { ...@@ -1943,8 +2023,6 @@ namespace AMDiS {
int mapGlobalDofIndex = recvBuffers[i + k][j]; int mapGlobalDofIndex = recvBuffers[i + k][j];
int type = 3; int type = 3;
MSG("NEW PER ASSOC %d <-> %d\n", globalDofIndex, mapGlobalDofIndex);
periodicDof[type][globalDofIndex] = mapGlobalDofIndex; periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
periodicDofAssociations[globalDofIndex].insert(type); periodicDofAssociations[globalDofIndex].insert(type);
} }
......