Commit 9ea77d93 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

And try to generalize the new parallel data structure also to 3d.

parent 3da16af9
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=1572864
max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
......@@ -198,7 +198,10 @@ namespace AMDiS {
virtual int getEdgeOfFace(int face, int edge) const = 0;
///
virtual GlobalEdge getEdge(int localEdgeIndex) const = 0;
virtual DofEdge getEdge(int localEdgeIndex) const = 0;
///
virtual DofFace getFace(int localFaceIndex) const = 0;
/// Returns the number of parts of type i in this element
virtual int getGeo(GeoIndex i) const = 0;
......
......@@ -52,6 +52,7 @@
#include "mpi.h"
#endif
#include "boost/tuple/tuple.hpp"
#include "AMDiS_fwd.h"
namespace AMDiS {
......@@ -78,8 +79,11 @@ namespace AMDiS {
/// Defines type for a vector of DOF pointers.
typedef std::vector<const DegreeOfFreedom*> DofContainer;
/// Defines a type for global edge identification.
typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> GlobalEdge;
/// Defines a type for global edge identification via its DOFs.
typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> DofEdge;
/// Defines a tzpe for global face identiication via its DOFs.
typedef boost::tuple<DegreeOfFreedom, DegreeOfFreedom, DegreeOfFreedom> DofFace;
/// returns the GeoIndex of d for dimension dim.
#define INDEX_OF_DIM(d, dim) (static_cast<GeoIndex>((d == dim) ? CENTER : d + 1))
......
......@@ -88,10 +88,16 @@ namespace AMDiS {
return 0;
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return GlobalEdge();
return DofEdge();
}
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
}
/// implements Element::sortFaceIndices
......
......@@ -266,18 +266,48 @@ namespace AMDiS {
return edgeOfFace[face][edge];
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
FUNCNAME("Tetrahedron::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("invalid edge\n");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("Invalid edge!\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge;
}
DofFace getFace(int localFaceIndex) const
{
FUNCNAME("Tetrahedron::getFace()");
TEST_EXIT_DBG(localFaceIndex >= 0 && localFaceIndex < 4)("Invalid face!\n");
// Get the three DOFs of the face.
DegreeOfFreedom dof0 = dof[vertexOfFace[localFaceIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfFace[localFaceIndex][1]][0];
DegreeOfFreedom dof2 = dof[vertexOfFace[localFaceIndex][2]][0];
// Sort the three DOFs of the face with respect to their values.
DegreeOfFreedom dofMin0, dofMin1, dofMin2;
if (dof0 < dof1 && dof0 < dof2) {
dofMin0 = dof0;
dofMin1 = std::min(dof1, dof2);
dofMin2 = std::max(dof1, dof2);
} else if (dof1 < dof0 && dof1 < dof2) {
dofMin0 = dof1;
dofMin1 = std::min(dof0, dof2);
dofMin2 = std::max(dof0, dof2);
} else {
TEST_EXIT_DBG(dof2 < dof0 && dof2 < dof1)("Should not happen!\n");
dofMin0 = dof2;
dofMin1 = std::min(dof0, dof1);
dofMin2 = std::max(dof0, dof1);
}
return DofFace(dofMin0, dofMin1, dofMin2);
}
protected:
/// vertexOfEdge[i][j] is the local number of the j-vertex of edge i
......
......@@ -170,18 +170,24 @@ namespace AMDiS {
return edge;
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
FUNCNAME("Triangle::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 3)("invalid edge\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge;
}
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
}
std::string getTypeName() const
{
return "Triangle";
......
......@@ -3,6 +3,8 @@
#include <fstream>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include "parallel/MeshDistributor.h"
#include "parallel/ParallelDebug.h"
......@@ -994,18 +996,22 @@ namespace AMDiS {
// element index, the second int denotes the local vertex index within
// the element.
std::map<DegreeOfFreedom, std::vector<ElementNeighbour> > vertexElements;
std::map<GlobalEdge, std::vector<ElementNeighbour> > edgeElements;
std::map<DofEdge, std::vector<ElementNeighbour> > edgeElements;
std::map<DofFace, std::vector<ElementNeighbour> > faceElements;
std::map<DegreeOfFreedom, std::map<int, ElementNeighbour> > vertexInRank;
std::map<GlobalEdge, std::map<int, ElementNeighbour> > edgeInRank;
std::map<DofEdge, std::map<int, ElementNeighbour> > edgeInRank;
std::map<DofFace, std::map<int, ElementNeighbour> > faceInRank;
std::map<int, Element*> elIndexMap;
std::map<int, int> elIndexTypeMap;
std::map<DegreeOfFreedom, int> vertexOwner;
std::map<GlobalEdge, int> edgeOwner;
std::map<DofEdge, int> edgeOwner;
std::map<DofFace, int> faceOwner;
std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
// === PHASE 1 ===
......@@ -1021,32 +1027,44 @@ namespace AMDiS {
for (int i = 0; i < el->getGeo(VERTEX); i++) {
DegreeOfFreedom vertex = el->getDOF(i, 0);
vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i));
if (vertexOwner.count(vertex))
vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]);
else
vertexOwner[vertex] = partitionVec[el->getIndex()];
vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i));
vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(EDGE); i++) {
GlobalEdge edge = el->getEdge(i);
DofEdge edge = el->getEdge(i);
edgeElements[edge].push_back(ElementNeighbour(el->getIndex(), i));
if (edgeOwner.count(edge))
edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]);
else
edgeOwner[edge] = partitionVec[el->getIndex()];
edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(FACE); i++) {
DofFace face = el->getFace(i);
faceElements[face].push_back(ElementNeighbour(el->getIndex(), i));
faceOwner[face] = std::max(faceOwner[face], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(EDGE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
GlobalEdge edge1 = el->getEdge(i);
GlobalEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i));
periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i);
switch (mesh->getDim()) {
case 2:
for (int i = 0; i < el->getGeo(EDGE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
DofEdge edge1 = el->getEdge(i);
DofEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i));
periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i);
}
}
break;
case 3:
for (int i = 0; i < el->getGeo(FACE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
DofFace face1 = el->getFace(i);
DofFace face2 = elInfo->getNeighbour(i)->getFace(elInfo->getOppVertex(i));
periodicFaces[std::make_pair(face1, face2)] = elInfo->getBoundary(i);
}
}
break;
}
elInfo = stack.traverseNext(elInfo);
......@@ -1060,24 +1078,31 @@ namespace AMDiS {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (vertexInRank[it->first].count(elOwner) == 0 ||
it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
vertexInRank[it->first][elOwner] = *it2;
}
}
for (std::map<GlobalEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin();
for (std::map<DofEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (edgeInRank[it->first].count(elOwner) == 0 ||
it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
edgeInRank[it->first][elOwner] = *it2;
}
}
for (std::map<DofFace, std::vector<ElementNeighbour> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex)
faceInRank[it->first][elOwner] = *it2;
}
}
// === PHASE 3 ===
......@@ -1141,7 +1166,7 @@ namespace AMDiS {
}
for (std::map<GlobalEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin();
for (std::map<DofEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin();
it != edgeInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = edgeOwner[it->first];
......@@ -1201,9 +1226,68 @@ namespace AMDiS {
}
for (std::map<DofFace, std::map<int, ElementNeighbour> >::iterator it = faceInRank.begin();
it != faceInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = faceOwner[it->first];
ElementNeighbour& rankBoundEl = it->second[mpiRank];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = rankBoundEl.ith;
if (owner == mpiRank) {
for (std::map<int, ElementNeighbour>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = it2->second.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == it2->second.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = myIntBoundary.getNewAtomic(it2->first);
b = bound;
b.rankObj.setReverseMode(b.neighObj, feSpace);
}
} else {
TEST_EXIT_DBG(it->second.count(owner) == 1)
("Should not happen!\n");
ElementNeighbour& ownerBoundEl = it->second[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = ownerBoundEl.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == ownerBoundEl.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner);
b = bound;
b.neighObj.setReverseMode(b.rankObj, feSpace);
}
}
}
// === PHASE 4 ===
for (std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType>::iterator it = periodicEdges.begin();
for (std::map<std::pair<DofEdge, DofEdge>, BoundaryType>::iterator it = periodicEdges.begin();
it != periodicEdges.end(); ++it) {
int perEdgeOwner0 = edgeOwner[it->first.first];
int perEdgeOwner1 = edgeOwner[it->first.second];
......@@ -1628,7 +1712,7 @@ namespace AMDiS {
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true);
#if 1
#if 0
MSG("------------- Debug information -------------\n");
MSG("nRankDofs = %d\n", nRankDofs);
MSG("nOverallDofs = %d\n", nOverallDofs);
......@@ -1655,16 +1739,12 @@ namespace AMDiS {
debug::getAllDofs(feSpace, testDofs);
for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin();
it != testDofs.end(); ++it) {
// WorldVector<double> coords;
// mesh->getDofIndexCoords(**it, feSpace, coords);
MSG("DOF %d: mapLocalGlobalDofs = %d vertexDof = %d isRankDof = %d\n", **it,
mapLocalGlobalDofs[**it],
vertexDof[*it],
isRankDof[**it]);
// MSG("coord = %f %f\n", coords[0], coords[1]);
}
#endif
#endif
#endif
}
......@@ -1943,8 +2023,6 @@ namespace AMDiS {
int mapGlobalDofIndex = recvBuffers[i + k][j];
int type = 3;
MSG("NEW PER ASSOC %d <-> %d\n", globalDofIndex, mapGlobalDofIndex);
periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
periodicDofAssociations[globalDofIndex].insert(type);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment