Skip to content

GitLab

Commit 9ea77d93 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

And try to generalize the new parallel data structure also to 3d.

parent 3da16af9
Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 57 deletions
AMDiS/libtool
View file @ 9ea77d93
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=1572864
max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
AMDiS/src/Element.h
View file @ 9ea77d93
......@@ -198,7 +198,10 @@ namespace AMDiS {
virtual int getEdgeOfFace(int face, int edge) const = 0;
///
virtual GlobalEdge getEdge(int localEdgeIndex) const = 0;
virtual DofEdge getEdge(int localEdgeIndex) const = 0;
///
virtual DofFace getFace(int localFaceIndex) const = 0;
/// Returns the number of parts of type i in this element
virtual int getGeo(GeoIndex i) const = 0;
......
AMDiS/src/Global.h
View file @ 9ea77d93
......@@ -52,6 +52,7 @@
#include "mpi.h"
#endif
#include "boost/tuple/tuple.hpp"
#include "AMDiS_fwd.h"
namespace AMDiS {
......@@ -78,8 +79,11 @@ namespace AMDiS {
/// Defines type for a vector of DOF pointers.
typedef std::vector<const DegreeOfFreedom*> DofContainer;
/// Defines a type for global edge identification.
typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> GlobalEdge;
/// Defines a type for global edge identification via its DOFs.
typedef std::pair<DegreeOfFreedom, DegreeOfFreedom> DofEdge;
/// Defines a tzpe for global face identiication via its DOFs.
typedef boost::tuple<DegreeOfFreedom, DegreeOfFreedom, DegreeOfFreedom> DofFace;
/// returns the GeoIndex of d for dimension dim.
#define INDEX_OF_DIM(d, dim) (static_cast<GeoIndex>((d == dim) ? CENTER : d + 1))
......
AMDiS/src/Line.h
View file @ 9ea77d93
......@@ -88,10 +88,16 @@ namespace AMDiS {
return 0;
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return GlobalEdge();
return DofEdge();
}
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
}
/// implements Element::sortFaceIndices
......
AMDiS/src/Tetrahedron.h
View file @ 9ea77d93
......@@ -266,18 +266,48 @@ namespace AMDiS {
return edgeOfFace[face][edge];
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
FUNCNAME("Tetrahedron::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("invalid edge\n");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 6)("Invalid edge!\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge;
}
DofFace getFace(int localFaceIndex) const
{
FUNCNAME("Tetrahedron::getFace()");
TEST_EXIT_DBG(localFaceIndex >= 0 && localFaceIndex < 4)("Invalid face!\n");
// Get the three DOFs of the face.
DegreeOfFreedom dof0 = dof[vertexOfFace[localFaceIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfFace[localFaceIndex][1]][0];
DegreeOfFreedom dof2 = dof[vertexOfFace[localFaceIndex][2]][0];
// Sort the three DOFs of the face with respect to their values.
DegreeOfFreedom dofMin0, dofMin1, dofMin2;
if (dof0 < dof1 && dof0 < dof2) {
dofMin0 = dof0;
dofMin1 = std::min(dof1, dof2);
dofMin2 = std::max(dof1, dof2);
} else if (dof1 < dof0 && dof1 < dof2) {
dofMin0 = dof1;
dofMin1 = std::min(dof0, dof2);
dofMin2 = std::max(dof0, dof2);
} else {
TEST_EXIT_DBG(dof2 < dof0 && dof2 < dof1)("Should not happen!\n");
dofMin0 = dof2;
dofMin1 = std::min(dof0, dof1);
dofMin2 = std::max(dof0, dof1);
}
return DofFace(dofMin0, dofMin1, dofMin2);
}
protected:
/// vertexOfEdge[i][j] is the local number of the j-vertex of edge i
......
AMDiS/src/Triangle.h
View file @ 9ea77d93
......@@ -170,18 +170,24 @@ namespace AMDiS {
return edge;
}
GlobalEdge getEdge(int localEdgeIndex) const
DofEdge getEdge(int localEdgeIndex) const
{
FUNCNAME("Triangle::getEdge()");
TEST_EXIT_DBG(localEdgeIndex >= 0 && localEdgeIndex < 3)("invalid edge\n");
DegreeOfFreedom dof0 = dof[vertexOfEdge[localEdgeIndex][0]][0];
DegreeOfFreedom dof1 = dof[vertexOfEdge[localEdgeIndex][1]][0];
GlobalEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
DofEdge edge = std::make_pair(min(dof0, dof1), max(dof0, dof1));
return edge;
}
DofFace getFace(int localFaceIndex) const
{
ERROR_EXIT("This does not work in 1D!\n");
return DofFace();
}
std::string getTypeName() const
{
return "Triangle";
......
AMDiS/src/parallel/MeshDistributor.cc
View file @ 9ea77d93
......@@ -3,6 +3,8 @@
#include <fstream>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include "parallel/MeshDistributor.h"
#include "parallel/ParallelDebug.h"
......@@ -994,18 +996,22 @@ namespace AMDiS {
// element index, the second int denotes the local vertex index within
// the element.
std::map<DegreeOfFreedom, std::vector<ElementNeighbour> > vertexElements;
std::map<GlobalEdge, std::vector<ElementNeighbour> > edgeElements;
std::map<DofEdge, std::vector<ElementNeighbour> > edgeElements;
std::map<DofFace, std::vector<ElementNeighbour> > faceElements;
std::map<DegreeOfFreedom, std::map<int, ElementNeighbour> > vertexInRank;
std::map<GlobalEdge, std::map<int, ElementNeighbour> > edgeInRank;
std::map<DofEdge, std::map<int, ElementNeighbour> > edgeInRank;
std::map<DofFace, std::map<int, ElementNeighbour> > faceInRank;
std::map<int, Element*> elIndexMap;
std::map<int, int> elIndexTypeMap;
std::map<DegreeOfFreedom, int> vertexOwner;
std::map<GlobalEdge, int> edgeOwner;
std::map<DofEdge, int> edgeOwner;
std::map<DofFace, int> faceOwner;
std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
// === PHASE 1 ===
......@@ -1021,32 +1027,44 @@ namespace AMDiS {
for (int i = 0; i < el->getGeo(VERTEX); i++) {
DegreeOfFreedom vertex = el->getDOF(i, 0);
vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i));
if (vertexOwner.count(vertex))
vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]);
else
vertexOwner[vertex] = partitionVec[el->getIndex()];
vertexElements[vertex].push_back(ElementNeighbour(el->getIndex(), i));
vertexOwner[vertex] = std::max(vertexOwner[vertex], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(EDGE); i++) {
GlobalEdge edge = el->getEdge(i);
DofEdge edge = el->getEdge(i);
edgeElements[edge].push_back(ElementNeighbour(el->getIndex(), i));
if (edgeOwner.count(edge))
edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]);
else
edgeOwner[edge] = partitionVec[el->getIndex()];
edgeOwner[edge] = std::max(edgeOwner[edge], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(FACE); i++) {
DofFace face = el->getFace(i);
faceElements[face].push_back(ElementNeighbour(el->getIndex(), i));
faceOwner[face] = std::max(faceOwner[face], partitionVec[el->getIndex()]);
}
for (int i = 0; i < el->getGeo(EDGE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
GlobalEdge edge1 = el->getEdge(i);
GlobalEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i));
periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i);
switch (mesh->getDim()) {
case 2:
for (int i = 0; i < el->getGeo(EDGE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
DofEdge edge1 = el->getEdge(i);
DofEdge edge2 = elInfo->getNeighbour(i)->getEdge(elInfo->getOppVertex(i));
periodicEdges[std::make_pair(edge1, edge2)] = elInfo->getBoundary(i);
}
}
break;
case 3:
for (int i = 0; i < el->getGeo(FACE); i++) {
if (mesh->isPeriodicAssociation(elInfo->getBoundary(i))) {
DofFace face1 = el->getFace(i);
DofFace face2 = elInfo->getNeighbour(i)->getFace(elInfo->getOppVertex(i));
periodicFaces[std::make_pair(face1, face2)] = elInfo->getBoundary(i);
}
}
break;
}
elInfo = stack.traverseNext(elInfo);
......@@ -1060,24 +1078,31 @@ namespace AMDiS {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (vertexInRank[it->first].count(elOwner) == 0 ||
it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
if (it2->elIndex > vertexInRank[it->first][elOwner].elIndex)
vertexInRank[it->first][elOwner] = *it2;
}
}
for (std::map<GlobalEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin();
for (std::map<DofEdge, std::vector<ElementNeighbour> >::iterator it = edgeElements.begin();
it != edgeElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (edgeInRank[it->first].count(elOwner) == 0 ||
it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
if (it2->elIndex > edgeInRank[it->first][elOwner].elIndex)
edgeInRank[it->first][elOwner] = *it2;
}
}
for (std::map<DofFace, std::vector<ElementNeighbour> >::iterator it = faceElements.begin();
it != faceElements.end(); ++it) {
for (std::vector<ElementNeighbour>::iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2) {
int elOwner = partitionVec[it2->elIndex];
if (it2->elIndex > faceInRank[it->first][elOwner].elIndex)
faceInRank[it->first][elOwner] = *it2;
}
}
// === PHASE 3 ===
......@@ -1141,7 +1166,7 @@ namespace AMDiS {
}
for (std::map<GlobalEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin();
for (std::map<DofEdge, std::map<int, ElementNeighbour> >::iterator it = edgeInRank.begin();
it != edgeInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = edgeOwner[it->first];
......@@ -1201,9 +1226,68 @@ namespace AMDiS {
}
for (std::map<DofFace, std::map<int, ElementNeighbour> >::iterator it = faceInRank.begin();
it != faceInRank.end(); ++it) {
if (it->second.count(mpiRank) && it->second.size() > 1) {
int owner = faceOwner[it->first];
ElementNeighbour& rankBoundEl = it->second[mpiRank];
AtomicBoundary bound;
bound.rankObj.el = elIndexMap[rankBoundEl.elIndex];
bound.rankObj.elIndex = rankBoundEl.elIndex;
bound.rankObj.elType = elIndexTypeMap[rankBoundEl.elIndex];
bound.rankObj.subObj = EDGE;
bound.rankObj.ithObj = rankBoundEl.ith;
if (owner == mpiRank) {
for (std::map<int, ElementNeighbour>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
if (it2->first == mpiRank)
continue;
bound.neighObj.el = elIndexMap[it2->second.elIndex];
bound.neighObj.elIndex = it2->second.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = it2->second.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == it2->second.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = myIntBoundary.getNewAtomic(it2->first);
b = bound;
b.rankObj.setReverseMode(b.neighObj, feSpace);
}
} else {
TEST_EXIT_DBG(it->second.count(owner) == 1)
("Should not happen!\n");
ElementNeighbour& ownerBoundEl = it->second[owner];
bound.neighObj.el = elIndexMap[ownerBoundEl.elIndex];
bound.neighObj.elIndex = ownerBoundEl.elIndex;
bound.neighObj.elType = -1;
bound.neighObj.subObj = FACE;
bound.neighObj.ithObj = ownerBoundEl.ith;
TEST_EXIT_DBG(rankBoundEl.boundaryIndex == ownerBoundEl.boundaryIndex)
("Should not happen!\n");
bound.type = rankBoundEl.boundaryIndex;
AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner);
b = bound;
b.neighObj.setReverseMode(b.rankObj, feSpace);
}
}
}
// === PHASE 4 ===
for (std::map<std::pair<GlobalEdge, GlobalEdge>, BoundaryType>::iterator it = periodicEdges.begin();
for (std::map<std::pair<DofEdge, DofEdge>, BoundaryType>::iterator it = periodicEdges.begin();
it != periodicEdges.end(); ++it) {
int perEdgeOwner0 = edgeOwner[it->first.first];
int perEdgeOwner1 = edgeOwner[it->first.second];
......@@ -1628,7 +1712,7 @@ namespace AMDiS {
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true);
#if 1
#if 0
MSG("------------- Debug information -------------\n");
MSG("nRankDofs = %d\n", nRankDofs);
MSG("nOverallDofs = %d\n", nOverallDofs);
......@@ -1655,16 +1739,12 @@ namespace AMDiS {
debug::getAllDofs(feSpace, testDofs);
for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin();
it != testDofs.end(); ++it) {
// WorldVector<double> coords;
// mesh->getDofIndexCoords(**it, feSpace, coords);
MSG("DOF %d: mapLocalGlobalDofs = %d vertexDof = %d isRankDof = %d\n", **it,
mapLocalGlobalDofs[**it],
vertexDof[*it],
isRankDof[**it]);
// MSG("coord = %f %f\n", coords[0], coords[1]);
}
#endif
#endif
#endif
}
......@@ -1943,8 +2023,6 @@ namespace AMDiS {
int mapGlobalDofIndex = recvBuffers[i + k][j];
int type = 3;
MSG("NEW PER ASSOC %d <-> %d\n", globalDofIndex, mapGlobalDofIndex);
periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
periodicDofAssociations[globalDofIndex].insert(type);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment