Commit aded0540 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Revision of parallel domain decomposition code.

parent b4439d35
...@@ -44,7 +44,7 @@ available_tags=" CXX F77" ...@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG # ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host deimos103: # Libtool was configured on host p2q071:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -82,13 +82,13 @@ AR="ar" ...@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="gcc" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -171,7 +171,7 @@ dlopen_self=unknown ...@@ -171,7 +171,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac` ...@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End: # End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX # ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host deimos103: # Libtool was configured on host p2q071:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -6798,13 +6798,13 @@ AR="ar" ...@@ -6798,13 +6798,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="g++" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -6887,7 +6887,7 @@ dlopen_self=unknown ...@@ -6887,7 +6887,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6954,11 +6954,11 @@ predeps="" ...@@ -6954,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a # Dependencies to place after the objects being linked to create a
# shared library. # shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s" postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking # The library search path used internally by the compiler when linking
# a shared library. # a shared library.
compiler_lib_search_path="-L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.." compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Method to check whether dependent libraries are shared objects. # Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all" deplibs_check_method="pass_all"
...@@ -7065,7 +7065,7 @@ include_expsyms="" ...@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77 # ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host deimos103: # Libtool was configured on host p2q071:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -7103,7 +7103,7 @@ AR="ar" ...@@ -7103,7 +7103,7 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
......
...@@ -566,71 +566,4 @@ namespace AMDiS { ...@@ -566,71 +566,4 @@ namespace AMDiS {
return result; return result;
} }
bool fitElementToMeshCode(RefinementManager *refineManager,
MeshStructure &code,
Element *el,
int ithSide,
int elType)
{
FUNCNAME("fitElementToMeshCode()");
if (code.empty())
return false;
int s1 = el->getSideOfChild(0, ithSide, elType);
int s2 = el->getSideOfChild(1, ithSide, elType);
TEST_EXIT_DBG(s1 != -1 || s2 != -1)("This should not happen!\n");
if (s1 != -1 && s2 != -1)
return fitElementToMeshCode2(refineManager, code, el, ithSide, elType);
if (el->isLeaf()) {
if (code.getNumElements() == 1 && code.isLeafElement())
return false;
el->setMark(1);
refineManager->refineElement(el->getMesh(), el);
}
if (s1 != -1)
return fitElementToMeshCode2(refineManager, code,
el->getFirstChild(), s1, el->getChildType(elType));
else
return fitElementToMeshCode2(refineManager, code,
el->getSecondChild(), s2, el->getChildType(elType));
}
bool fitElementToMeshCode2(RefinementManager *refineManager, MeshStructure &code,
Element *el, int ithSide, int elType)
{
if (code.isLeafElement())
return false;
bool value = false;
if (el->isLeaf()) {
el->setMark(1);
refineManager->refineElement(el->getMesh(), el);
value = true;
}
int s1 = el->getSideOfChild(0, ithSide, elType);
int s2 = el->getSideOfChild(1, ithSide, elType);
if (s1 != -1) {
code.nextElement();
value |= fitElementToMeshCode2(refineManager, code, el->getFirstChild(),
s1, el->getChildType(elType));
}
if (s2 != -1) {
code.nextElement();
value |= fitElementToMeshCode2(refineManager, code, el->getSecondChild(),
s2, el->getChildType(elType));
}
return value;
}
} }
...@@ -574,18 +574,6 @@ namespace AMDiS { ...@@ -574,18 +574,6 @@ namespace AMDiS {
friend class Mesh; friend class Mesh;
}; };
bool fitElementToMeshCode(RefinementManager *refineManager,
MeshStructure &code,
Element *el,
int ithSide,
int elType);
bool fitElementToMeshCode2(RefinementManager *refineManager,
MeshStructure &code,
Element *el,
int ithSide,
int elType);
} }
#endif // AMDIS_ELEMENT_H #endif // AMDIS_ELEMENT_H
......
...@@ -76,12 +76,14 @@ namespace AMDiS { ...@@ -76,12 +76,14 @@ namespace AMDiS {
SerUtil::serialize(out, bound.rankObj.subObj); SerUtil::serialize(out, bound.rankObj.subObj);
SerUtil::serialize(out, bound.rankObj.ithObj); SerUtil::serialize(out, bound.rankObj.ithObj);
SerUtil::serialize(out, bound.rankObj.reverseMode); SerUtil::serialize(out, bound.rankObj.reverseMode);
serializeExcludeList(out, bound.rankObj.excludedSubstructures);
SerUtil::serialize(out, bound.neighObj.elIndex); SerUtil::serialize(out, bound.neighObj.elIndex);
SerUtil::serialize(out, bound.neighObj.elType); SerUtil::serialize(out, bound.neighObj.elType);
SerUtil::serialize(out, bound.neighObj.subObj); SerUtil::serialize(out, bound.neighObj.subObj);
SerUtil::serialize(out, bound.neighObj.ithObj); SerUtil::serialize(out, bound.neighObj.ithObj);
SerUtil::serialize(out, bound.neighObj.reverseMode); SerUtil::serialize(out, bound.neighObj.reverseMode);
serializeExcludeList(out, bound.neighObj.excludedSubstructures);
} }
} }
} }
...@@ -107,16 +109,48 @@ namespace AMDiS { ...@@ -107,16 +109,48 @@ namespace AMDiS {
SerUtil::deserialize(in, bound.rankObj.subObj); SerUtil::deserialize(in, bound.rankObj.subObj);
SerUtil::deserialize(in, bound.rankObj.ithObj); SerUtil::deserialize(in, bound.rankObj.ithObj);
SerUtil::deserialize(in, bound.rankObj.reverseMode); SerUtil::deserialize(in, bound.rankObj.reverseMode);
deserializeExcludeList(in, bound.rankObj.excludedSubstructures);
SerUtil::deserialize(in, bound.neighObj.elIndex); SerUtil::deserialize(in, bound.neighObj.elIndex);
SerUtil::deserialize(in, bound.neighObj.elType); SerUtil::deserialize(in, bound.neighObj.elType);
SerUtil::deserialize(in, bound.neighObj.subObj); SerUtil::deserialize(in, bound.neighObj.subObj);
SerUtil::deserialize(in, bound.neighObj.ithObj); SerUtil::deserialize(in, bound.neighObj.ithObj);
SerUtil::deserialize(in, bound.neighObj.reverseMode); SerUtil::deserialize(in, bound.neighObj.reverseMode);
deserializeExcludeList(in, bound.neighObj.excludedSubstructures);
bound.rankObj.el = elIndexMap[bound.rankObj.elIndex]; bound.rankObj.el = elIndexMap[bound.rankObj.elIndex];
bound.neighObj.el = NULL; bound.neighObj.el = NULL;
} }
} }
} }
void InteriorBoundary::serializeExcludeList(std::ostream &out, ExcludeList &list)
{
int size = list.size();
SerUtil::serialize(out, size);
for (int i = 0; i < size; i++) {
SerUtil::serialize(out, list[i].first);
SerUtil::serialize(out, list[i].second);
}
}
void InteriorBoundary::deserializeExcludeList(std::istream &in, ExcludeList &list)
{
int size = 0;
SerUtil::deserialize(in, size);
list.resize(0);
list.reserve(size);
for (int i = 0; i < size; i++) {
GeoIndex a;
int b;
SerUtil::deserialize(in, a);
SerUtil::deserialize(in, b);
list.push_back(std::make_pair(a, b));
}
}
} }
...@@ -31,12 +31,14 @@ ...@@ -31,12 +31,14 @@
namespace AMDiS { namespace AMDiS {
typedef std::vector<std::pair<GeoIndex, int> > ExcludeList;
/// Defines the geometrical objects that forms the boundary; /// Defines the geometrical objects that forms the boundary;
struct BoundaryObject { struct BoundaryObject {
BoundaryObject() BoundaryObject()
: reverseMode(false), : reverseMode(false),
notIncludedSubStructures(0) excludedSubstructures(0)
{} {}
BoundaryObject(Element *e, int eType, GeoIndex sObj, int iObj) BoundaryObject(Element *e, int eType, GeoIndex sObj, int iObj)
...@@ -46,7 +48,7 @@ namespace AMDiS { ...@@ -46,7 +48,7 @@ namespace AMDiS {
subObj(sObj), subObj(sObj),
ithObj(iObj), ithObj(iObj),
reverseMode(false), reverseMode(false),
notIncludedSubStructures(0) excludedSubstructures(0)
{} {}
void setReverseMode(BoundaryObject &otherBound, FiniteElemSpace *feSpace); void setReverseMode(BoundaryObject &otherBound, FiniteElemSpace *feSpace);
...@@ -83,7 +85,15 @@ namespace AMDiS { ...@@ -83,7 +85,15 @@ namespace AMDiS {
bool reverseMode; bool reverseMode;
std::vector<std::pair<GeoIndex, int> > notIncludedSubStructures; /** \brief
* In many situations it may be necessary to exclude some parts of the element to
* be part of the boundary. In 3d, when a face is part of the boundary, an edge or
* an vertex may be exludeded. In 2d only vertices may be exluded to be part of
* an edge boundary. This list contains pairs of exludeded structures. The first
* component of every pair denotes if it is a vertex or an edge, and the second
* component denotes the local index of the structure.
*/
ExcludeList excludedSubstructures;
}; };
/** \brief /** \brief
...@@ -117,6 +127,11 @@ namespace AMDiS { ...@@ -117,6 +127,11 @@ namespace AMDiS {
/// Reads the state of an interior boundary from a file. /// Reads the state of an interior boundary from a file.
void deserialize(std::istream &in, std::map<int, Element*> &elIndexMap); void deserialize(std::istream &in, std::map<int, Element*> &elIndexMap);
protected:
void serializeExcludeList(std::ostream &out, ExcludeList &list);
void deserializeExcludeList(std::istream &in, ExcludeList &list);
public: public:
RankToBoundMap boundary; RankToBoundMap boundary;
}; };
......
...@@ -287,14 +287,12 @@ namespace AMDiS { ...@@ -287,14 +287,12 @@ namespace AMDiS {
switch(mode) { switch(mode) {
case INITIAL: case INITIAL:
// TODO: Removed weight because it does not work correctly for very large
// macro meshes.
ParMETIS_V3_PartKway(parMetisMesh->getElementDist(), ParMETIS_V3_PartKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(), parMetisGraph.getXAdj(),
parMetisGraph.getAdjncy(), parMetisGraph.getAdjncy(),
wgts,
NULL, NULL,
NULL, &wgtflag,
NULL,
&numflag, &numflag,
&ncon, &ncon,
&nparts, &nparts,
......
#include <algorithm> #include <algorithm>
#include <iostream>
#include <fstream>
#include "ParallelDomainBase.h" #include "ParallelDomainBase.h"
#include "ParMetisPartitioner.h" #include "ParMetisPartitioner.h"
...@@ -49,6 +51,8 @@ namespace AMDiS { ...@@ -49,6 +51,8 @@ namespace AMDiS {
mesh(fe->getMesh()), mesh(fe->getMesh()),
refineManager(refinementManager), refineManager(refinementManager),
initialPartitionMesh(true), initialPartitionMesh(true),
d_nnz(NULL),
o_nnz(NULL),
nRankDofs(0), nRankDofs(0),
rstart(0), rstart(0),
nComponents(1), nComponents(1),
...@@ -364,38 +368,25 @@ namespace AMDiS { ...@@ -364,38 +368,25 @@ namespace AMDiS {
} }
void ParallelDomainBase::fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec) void ParallelDomainBase::createPetscNnzStructure(Matrix<DOFMatrix*> *mat)
{ {
FUNCNAME("ParallelDomainBase::fillPetscMatrix()"); FUNCNAME("ParallelDomainBase::createPetscNnzStructure()");
clock_t first = clock();
// === Create PETSc vector (rhs, solution and a temporary vector). === TEST_EXIT_DBG(!d_nnz)("There is something wrong!\n");
TEST_EXIT_DBG(!o_nnz)("There is something wrong!\n");
VecCreate(PETSC_COMM_WORLD, &petscRhsVec); d_nnz = new int[nRankRows];
VecSetSizes(petscRhsVec, nRankRows, nOverallRows); o_nnz = new int[nRankRows];
VecSetType(petscRhsVec, VECMPI); for (int i = 0; i < nRankRows; i++) {
d_nnz[i] = 0;
VecCreate(PETSC_COMM_WORLD, &petscSolVec); o_nnz[i] = 0;
VecSetSizes(petscSolVec, nRankRows, nOverallRows); }
VecSetType(petscSolVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end; using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits; namespace traits = mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix; typedef DOFMatrix::base_matrix_type Matrix;
typedef std::vector<std::pair<int, int> > MatrixNnzEntry; typedef std::vector<std::pair<int, int> > MatrixNnzEntry;
int d_nnz[nRankRows];
int o_nnz[nRankRows];
for (int i = 0; i < nRankRows; i++) {
d_nnz[i] = 0;
o_nnz[i] = 0;
}
// Stores to each rank a list of nnz entries (i.e. pairs of row and column index) // Stores to each rank a list of nnz entries (i.e. pairs of row and column index)
// that this rank will send to. This nnz entries will be assembled on this rank, // that this rank will send to. This nnz entries will be assembled on this rank,
// but because the row DOFs are not DOFs of this rank they will be send to the // but because the row DOFs are not DOFs of this rank they will be send to the
...@@ -531,12 +522,39 @@ namespace AMDiS { ...@@ -531,12 +522,39 @@ namespace AMDiS {
} }
} }
} }
}
void ParallelDomainBase::fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec)
{
FUNCNAME("ParallelDomainBase::fillPetscMatrix()");
clock_t first = clock();
// === Create PETSc vector (rhs, solution and a temporary vector). ===
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
if (!d_nnz)
createPetscNnzStructure(mat);
// === Create PETSc matrix with the computed nnz data structure. === // === Create PETSc matrix with the computed nnz data structure. ===
MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows, MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
0, d_nnz, 0, o_nnz, &petscMatrix); 0, d_nnz, 0, o_nnz, &petscMatrix);
INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock()));
#if (DEBUG != 0) #if (DEBUG != 0)
int a, b; int a, b;
MatGetOwnershipRange(petscMatrix, &a, &b); MatGetOwnershipRange(petscMatrix, &a, &b);
...@@ -551,6 +569,8 @@ namespace AMDiS { ...@@ -551,6 +569,8 @@ namespace AMDiS {
if ((*mat)[i][j]) if ((*mat)[i][j])
setDofMatrix((*mat)[i][j], nComponents, i, j); setDofMatrix((*mat)[i][j], nComponents, i, j);
INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock()));
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
...@@ -727,10 +747,20 @@ namespace AMDiS { ...@@ -727,10 +747,20 @@ namespace AMDiS {
do { do {
// To check the interior boundaries, the ownership of the boundaries is not // To check the interior boundaries, the ownership of the boundaries is not
// important. Therefore, we add all boundaries to one boundary container. // important. Therefore, we add all boundaries to one boundary container.
RankToBoundMap allBound = myIntBoundary.boundary; RankToBoundMap allBound;
for (RankToBoundMap::iterator it = myIntBoundary.boundary.begin();
it != myIntBoundary.boundary.end(); ++it)
for (std::vector<AtomicBoundary>::iterator bit = it->second.begin();
bit != it->second.end(); ++bit)
if (bit->rankObj.subObj == FACE)
allBound[it->first].push_back(*bit);
for (RankToBoundMap::iterator it = otherIntBoundary.boundary.begin(); for (RankToBoundMap::iterator it = otherIntBoundary.boundary.begin();
it != otherIntBoundary.boundary.end(); ++it) it != otherIntBoundary.boundary.end(); ++it)
allBound[it->first] = it->second; for (std::vector<AtomicBoundary>::iterator bit = it->second.begin();
bit != it->second.end(); ++bit)
if (bit->rankObj.subObj == FACE)
allBound[it->first].push_back(*bit);
// === Check the boundaries and adapt mesh if necessary. === // === Check the boundaries and adapt mesh if necessary. ===
...@@ -741,6 +771,7 @@ namespace AMDiS { ...@@ -741,6 +771,7 @@ namespace AMDiS {
int sendValue = static_cast<int>(!meshChanged); int sendValue = static_cast<int>(!meshChanged);
recvAllValues = 0; recvAllValues = 0;
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM); mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
MSG("LOOP\n");
} while (recvAllValues != 0);