Commit d552afce authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on adaptivity in parallelization code and some code refactoring in this part too.

parent 35ef6dc6
...@@ -82,13 +82,13 @@ AR="ar" ...@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.3.2-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="gcc" CC="/usr/lib/openmpi/1.3.2-gcc//bin/mpicc"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -174,7 +174,7 @@ dlopen_self=unknown ...@@ -174,7 +174,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6801,13 +6801,13 @@ AR="ar" ...@@ -6801,13 +6801,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.3.2-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="g++" CC="/usr/lib/openmpi/1.3.2-gcc//bin/mpiCC"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -6893,7 +6893,7 @@ dlopen_self=unknown ...@@ -6893,7 +6893,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6960,11 +6960,11 @@ predeps="" ...@@ -6960,11 +6960,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a # Dependencies to place after the objects being linked to create a
# shared library. # shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s" postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking # The library search path used internally by the compiler when linking
# a shared library. # a shared library.
compiler_lib_search_path=`echo "-L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"` compiler_lib_search_path=`echo "-L/usr/lib/openmpi/1.3.2-gcc/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
# Method to check whether dependent libraries are shared objects. # Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all" deplibs_check_method="pass_all"
...@@ -7109,7 +7109,7 @@ AR="ar" ...@@ -7109,7 +7109,7 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.3.2-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
......
...@@ -58,6 +58,7 @@ namespace AMDiS { ...@@ -58,6 +58,7 @@ namespace AMDiS {
class MacroInfo; class MacroInfo;
class Marker; class Marker;
class Mesh; class Mesh;
class MeshStructure;
class OEMSolver; class OEMSolver;
class Operator; class Operator;
class OperatorTerm; class OperatorTerm;
......
...@@ -195,7 +195,7 @@ namespace AMDiS { ...@@ -195,7 +195,7 @@ namespace AMDiS {
if (condition && condition->isDirichlet()) { if (condition && condition->isDirichlet()) {
if (condition->applyBoundaryCondition()) { if (condition->applyBoundaryCondition()) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (rankDofs[rowIndices[i]]) if ((*rankDofs)[rowIndices[i]])
applyDBCs.insert(static_cast<int>(row)); applyDBCs.insert(static_cast<int>(row));
#else #else
applyDBCs.insert(static_cast<int>(row)); applyDBCs.insert(static_cast<int>(row));
......
...@@ -406,7 +406,7 @@ namespace AMDiS { ...@@ -406,7 +406,7 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void setRankDofs(std::map<DegreeOfFreedom, bool>& dofmap) void setRankDofs(std::map<DegreeOfFreedom, bool>& dofmap)
{ {
rankDofs = dofmap; rankDofs = &dofmap;
} }
#endif #endif
...@@ -488,7 +488,7 @@ namespace AMDiS { ...@@ -488,7 +488,7 @@ namespace AMDiS {
int nnzPerRow; int nnzPerRow;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
std::map<DegreeOfFreedom, bool> rankDofs; std::map<DegreeOfFreedom, bool> *rankDofs;
#endif #endif
/// Inserter object: implemented as pointer, allocated and deallocated as needed /// Inserter object: implemented as pointer, allocated and deallocated as needed
......
...@@ -211,16 +211,15 @@ namespace AMDiS { ...@@ -211,16 +211,15 @@ namespace AMDiS {
} }
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
inline void setRankDofs(std::map<DegreeOfFreedom, bool> dofmap) inline void setRankDofs(std::map<DegreeOfFreedom, bool> &dofmap)
{ {
rankDofs.clear(); // rankDofs = &dofmap;
// rankDofs = dofmap;
} }
inline bool isRankDof(DegreeOfFreedom dof) inline bool isRankDof(DegreeOfFreedom dof)
{ {
return rankDofs[dof]; TEST_EXIT_DBG(rankDofs)("No rank dofs set!\n");
return (*rankDofs)[dof];
} }
#endif #endif
...@@ -269,7 +268,7 @@ namespace AMDiS { ...@@ -269,7 +268,7 @@ namespace AMDiS {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
public: public:
std::map<DegreeOfFreedom, bool> rankDofs; std::map<DegreeOfFreedom, bool> *rankDofs;
#endif #endif
}; };
......
...@@ -54,7 +54,7 @@ namespace AMDiS { ...@@ -54,7 +54,7 @@ namespace AMDiS {
for (int i = 0; i < nBasFcts; i++) { for (int i = 0; i < nBasFcts; i++) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (vector->isRankDof(dofIndices[i])) // if (vector->isRankDof(dofIndices[i]))
#endif #endif
if (localBound[i] == boundaryType) { if (localBound[i] == boundaryType) {
if (f) { if (f) {
......
...@@ -344,10 +344,7 @@ namespace AMDiS { ...@@ -344,10 +344,7 @@ namespace AMDiS {
*/ */
double calcDet(const FixVec<WorldVector<double>, VERTEX> &coords) const; double calcDet(const FixVec<WorldVector<double>, VERTEX> &coords) const;
/** \brief /// Checks whether flag is set in ElInfo's \ref fillFlag. If not, the program exits.
* Checks whether flag is set in ElInfo's \ref fillFlag. If not, the program
* exits.
*/
void testFlag(const Flag& flag) const; void testFlag(const Flag& flag) const;
/** \brief /** \brief
...@@ -411,7 +408,7 @@ namespace AMDiS { ...@@ -411,7 +408,7 @@ namespace AMDiS {
ERROR("virtual function not implemented in this sub-class "); ERROR("virtual function not implemented in this sub-class ");
return(0.0); return 0.0;
} }
/// Get ElInfo's \ref elType. /// Get ElInfo's \ref elType.
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include "FixVec.h" #include "FixVec.h"
#include "ElementRegion_ED.h" #include "ElementRegion_ED.h"
#include "Serializer.h" #include "Serializer.h"
#include "MeshStructure.h"
namespace AMDiS { namespace AMDiS {
...@@ -565,4 +566,31 @@ namespace AMDiS { ...@@ -565,4 +566,31 @@ namespace AMDiS {
return result; return result;
} }
void fitElementToMeshCode(RefinementManager *refineManager, MeshStructure &code,
Element *el, int ithSide, int elType)
{
if (code.isLeafElement())
return;
if (el->isLeaf()) {
el->setMark(1);
refineManager->refineMesh(el->getMesh());
}
int s1 = el->getSideOfChild(0, ithSide, elType);
int s2 = el->getSideOfChild(1, ithSide, elType);
code.nextElement();
if (s1 != -1)
fitElementToMeshCode(refineManager, code,
el->getFirstChild(), s1, el->getChildType(elType));
code.nextElement();
if (s2 != -1)
fitElementToMeshCode(refineManager, code,
el->getSecondChild(), s2, el->getChildType(elType));
}
} }
...@@ -574,6 +574,11 @@ namespace AMDiS { ...@@ -574,6 +574,11 @@ namespace AMDiS {
friend class Mesh; friend class Mesh;
}; };
void fitElementToMeshCode(RefinementManager *refineManager,
MeshStructure &code,
Element *el,
int ithSide,
int elType);
} }
#endif // AMDIS_ELEMENT_H #endif // AMDIS_ELEMENT_H
......
...@@ -46,7 +46,7 @@ namespace AMDiS { ...@@ -46,7 +46,7 @@ namespace AMDiS {
static FiniteElemSpace *provideFESpace(DOFAdmin *admin, static FiniteElemSpace *provideFESpace(DOFAdmin *admin,
const BasisFunction *basFcts, const BasisFunction *basFcts,
Mesh *mesh, Mesh *mesh,
std::string name_ = ""); std::string name = "");
/// Destructor. /// Destructor.
~FiniteElemSpace(); ~FiniteElemSpace();
...@@ -92,7 +92,7 @@ namespace AMDiS { ...@@ -92,7 +92,7 @@ namespace AMDiS {
* Constructs a FiniteElemSpace with name name_ and the given DOFAdmin, * Constructs a FiniteElemSpace with name name_ and the given DOFAdmin,
* BasisFunction and Mesh. * BasisFunction and Mesh.
*/ */
FiniteElemSpace(DOFAdmin* admin_, FiniteElemSpace(DOFAdmin* admin,
const BasisFunction* basisFcts, const BasisFunction* basisFcts,
Mesh* mesh, Mesh* mesh,
std::string name = ""); std::string name = "");
......
...@@ -99,7 +99,7 @@ namespace AMDiS { ...@@ -99,7 +99,7 @@ namespace AMDiS {
} }
/// Returns \ref dim of the mesh /// Returns \ref dim of the mesh
inline int getDim() const \ inline int getDim() const
{ {
return dim; return dim;
} }
......
...@@ -53,6 +53,10 @@ namespace AMDiS { ...@@ -53,6 +53,10 @@ namespace AMDiS {
reset(); reset();
} }
/** \brief
* Sets all position counters, that are used to traverse the code, to the starting
* position. The code itself is not changed.
*/
void reset(); void reset();
inline void commit() inline void commit()
......
...@@ -40,13 +40,13 @@ namespace AMDiS { ...@@ -40,13 +40,13 @@ namespace AMDiS {
ParallelDomainBase::ParallelDomainBase(ProblemIterationInterface *iIF, ParallelDomainBase::ParallelDomainBase(ProblemIterationInterface *iIF,
ProblemTimeInterface *tIF, ProblemTimeInterface *tIF,
FiniteElemSpace *fe, FiniteElemSpace *fe,
RefinementManager *refineManager) RefinementManager *refinementManager)
: iterationIF(iIF), : iterationIF(iIF),
timeIF(tIF), timeIF(tIF),
name(iIF->getName()), name(iIF->getName()),
feSpace(fe), feSpace(fe),
mesh(fe->getMesh()), mesh(fe->getMesh()),
refinementManager(refineManager), refineManager(refinementManager),
initialPartitionMesh(true), initialPartitionMesh(true),
nRankDofs(0), nRankDofs(0),
rstart(0), rstart(0),
...@@ -100,14 +100,7 @@ namespace AMDiS { ...@@ -100,14 +100,7 @@ namespace AMDiS {
// === Create new global and local DOF numbering. === // === Create new global and local DOF numbering. ===
// Set of all DOFs of the rank. createLocalGlobalNumbering();
std::vector<const DegreeOfFreedom*> rankDofs;
// Number of DOFs in ranks partition that are owned by the rank.
nRankDofs = 0;
// Number of all DOFs in the macro mesh.
int nOverallDOFs = 0;
createLocalGlobalNumbering(rankDofs, nRankDofs, nOverallDOFs);
// === Create interior boundary information === // === Create interior boundary information ===
...@@ -120,6 +113,7 @@ namespace AMDiS { ...@@ -120,6 +113,7 @@ namespace AMDiS {
#if (DEBUG != 0) #if (DEBUG != 0)
dbgTestElementMap(elMap); dbgTestElementMap(elMap);
dbgTestInteriorBoundary(); dbgTestInteriorBoundary();
dbgTestCommonDofs(true);
#endif #endif
// === Reset all DOFAdmins of the mesh. === // === Reset all DOFAdmins of the mesh. ===
...@@ -136,33 +130,14 @@ namespace AMDiS { ...@@ -136,33 +130,14 @@ namespace AMDiS {
GET_PARAMETER(0, mesh->getName() + "->global refinements", "%d", &globalRefinement); GET_PARAMETER(0, mesh->getName() + "->global refinements", "%d", &globalRefinement);
if (globalRefinement > 0) { if (globalRefinement > 0) {
refinementManager->globalRefine(mesh, globalRefinement); refineManager->globalRefine(mesh, globalRefinement);
#if (DEBUG != 0)
elMap.clear();
dbgCreateElementMap(elMap);
#endif
updateLocalGlobalNumbering(nRankDofs, nOverallDOFs);
updateDofAdmins(); updateLocalGlobalNumbering();
#if (DEBUG != 0)
dbgTestElementMap(elMap);
#endif
// === Update periodic mapping, if there are periodic boundaries. === // === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap(); createPeriodicMap();
} }
lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0)
dbgTestCommonDofs(true);
#endif
nRankRows = nRankDofs * nComponents;
nOverallRows = nOverallDOFs * nComponents;
} }
...@@ -759,6 +734,8 @@ namespace AMDiS { ...@@ -759,6 +734,8 @@ namespace AMDiS {
void ParallelDomainBase::checkMeshChange() void ParallelDomainBase::checkMeshChange()
{ {
FUNCNAME("ParallelDomainBase::checkMeshChange()");
// === If mesh has not been changed, return. === // === If mesh has not been changed, return. ===
if (mesh->getChangeIndex() == lastMeshChangeIndex) if (mesh->getChangeIndex() == lastMeshChangeIndex)
...@@ -811,8 +788,14 @@ namespace AMDiS { ...@@ -811,8 +788,14 @@ namespace AMDiS {
elCode.init(boundIt->rankObj.el, boundIt->rankObj.ithObj, elCode.init(boundIt->rankObj.el, boundIt->rankObj.ithObj,
boundIt->rankObj.elType); boundIt->rankObj.elType);
if (elCode.getCode() != recvCodes[i].getCode()) if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
// recvCodes[i].reset();
// fitElementToMeshCode(refineManager, recvCodes[i], boundIt->rankObj.el,
// boundIt->rankObj.ithObj, boundIt->rankObj.elType);
meshFitTogether = false; meshFitTogether = false;
}
i++; i++;
} }
...@@ -823,7 +806,7 @@ namespace AMDiS { ...@@ -823,7 +806,7 @@ namespace AMDiS {
exit(0); exit(0);
} }
lastMeshChangeIndex = mesh->getChangeIndex(); updateLocalGlobalNumbering();
} }
...@@ -1207,23 +1190,21 @@ namespace AMDiS { ...@@ -1207,23 +1190,21 @@ namespace AMDiS {
} }
void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDofs, void ParallelDomainBase::createLocalGlobalNumbering()
int& nRankDofs,
int& nOverallDOFs)
{ {
FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()");
// === Get rank information about DOFs. === // === Get rank information about DOFs. ===
// Stores to each DOF pointer the set of ranks the DOF is part of. // Stores to each DOF pointer the set of ranks the DOF is part of.
std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs; DofToPartitions partitionDofs;
DofContainer rankAllDofs; DofContainer rankDofs, rankAllDofs;
DofToRank boundaryDofs; DofToRank boundaryDofs;
createDofMemberInfo(partitionDOFs, rankDofs, rankAllDofs, boundaryDofs, vertexDof); createDofMemberInfo(partitionDofs, rankDofs, rankAllDofs, boundaryDofs, vertexDof);
nRankDofs = rankDofs.size(); nRankDofs = rankDofs.size();
nOverallDOFs = partitionDOFs.size(); int nOverallDofs = partitionDofs.size();
// === Get starting position for global rank dof ordering. ==== // === Get starting position for global rank dof ordering. ====
...@@ -1284,8 +1265,8 @@ namespace AMDiS { ...@@ -1284,8 +1265,8 @@ namespace AMDiS {
// If the boundary dof is a rank dof, it must be send to other ranks. // If the boundary dof is a rank dof, it must be send to other ranks.
// Search for all ranks that have this dof too. // Search for all ranks that have this dof too.
for (std::set<int>::iterator itRanks = partitionDOFs[it->first].begin(); for (std::set<int>::iterator itRanks = partitionDofs[it->first].begin();
itRanks != partitionDOFs[it->first].end(); itRanks != partitionDofs[it->first].end();
++itRanks) { ++itRanks) {
if (*itRanks != mpiRank) { if (*itRanks != mpiRank) {
TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1) TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1)
...@@ -1376,12 +1357,22 @@ namespace AMDiS { ...@@ -1376,12 +1357,22 @@ namespace AMDiS {
createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex, createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
rankDofsNewGlobalIndex); rankDofsNewGlobalIndex);
nRankRows = nRankDofs * nComponents;
nOverallRows = nOverallDofs * nComponents;
lastMeshChangeIndex = mesh->getChangeIndex();
} }
void ParallelDomainBase::updateLocalGlobalNumbering(int& nRankDofs, int& nOverallDOFs) void ParallelDomainBase::updateLocalGlobalNumbering()
{ {
FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()");
#if (DEBUG != 0)
ElementIdxToDofs elMap;
dbgCreateElementMap(elMap);
#endif
typedef std::set<const DegreeOfFreedom*> DofSet; typedef std::set<const DegreeOfFreedom*> DofSet;
// === Get all DOFs in ranks partition. === // === Get all DOFs in ranks partition. ===
...@@ -1525,10 +1516,11 @@ namespace AMDiS { ...@@ -1525,10 +1516,11 @@ namespace AMDiS {
// === Calculate number of overall DOFs of all partitions. === // === Calculate number of overall DOFs of all partitions. ===
mpiComm.Allreduce(&nRankDofs, &nOverallDOFs, 1, MPI_INT, MPI_SUM);