Commit 0addc7a4 authored by Thomas Witkowski's avatar Thomas Witkowski

Code refactoring to make the implementation of mesh repartitioning easier.

parent 913de2db
......@@ -51,32 +51,32 @@ build_old_libs=yes
pic_mode=default
# Whether or not to optimize for fast installation.
fast_install=yes
fast_install=needless
# The host system.
host_alias=
host=i686-redhat-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-redhat-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# A sed program that does not truncate output.
SED="/bin/sed"
SED="/usr/bin/sed"
# Sed that helps us avoid accidentally triggering echo(1) options like -n.
Xsed="$SED -e 1s/^X//"
# A grep program that handles long lines.
GREP="/bin/grep"
GREP="/usr/bin/grep"
# An ERE matcher.
EGREP="/bin/grep -E"
EGREP="/usr/bin/grep -E"
# A literal string matcher.
FGREP="/bin/grep -F"
FGREP="/usr/bin/grep -F"
# A BSD- or MS-compatible name lister.
NM="/usr/bin/nm -B"
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=98304
max_cmd_len=1572864
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="gcc"
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -204,7 +204,7 @@ runpath_var=LD_RUN_PATH
shlibpath_var=LD_LIBRARY_PATH
# Is shlibpath searched before the hard-coded library search path?
shlibpath_overrides_runpath=no
shlibpath_overrides_runpath=yes
# Format of library name prefix.
libname_spec="lib\$name"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/u/backofen/adds/local/lib /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib /lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib /usr/lib/xulrunner-1.9.2 "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -253,13 +253,13 @@ striplib="strip --strip-unneeded"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Commands used to build an old-style archive.
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="gcc"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -274,7 +274,7 @@ wl="-Wl,"
pic_flag=" -fPIC -DPIC"
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Does compiler simultaneously support -c and -o options?
compiler_c_o="yes"
......@@ -8908,13 +8908,13 @@ build_old_libs=`case $build_libtool_libs in yes) echo no;; *) echo yes;; esac`
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Commands used to build an old-style archive.
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="g++"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8929,7 +8929,7 @@ wl="-Wl,"
pic_flag=" -fPIC -DPIC"
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Does compiler simultaneously support -c and -o options?
compiler_c_o="yes"
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/u/backofen/adds/local/lib /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predeps=""
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/u/backofen/adds/local/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
......@@ -65,7 +65,6 @@
#include "OperatorTerm.h"
#include "Parameters.h"
#include "Parametric.h"
#include "PartitionElementData.h"
#include "PeriodicMap.h"
#include "PeriodicBC.h"
#include "PngWriter.h"
......
......@@ -51,7 +51,7 @@ namespace AMDiS {
MeshStructure elementStructure;
elementStructure.init(structureCode, codeSize);
elementStructure.fitMeshToStructure(mesh, refManager, false,
elementStructure.fitMeshToStructure(mesh, refManager,
false, elIndex, ignoreFinerMesh);
uint32_t nValuesPerVector = 0;
......
......@@ -11,7 +11,6 @@
#include "LeafData.h"
#include "SurfaceRegion_ED.h"
#include "ElementRegion_ED.h"
#include "PartitionElementData.h"
#include "DOFMatrix.h"
#include "UmfPackSolver.h"
#include "time/RosenbrockMethod.h"
......@@ -120,9 +119,6 @@ namespace AMDiS {
creator = new ElementRegion_ED::Creator;
addCreator("ElementRegion_ED", creator);
creator = new PartitionElementData::Creator;
addCreator("PartitionElementData", creator);
}
......
#include "ElementData.h"
#include "PartitionElementData.h"
namespace AMDiS {
void ElementData::coarsenElementData(Element* parent,
......@@ -8,13 +8,6 @@ namespace AMDiS {
int elTypeParent)
{
if (decorated) {
PartitionElementData* ped = NULL;
ped = dynamic_cast<PartitionElementData*>(thisChild->getElementData(PARTITION_ED));
ped = NULL;
ped = dynamic_cast<PartitionElementData*>(otherChild->getElementData(PARTITION_ED));
ped = NULL;
ped = dynamic_cast<PartitionElementData*> (parent->getElementData(PARTITION_ED));
decorated->coarsenElementData(parent, thisChild, otherChild, elTypeParent);
delete decorated;
decorated = NULL;
......
#include "MeshStructure.h"
#include "MeshStructure_ED.h"
#include "PartitionElementData.h"
#include "Mesh.h"
#include "Element.h"
#include "Traverse.h"
......@@ -226,7 +225,6 @@ namespace AMDiS {
void MeshStructure::fitMeshToStructure(Mesh *mesh,
RefinementManager *manager,
bool checkPartition,
bool debugMode,
int macroElIndex,
bool ignoreFinerMesh)
......@@ -267,23 +265,9 @@ namespace AMDiS {
cont = skipBranch(structure);
structure->commit();
bool decorate = true;
if (checkPartition) {
PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
TEST_EXIT_DBG(partitionData)("no partition element data\n");
PartitionStatus status = partitionData->getPartitionStatus();
if (debugMode == false && (status == OUT || status == UNDEFINED))
decorate = false;
}
if (decorate) {
MeshStructure_ED *elData = new MeshStructure_ED(element->getElementData());
elData->setStructure(structure);
element->setElementData(elData);
} else {
delete structure;
}
MeshStructure_ED *elData = new MeshStructure_ED(element->getElementData());
elData->setStructure(structure);
element->setElementData(elData);
} else {
cont = nextElement();
}
......
......@@ -109,7 +109,6 @@ namespace AMDiS {
*/
void fitMeshToStructure(Mesh *mesh,
RefinementManager *manager,
bool checkPartition = false,
bool debugMode = false,
int macroElIndex = -1,
bool ignoreFinerMesh = false);
......
// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// ============================================================================
// == ==
// == TU Dresden ==
// == ==
// == Institut fr Wissenschaftliches Rechnen ==
// == Zellescher Weg 12-14 ==
// == 01069 Dresden ==
// == germany ==
// == ==
// ============================================================================
// == ==
// == https://gforge.zih.tu-dresden.de/projects/amdis/ ==
// == ==
// ============================================================================
/** \file PartitionElementData.h */
#ifndef AMDIS_PARTITIONELEMENTDATA_H
#define AMDIS_PARTITIONELEMENTDATA_H
#include "Element.h"
#include "ElementData.h"
#include "FixVec.h"
#include "Serializer.h"
namespace AMDiS {
const int PARTITION_ED = 7;
enum PartitionStatus {
UNDEFINED = -1,
IN = 0,
OVERLAP = 1,
OUT = 2
};
class PartitionElementData : public ElementData
{
public:
inline bool isOfType(int typeID) const
{
if (typeID == PARTITION_ED)
return true;
return false;
}
class Creator : public CreatorInterface<ElementData>
{
public:
ElementData* create()
{
return new PartitionElementData;
}
};
PartitionElementData(ElementData *decorated = NULL)
: ElementData(decorated),
status(UNDEFINED),
level(0)
{}
~PartitionElementData()
{}
bool refineElementData(Element* parent,
Element* child1,
Element* child2,
int elType)
{
ElementData::refineElementData(parent, child1, child2, elType);
PartitionElementData *child1Data =
new PartitionElementData(child1->getElementData());
PartitionElementData *child2Data =
new PartitionElementData(child2->getElementData());
child1Data->setPartitionStatus(status);
child2Data->setPartitionStatus(status);
child1Data->setLevel(level + 1);
child2Data->setLevel(level + 1);
child1->setElementData(child1Data);
child2->setElementData(child2Data);
return false;
}
ElementData *clone() const
{
PartitionElementData *newObj = new PartitionElementData;
newObj->decorated = ElementData::clone();
return newObj;
}
inline std::string getTypeName() const
{
return "PartitionElementData";
}
inline const int getTypeID() const
{
return PARTITION_ED;
}
void serialize(std::ostream& out)
{
ElementData::serialize(out);
SerUtil::serialize(out, status);
SerUtil::serialize(out, level);
}
void deserialize(std::istream& in)
{
ElementData::deserialize(in);
SerUtil::deserialize(in, status);
SerUtil::deserialize(in, level);
}
inline void setPartitionStatus(PartitionStatus s)
{
status = s;
}
inline PartitionStatus getPartitionStatus()
{
return status;
}
inline void setLevel(int l)
{
level = l;
}
inline int getLevel()
{
return level;
}
void descend(Element *element)
{
if (!element->isLeaf()) {
Element *child0 = element->getChild(0);
Element *child1 = element->getChild(1);
// get partition data
PartitionElementData *child0Data = dynamic_cast<PartitionElementData*>
(child0->getElementData(PARTITION_ED));
PartitionElementData *child1Data = dynamic_cast<PartitionElementData*>
(child1->getElementData(PARTITION_ED));
TEST_EXIT(child0Data && child1Data)("no partition data\n");
child0Data->setPartitionStatus(this->getPartitionStatus());
child1Data->setPartitionStatus(this->getPartitionStatus());
child0Data->descend(child0);
child1Data->descend(child1);
}
}
protected:
PartitionStatus status;
int level;
};
}
#endif
......@@ -14,7 +14,6 @@
#include "ElInfo.h"
#include "Element.h"
#include "MacroElement.h"
#include "PartitionElementData.h"
#include "DOFMatrix.h"
#include "DOFVector.h"
#include "SystemVector.h"
......@@ -112,7 +111,6 @@ namespace AMDiS {
MSG("Skip write part mesh!\n");
}
}
ParallelDebug::testAllElements(*this);
#endif
......@@ -145,6 +143,7 @@ namespace AMDiS {
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
......@@ -944,7 +943,7 @@ namespace AMDiS {
partitioner->partition(&elemWeights, INITIAL);
} else {
oldPartitionVec = partitionVec;
partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
partitioner->partition(&elemWeights, ADAPTIVE_REPART, 1000.0);
}
partitioner->fillCoarsePartitionVec(&partitionVec);
......@@ -1003,6 +1002,10 @@ namespace AMDiS {
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
partitionMesh();
#if (DEBUG != 0)
ParallelDebug::testAllElements(*this);
#endif
MSG("DONE\n");
}
......@@ -1412,13 +1415,9 @@ namespace AMDiS {
std::set<MacroElement*> macrosToRemove;
for (std::deque<MacroElement*>::iterator it = mesh->firstMacroElement();
it != mesh->endOfMacroElements();
++it) {
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
((*it)->getElement()->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN)
macrosToRemove.insert(*it);
}
++it)
if (partitioner->getElementInRank()[(*it)->getIndex()] == false)
macrosToRemove.insert(*it);
mesh->removeMacroElements(macrosToRemove, feSpace);
}
......
This diff is collapsed.
......@@ -46,6 +46,7 @@ namespace AMDiS {
{
public:
ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm,
std::map<int, bool>& elementInRank,
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal);
~ParMetisMesh();
......@@ -185,27 +186,35 @@ namespace AMDiS {
*/
void fillCoarsePartitionVec(std::map<int, int> *partitionVec);
void fillLeafPartitionVec(std::map<int, int> *coarseVec,
std::map<int, int> *fineVec);
/* \brief
* Creates an initial paritioning of the AMDiS mesh by seting the partition status
* of all elements to either IN or UNDEFINED.
*/
void createPartitionData();
void deletePartitionData();
void useLocalGlobalDofMap(std::map<DegreeOfFreedom, DegreeOfFreedom> *m)
{
mapLocalGlobal = m;
}
std::map<int, bool>& getElementInRank()
{
return elementInRank;
}
std::vector<std::vector<int> >& getRecvElements()
{
return recvElements;
}
std::vector<std::vector<int> >& getSendElements()
{
return sendElements;
}
protected:
void distributePartitioning(int *part);
void descendPartitionData(Element *element);
protected:
Mesh *mesh;
......@@ -214,6 +223,10 @@ namespace AMDiS {
ParMetisMesh *parMetisMesh;
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal;
std::map<int, bool> elementInRank;
std::vector<std::vector<int> > recvElements, sendElements;
};
}
......
#include "ParallelDebug.h"
#include "MeshDistributor.h"
#include "PartitionElementData.h"
#include "ProblemVec.h"
#include "DOFVector.h"
#include "FixVec.h"
......@@ -318,33 +317,44 @@ namespace AMDiS {
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
}
void ParallelDebug::testAllElements(MeshDistributor &pdb)
{
FUNCNAME("ParallelDebug::testAllElements()");
std::set<int> macroElements;
int minElementIndex = std::numeric_limits<int>::max();
int maxElementIndex = std::numeric_limits<int>::min();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
int elIndex = elInfo->getElement()->getIndex();
minElementIndex = std::min(minElementIndex, elIndex);
maxElementIndex = std::max(maxElementIndex, elIndex);
macroElements.insert(elInfo->getElement()->getIndex());
elInfo = stack.traverseNext(elInfo);
}
int globalMinIndex, globalMaxIndex;
pdb.mpiComm.Allreduce(&minElementIndex, &globalMinIndex, 1, MPI_INT, MPI_MIN);
pdb.mpiComm.Allreduce(&maxElementIndex, &globalMaxIndex, 1, MPI_INT, MPI_MAX);
int sendId = 0;
if (partitionData->getPartitionStatus() == IN)
sendId = 1;
TEST_EXIT(globalMinIndex == 0)("No macro element with index 0!\n");
for (int i = 0; i <= globalMaxIndex; i++) {
int sendId = macroElements.count(i);
int recvId = 0;
pdb.mpiComm.Allreduce(&sendId, &recvId, 1, MPI_INT, MPI_SUM);
if (recvId != 1 && pdb.mpiRank == 0) {
if (recvId == 0)
ERROR_EXIT("Element %d has no member partition!\n", element->getIndex());
if (recvId == 0) {
ERROR_EXIT("Element %d has no member partition!\n", i);
}
if (recvId > 1)
ERROR_EXIT("Element %d is member of more than pne partition!\n",
element->getIndex());
if (recvId > 1) {
ERROR_EXIT("Element %d is member of more than pne partition!\n", i);
}
}
elInfo = stack.traverseNext(elInfo);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment