From 0addc7a4a3729d433d7c11c579f9c3c50b30b75a Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Fri, 19 Nov 2010 11:32:32 +0000
Subject: [PATCH] Code refactoring to make the implementation of mesh
repartitioning easier.
---
AMDiS/libtool | 46 +++---
AMDiS/src/AMDiS.h | 1 -
AMDiS/src/ArhReader.cc | 2 +-
AMDiS/src/CreatorMap.cc | 4 -
AMDiS/src/ElementData.cc | 9 +-
AMDiS/src/MeshStructure.cc | 22 +--
AMDiS/src/MeshStructure.h | 1 -
AMDiS/src/PartitionElementData.h | 170 ----------------------
AMDiS/src/parallel/MeshDistributor.cc | 19 ++-
AMDiS/src/parallel/ParMetisPartitioner.cc | 168 +++++----------------
AMDiS/src/parallel/ParMetisPartitioner.h | 27 +++-
AMDiS/src/parallel/ParallelDebug.cc | 40 +++--
12 files changed, 120 insertions(+), 389 deletions(-)
delete mode 100644 AMDiS/src/PartitionElementData.h
diff --git a/AMDiS/libtool b/AMDiS/libtool
index f674d32d..730f91e3 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -51,32 +51,32 @@ build_old_libs=yes
pic_mode=default
# Whether or not to optimize for fast installation.
-fast_install=yes
+fast_install=needless
# The host system.
host_alias=
-host=i686-redhat-linux-gnu
+host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
-build=i686-redhat-linux-gnu
+build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# A sed program that does not truncate output.
-SED="/bin/sed"
+SED="/usr/bin/sed"
# Sed that helps us avoid accidentally triggering echo(1) options like -n.
Xsed="$SED -e 1s/^X//"
# A grep program that handles long lines.
-GREP="/bin/grep"
+GREP="/usr/bin/grep"
# An ERE matcher.
-EGREP="/bin/grep -E"
+EGREP="/usr/bin/grep -E"
# A literal string matcher.
-FGREP="/bin/grep -F"
+FGREP="/usr/bin/grep -F"
# A BSD- or MS-compatible name lister.
NM="/usr/bin/nm -B"
@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
-max_cmd_len=98304
+max_cmd_len=1572864
# Object file suffix (normally "o").
objext=o
@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
-LTCC="gcc"
+LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
@@ -204,7 +204,7 @@ runpath_var=LD_RUN_PATH
shlibpath_var=LD_LIBRARY_PATH
# Is shlibpath searched before the hard-coded library search path?
-shlibpath_overrides_runpath=no
+shlibpath_overrides_runpath=yes
# Format of library name prefix.
libname_spec="lib\$name"
@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
-sys_lib_search_path_spec="/u/backofen/adds/local/lib /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib /lib"
+sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib /usr/lib/xulrunner-1.9.2 "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
# Whether dlopen is supported.
dlopen_support=unknown
@@ -253,13 +253,13 @@ striplib="strip --strip-unneeded"
# The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Commands used to build an old-style archive.
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="gcc"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -274,7 +274,7 @@ wl="-Wl,"
pic_flag=" -fPIC -DPIC"
# Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
# Does compiler simultaneously support -c and -o options?
compiler_c_o="yes"
@@ -8908,13 +8908,13 @@ build_old_libs=`case $build_libtool_libs in yes) echo no;; *) echo yes;; esac`
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Commands used to build an old-style archive.
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="g++"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -8929,7 +8929,7 @@ wl="-Wl,"
pic_flag=" -fPIC -DPIC"
# Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
# Does compiler simultaneously support -c and -o options?
compiler_c_o="yes"
@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
-compiler_lib_search_dirs="/u/backofen/adds/local/lib /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib/gcc/i386-redhat-linux/4.1.2 /usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
+compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
-predep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o"
-postdep_objects="/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predeps=""
-postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
-compiler_lib_search_path="-L/u/backofen/adds/local/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
+compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
diff --git a/AMDiS/src/AMDiS.h b/AMDiS/src/AMDiS.h
index 1734ff92..74da8b19 100644
--- a/AMDiS/src/AMDiS.h
+++ b/AMDiS/src/AMDiS.h
@@ -65,7 +65,6 @@
#include "OperatorTerm.h"
#include "Parameters.h"
#include "Parametric.h"
-#include "PartitionElementData.h"
#include "PeriodicMap.h"
#include "PeriodicBC.h"
#include "PngWriter.h"
diff --git a/AMDiS/src/ArhReader.cc b/AMDiS/src/ArhReader.cc
index 912cd865..1709c988 100644
--- a/AMDiS/src/ArhReader.cc
+++ b/AMDiS/src/ArhReader.cc
@@ -51,7 +51,7 @@ namespace AMDiS {
MeshStructure elementStructure;
elementStructure.init(structureCode, codeSize);
- elementStructure.fitMeshToStructure(mesh, refManager, false,
+ elementStructure.fitMeshToStructure(mesh, refManager,
false, elIndex, ignoreFinerMesh);
uint32_t nValuesPerVector = 0;
diff --git a/AMDiS/src/CreatorMap.cc b/AMDiS/src/CreatorMap.cc
index fbe1bdbd..4e180164 100644
--- a/AMDiS/src/CreatorMap.cc
+++ b/AMDiS/src/CreatorMap.cc
@@ -11,7 +11,6 @@
#include "LeafData.h"
#include "SurfaceRegion_ED.h"
#include "ElementRegion_ED.h"
-#include "PartitionElementData.h"
#include "DOFMatrix.h"
#include "UmfPackSolver.h"
#include "time/RosenbrockMethod.h"
@@ -120,9 +119,6 @@ namespace AMDiS {
creator = new ElementRegion_ED::Creator;
addCreator("ElementRegion_ED", creator);
-
- creator = new PartitionElementData::Creator;
- addCreator("PartitionElementData", creator);
}
diff --git a/AMDiS/src/ElementData.cc b/AMDiS/src/ElementData.cc
index 892a7e3e..ee81220a 100644
--- a/AMDiS/src/ElementData.cc
+++ b/AMDiS/src/ElementData.cc
@@ -1,5 +1,5 @@
#include "ElementData.h"
-#include "PartitionElementData.h"
+
namespace AMDiS {
void ElementData::coarsenElementData(Element* parent,
@@ -8,13 +8,6 @@ namespace AMDiS {
int elTypeParent)
{
if (decorated) {
- PartitionElementData* ped = NULL;
- ped = dynamic_cast<PartitionElementData*>(thisChild->getElementData(PARTITION_ED));
- ped = NULL;
- ped = dynamic_cast<PartitionElementData*>(otherChild->getElementData(PARTITION_ED));
- ped = NULL;
- ped = dynamic_cast<PartitionElementData*> (parent->getElementData(PARTITION_ED));
-
decorated->coarsenElementData(parent, thisChild, otherChild, elTypeParent);
delete decorated;
decorated = NULL;
diff --git a/AMDiS/src/MeshStructure.cc b/AMDiS/src/MeshStructure.cc
index e499031f..1c8f656b 100644
--- a/AMDiS/src/MeshStructure.cc
+++ b/AMDiS/src/MeshStructure.cc
@@ -1,6 +1,5 @@
#include "MeshStructure.h"
#include "MeshStructure_ED.h"
-#include "PartitionElementData.h"
#include "Mesh.h"
#include "Element.h"
#include "Traverse.h"
@@ -226,7 +225,6 @@ namespace AMDiS {
void MeshStructure::fitMeshToStructure(Mesh *mesh,
RefinementManager *manager,
- bool checkPartition,
bool debugMode,
int macroElIndex,
bool ignoreFinerMesh)
@@ -267,23 +265,9 @@ namespace AMDiS {
cont = skipBranch(structure);
structure->commit();
- bool decorate = true;
- if (checkPartition) {
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
- TEST_EXIT_DBG(partitionData)("no partition element data\n");
- PartitionStatus status = partitionData->getPartitionStatus();
- if (debugMode == false && (status == OUT || status == UNDEFINED))
- decorate = false;
- }
-
- if (decorate) {
- MeshStructure_ED *elData = new MeshStructure_ED(element->getElementData());
- elData->setStructure(structure);
- element->setElementData(elData);
- } else {
- delete structure;
- }
+ MeshStructure_ED *elData = new MeshStructure_ED(element->getElementData());
+ elData->setStructure(structure);
+ element->setElementData(elData);
} else {
cont = nextElement();
}
diff --git a/AMDiS/src/MeshStructure.h b/AMDiS/src/MeshStructure.h
index c50bb7df..9c202957 100644
--- a/AMDiS/src/MeshStructure.h
+++ b/AMDiS/src/MeshStructure.h
@@ -109,7 +109,6 @@ namespace AMDiS {
*/
void fitMeshToStructure(Mesh *mesh,
RefinementManager *manager,
- bool checkPartition = false,
bool debugMode = false,
int macroElIndex = -1,
bool ignoreFinerMesh = false);
diff --git a/AMDiS/src/PartitionElementData.h b/AMDiS/src/PartitionElementData.h
deleted file mode 100644
index 8caf7c9d..00000000
--- a/AMDiS/src/PartitionElementData.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// ============================================================================
-// == ==
-// == AMDiS - Adaptive multidimensional simulations ==
-// == ==
-// ============================================================================
-// == ==
-// == TU Dresden ==
-// == ==
-// == Institut für Wissenschaftliches Rechnen ==
-// == Zellescher Weg 12-14 ==
-// == 01069 Dresden ==
-// == germany ==
-// == ==
-// ============================================================================
-// == ==
-// == https://gforge.zih.tu-dresden.de/projects/amdis/ ==
-// == ==
-// ============================================================================
-
-/** \file PartitionElementData.h */
-
-#ifndef AMDIS_PARTITIONELEMENTDATA_H
-#define AMDIS_PARTITIONELEMENTDATA_H
-
-#include "Element.h"
-#include "ElementData.h"
-#include "FixVec.h"
-#include "Serializer.h"
-
-namespace AMDiS {
-
- const int PARTITION_ED = 7;
-
- enum PartitionStatus {
- UNDEFINED = -1,
- IN = 0,
- OVERLAP = 1,
- OUT = 2
- };
-
- class PartitionElementData : public ElementData
- {
- public:
- inline bool isOfType(int typeID) const
- {
- if (typeID == PARTITION_ED)
- return true;
- return false;
- }
-
- class Creator : public CreatorInterface<ElementData>
- {
- public:
- ElementData* create()
- {
- return new PartitionElementData;
- }
- };
-
- PartitionElementData(ElementData *decorated = NULL)
- : ElementData(decorated),
- status(UNDEFINED),
- level(0)
- {}
-
- ~PartitionElementData()
- {}
-
- bool refineElementData(Element* parent,
- Element* child1,
- Element* child2,
- int elType)
- {
- ElementData::refineElementData(parent, child1, child2, elType);
-
- PartitionElementData *child1Data =
- new PartitionElementData(child1->getElementData());
- PartitionElementData *child2Data =
- new PartitionElementData(child2->getElementData());
- child1Data->setPartitionStatus(status);
- child2Data->setPartitionStatus(status);
- child1Data->setLevel(level + 1);
- child2Data->setLevel(level + 1);
- child1->setElementData(child1Data);
- child2->setElementData(child2Data);
- return false;
- }
-
- ElementData *clone() const
- {
- PartitionElementData *newObj = new PartitionElementData;
- newObj->decorated = ElementData::clone();
- return newObj;
- }
-
- inline std::string getTypeName() const
- {
- return "PartitionElementData";
- }
-
- inline const int getTypeID() const
- {
- return PARTITION_ED;
- }
-
- void serialize(std::ostream& out)
- {
- ElementData::serialize(out);
- SerUtil::serialize(out, status);
- SerUtil::serialize(out, level);
- }
-
- void deserialize(std::istream& in)
- {
- ElementData::deserialize(in);
- SerUtil::deserialize(in, status);
- SerUtil::deserialize(in, level);
- }
-
- inline void setPartitionStatus(PartitionStatus s)
- {
- status = s;
- }
-
- inline PartitionStatus getPartitionStatus()
- {
- return status;
- }
-
- inline void setLevel(int l)
- {
- level = l;
- }
-
- inline int getLevel()
- {
- return level;
- }
-
- void descend(Element *element)
- {
- if (!element->isLeaf()) {
- Element *child0 = element->getChild(0);
- Element *child1 = element->getChild(1);
-
- // get partition data
- PartitionElementData *child0Data = dynamic_cast<PartitionElementData*>
- (child0->getElementData(PARTITION_ED));
- PartitionElementData *child1Data = dynamic_cast<PartitionElementData*>
- (child1->getElementData(PARTITION_ED));
-
- TEST_EXIT(child0Data && child1Data)("no partition data\n");
-
- child0Data->setPartitionStatus(this->getPartitionStatus());
- child1Data->setPartitionStatus(this->getPartitionStatus());
-
- child0Data->descend(child0);
- child1Data->descend(child1);
- }
- }
-
- protected:
- PartitionStatus status;
-
- int level;
- };
-
-}
-
-#endif
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index a2705100..c0d78f4a 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -14,7 +14,6 @@
#include "ElInfo.h"
#include "Element.h"
#include "MacroElement.h"
-#include "PartitionElementData.h"
#include "DOFMatrix.h"
#include "DOFVector.h"
#include "SystemVector.h"
@@ -112,7 +111,6 @@ namespace AMDiS {
MSG("Skip write part mesh!\n");
}
}
- ParallelDebug::testAllElements(*this);
#endif
@@ -145,6 +143,7 @@ namespace AMDiS {
#if (DEBUG != 0)
MSG("AMDiS runs in debug mode, so make some test ...\n");
+ ParallelDebug::testAllElements(*this);
debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true);
@@ -944,7 +943,7 @@ namespace AMDiS {
partitioner->partition(&elemWeights, INITIAL);
} else {
oldPartitionVec = partitionVec;
- partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
+ partitioner->partition(&elemWeights, ADAPTIVE_REPART, 1000.0);
}
partitioner->fillCoarsePartitionVec(&partitionVec);
@@ -1003,6 +1002,10 @@ namespace AMDiS {
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
partitionMesh();
+#if (DEBUG != 0)
+ ParallelDebug::testAllElements(*this);
+#endif
+
MSG("DONE\n");
}
@@ -1412,13 +1415,9 @@ namespace AMDiS {
std::set<MacroElement*> macrosToRemove;
for (std::deque<MacroElement*>::iterator it = mesh->firstMacroElement();
it != mesh->endOfMacroElements();
- ++it) {
- PartitionElementData *partitionData =
- dynamic_cast<PartitionElementData*>
- ((*it)->getElement()->getElementData(PARTITION_ED));
- if (partitionData->getPartitionStatus() != IN)
- macrosToRemove.insert(*it);
- }
+ ++it)
+ if (partitioner->getElementInRank()[(*it)->getIndex()] == false)
+ macrosToRemove.insert(*it);
mesh->removeMacroElements(macrosToRemove, feSpace);
}
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.cc b/AMDiS/src/parallel/ParMetisPartitioner.cc
index 1fc8d5b2..b92f6570 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.cc
+++ b/AMDiS/src/parallel/ParMetisPartitioner.cc
@@ -5,13 +5,13 @@
#include "ElInfo.h"
#include "Element.h"
#include "FixVec.h"
-#include "PartitionElementData.h"
#include "DOFVector.h"
#include "mpi.h"
namespace AMDiS {
ParMetisMesh::ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm,
+ std::map<int, bool>& elementInRank,
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal)
: dim(mesh->getDim()),
nElements(0),
@@ -25,15 +25,9 @@ namespace AMDiS {
int dow = Global::getGeo(WORLD);
TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
+ ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
- // get partition data
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (elInfo->getElement()->getElementData(PARTITION_ED));
-
- if (partitionData &&
- partitionData->getPartitionStatus() == IN &&
- partitionData->getLevel() == 0)
+ if (elementInRank[elInfo->getElement()->getIndex()])
elementCounter++;
elInfo = stack.traverseNext(elInfo);
@@ -73,20 +67,13 @@ namespace AMDiS {
elementCounter = 0;
- elInfo = stack.traverseFirst(mesh, -1,
- Mesh::CALL_EVERY_EL_PREORDER | Mesh::FILL_COORDS);
+ elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL | Mesh::FILL_COORDS);
while (elInfo) {
Element *element = elInfo->getElement();
int index = element->getIndex();
- // get partition data
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
-
// if element in partition
- if (partitionData &&
- partitionData->getPartitionStatus() == IN &&
- partitionData->getLevel() == 0) {
+ if (elementInRank[index]) {
// remember index
setParMetisIndex(index, elementCounter);
setAMDiSIndex(elementCounter, index);
@@ -117,6 +104,7 @@ namespace AMDiS {
elementCounter++;
}
+
elInfo = stack.traverseNext(elInfo);
}
}
@@ -157,8 +145,6 @@ namespace AMDiS {
ncommonnodes = parMetisMesh->getDim();
MPI_Comm tmpComm = MPI_Comm(*comm);
- int mpiRank = MPI::COMM_WORLD.Get_rank();
- int mpiSize = MPI::COMM_WORLD.Get_size();
ParMETIS_V3_Mesh2Dual(parMetisMesh->getElementDist(),
parMetisMesh->getElementPtr(),
@@ -178,18 +164,6 @@ namespace AMDiS {
}
- void ParMetisPartitioner::deletePartitionData()
- {
- TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
- while (elInfo) {
- Element *element = elInfo->getElement();
- element->deleteElementData(PARTITION_ED);
- elInfo = stack.traverseNext(elInfo);
- }
- }
-
-
void ParMetisPartitioner::createPartitionData()
{
FUNCNAME("ParMetrisPartitioner::createPartitionData()");
@@ -201,25 +175,20 @@ namespace AMDiS {
// === Create initial partitioning of the AMDiS mesh. ===
+ elementInRank.clear();
+
TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
+ ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
Element *element = elInfo->getElement();
- TEST_EXIT(element->getElementData(PARTITION_ED) == NULL)
- ("mesh already partitioned\n");
-
- PartitionElementData *elData =
- new PartitionElementData(element->getElementData());
- element->setElementData(elData);
-
if ((element->getIndex() >= mpiRank * elPerRank &&
element->getIndex() < (mpiRank + 1) * elPerRank) ||
(element->getIndex() >= mpiSize * elPerRank &&
mpiRank == mpiSize - 1))
- elData->setPartitionStatus(IN);
+ elementInRank[element->getIndex()] = true;
else
- elData->setPartitionStatus(UNDEFINED);
+ elementInRank[element->getIndex()] = false;
elInfo = stack.traverseNext(elInfo);
}
@@ -238,7 +207,8 @@ namespace AMDiS {
if (parMetisMesh)
delete parMetisMesh;
- parMetisMesh = new ParMetisMesh(mesh, mpiComm, mapLocalGlobal);
+ TEST_EXIT(elementInRank.size() != 0)("Should not happen!\n");
+ parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal);
int nElements = parMetisMesh->getNumElements();
@@ -249,20 +219,11 @@ namespace AMDiS {
float *ptr_floatWgts = floatWgts;
TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
+ ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
- Element *element = elInfo->getElement();
-
- // get partition data
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
-
- if (partitionData &&
- partitionData->getPartitionStatus() == IN &&
- partitionData->getLevel() == 0) {
-
- int index = element->getIndex();
+ int index = elInfo->getElement()->getIndex();
+ if (elementInRank[index]) {
// get weight
float wgt = static_cast<float>((*elemWeights)[index]);
maxWgt = max(wgt, maxWgt);
@@ -306,7 +267,7 @@ namespace AMDiS {
MPI_Comm tmpComm = MPI_Comm(*mpiComm);
- switch(mode) {
+ switch (mode) {
case INITIAL:
ParMETIS_V3_PartKway(parMetisMesh->getElementDist(),
parMetisGraph.getXAdj(),
@@ -346,6 +307,7 @@ namespace AMDiS {
&edgecut,
part,
&tmpComm);
+
delete [] vsize;
}
break;
@@ -365,6 +327,7 @@ namespace AMDiS {
&edgecut,
part,
&tmpComm);
+
break;
default:
ERROR_EXIT("unknown partitioning mode\n");
@@ -392,7 +355,7 @@ namespace AMDiS {
// update ParMETIS mesh to new partitioning
if (!parMetisMesh)
- parMetisMesh = new ParMetisMesh(mesh, mpiComm, mapLocalGlobal);
+ parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal);
int mpiRank = mpiComm->Get_rank();
int mpiSize = mpiComm->Get_size();
@@ -438,12 +401,13 @@ namespace AMDiS {
for (int i = 0; i < mpiSize; i++)
nPartitionElements[i] = 0;
for (int i = 0; i < nElements; i++)
- nPartitionElements[part[i]]++;
+ nPartitionElements[part[i]]++;
// collect number of partition elements from all ranks for this rank
int *nRankElements = new int[mpiSize];
mpiComm->Alltoall(nPartitionElements, 1, MPI_INT, nRankElements, 1, MPI_INT);
+
// sum up partition elements over all ranks
int *sumPartitionElements = new int[mpiSize];
mpiComm->Allreduce(nPartitionElements, sumPartitionElements, mpiSize,
@@ -458,12 +422,17 @@ namespace AMDiS {
int *partitionElements = new int[nElements];
int **partitionPtr = new int*[mpiSize];
- for (int i = 0; i < mpiSize; i++)
+ sendElements.resize(mpiSize);
+ for (int i = 0; i < mpiSize; i++) {
partitionPtr[i] = partitionElements + bufferOffset[i];
+ sendElements[i].clear();
+ }
for (int i = 0; i < nElements; i++) {
int partition = part[i];
int amdisIndex = parMetisMesh->getAMDiSIndex(i);
+
+ sendElements[partition].push_back(amdisIndex);
*(partitionPtr[partition]) = amdisIndex;
++(partitionPtr[partition]);
}
@@ -484,37 +453,23 @@ namespace AMDiS {
recvBufferOffset,
MPI_INT);
+ TEST_EXIT(elementInRank.size() != 0)("Should not happen!\n");
+ for (std::map<int, bool>::iterator it = elementInRank.begin();
+ it != elementInRank.end(); ++it)
+ elementInRank[it->first] = false;
// Create map which stores for each element index on ther partitioning level
// if the element is in the partition of this rank.
- std::map<int, bool> elementInPartition;
+ recvElements.resize(mpiSize);
for (int i = 0; i < mpiSize; i++) {
+ recvElements[i].clear();
+
int *rankStart = rankElements + recvBufferOffset[i];
int *rankEnd = rankStart + nRankElements[i];
- for (int *rankPtr = rankStart; rankPtr < rankEnd; ++rankPtr)
- elementInPartition[*rankPtr] = true;
- }
-
- TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
- while (elInfo) {
- Element *element = elInfo->getElement();
-
- // get partition data
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
-
- if (partitionData && partitionData->getLevel() == 0) {
- int amdisIndex = element->getIndex();
- if (elementInPartition[amdisIndex])
- partitionData->setPartitionStatus(IN);
- else
- partitionData->setPartitionStatus(OUT);
-
- descendPartitionData(element);
+ for (int *rankPtr = rankStart; rankPtr < rankEnd; ++rankPtr) {
+ elementInRank[*rankPtr] = true;
+ recvElements[i].push_back(*rankPtr);
}
-
- elInfo = stack.traverseNext(elInfo);
}
delete parMetisMesh;
@@ -530,51 +485,4 @@ namespace AMDiS {
delete [] recvBufferOffset;
}
-
- void ParMetisPartitioner::descendPartitionData(Element *element)
- {
- FUNCNAME("ParMetisPartitioner::descendPartitionData()");
-
- if (!element->isLeaf()) {
- Element *child0 = element->getChild(0);
- Element *child1 = element->getChild(1);
-
- // get partition data
- PartitionElementData *parentData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
- PartitionElementData *child0Data = dynamic_cast<PartitionElementData*>
- (child0->getElementData(PARTITION_ED));
- PartitionElementData *child1Data = dynamic_cast<PartitionElementData*>
- (child1->getElementData(PARTITION_ED));
-
- TEST_EXIT(parentData && child0Data && child1Data)("no partition data\n");
-
- child0Data->setPartitionStatus(parentData->getPartitionStatus());
- child1Data->setPartitionStatus(parentData->getPartitionStatus());
-
- descendPartitionData(child0);
- descendPartitionData(child1);
- }
- }
-
-
- void ParMetisPartitioner::fillLeafPartitionVec(std::map<int, int> *coarseVec,
- std::map<int, int> *fineVec)
- {
- int partition = -1;
- TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
- while (elInfo) {
- Element *element = elInfo->getElement();
- PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
- (element->getElementData(PARTITION_ED));
- if (partitionData) {
- if (partitionData->getLevel() == 0)
- partition = (*(coarseVec))[element->getIndex()];
- if (element->isLeaf())
- (*(fineVec))[element->getIndex()] = partition;
- }
- elInfo = stack.traverseNext(elInfo);
- }
- }
}
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.h b/AMDiS/src/parallel/ParMetisPartitioner.h
index c860fb29..e9ca5f5b 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.h
+++ b/AMDiS/src/parallel/ParMetisPartitioner.h
@@ -46,6 +46,7 @@ namespace AMDiS {
{
public:
ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm,
+ std::map<int, bool>& elementInRank,
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal);
~ParMetisMesh();
@@ -185,27 +186,35 @@ namespace AMDiS {
*/
void fillCoarsePartitionVec(std::map<int, int> *partitionVec);
- void fillLeafPartitionVec(std::map<int, int> *coarseVec,
- std::map<int, int> *fineVec);
-
/* \brief
* Creates an initial paritioning of the AMDiS mesh by seting the partition status
* of all elements to either IN or UNDEFINED.
*/
void createPartitionData();
- void deletePartitionData();
-
void useLocalGlobalDofMap(std::map<DegreeOfFreedom, DegreeOfFreedom> *m)
{
mapLocalGlobal = m;
}
+ std::map<int, bool>& getElementInRank()
+ {
+ return elementInRank;
+ }
+
+ std::vector<std::vector<int> >& getRecvElements()
+ {
+ return recvElements;
+ }
+
+ std::vector<std::vector<int> >& getSendElements()
+ {
+ return sendElements;
+ }
+
protected:
void distributePartitioning(int *part);
- void descendPartitionData(Element *element);
-
protected:
Mesh *mesh;
@@ -214,6 +223,10 @@ namespace AMDiS {
ParMetisMesh *parMetisMesh;
std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal;
+
+ std::map<int, bool> elementInRank;
+
+ std::vector<std::vector<int> > recvElements, sendElements;
};
}
diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc
index 15289a95..f5af359b 100644
--- a/AMDiS/src/parallel/ParallelDebug.cc
+++ b/AMDiS/src/parallel/ParallelDebug.cc
@@ -1,6 +1,5 @@
#include "ParallelDebug.h"
#include "MeshDistributor.h"
-#include "PartitionElementData.h"
#include "ProblemVec.h"
#include "DOFVector.h"
#include "FixVec.h"
@@ -318,33 +317,44 @@ namespace AMDiS {
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
}
+
void ParallelDebug::testAllElements(MeshDistributor &pdb)
{
FUNCNAME("ParallelDebug::testAllElements()");
+ std::set<int> macroElements;
+ int minElementIndex = std::numeric_limits<int>::max();
+ int maxElementIndex = std::numeric_limits<int>::min();
+
TraverseStack stack;
- ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
+ ElInfo *elInfo = stack.traverseFirst(pdb.mesh, 0, Mesh::CALL_EL_LEVEL);
while (elInfo) {
- Element *element = elInfo->getElement();
- PartitionElementData *partitionData =
- dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
+ int elIndex = elInfo->getElement()->getIndex();
+ minElementIndex = std::min(minElementIndex, elIndex);
+ maxElementIndex = std::max(maxElementIndex, elIndex);
+ macroElements.insert(elInfo->getElement()->getIndex());
+ elInfo = stack.traverseNext(elInfo);
+ }
+
+ int globalMinIndex, globalMaxIndex;
+ pdb.mpiComm.Allreduce(&minElementIndex, &globalMinIndex, 1, MPI_INT, MPI_MIN);
+ pdb.mpiComm.Allreduce(&maxElementIndex, &globalMaxIndex, 1, MPI_INT, MPI_MAX);
- int sendId = 0;
- if (partitionData->getPartitionStatus() == IN)
- sendId = 1;
+ TEST_EXIT(globalMinIndex == 0)("No macro element with index 0!\n");
+ for (int i = 0; i <= globalMaxIndex; i++) {
+ int sendId = macroElements.count(i);
int recvId = 0;
pdb.mpiComm.Allreduce(&sendId, &recvId, 1, MPI_INT, MPI_SUM);
if (recvId != 1 && pdb.mpiRank == 0) {
- if (recvId == 0)
- ERROR_EXIT("Element %d has no member partition!\n", element->getIndex());
+ if (recvId == 0) {
+ ERROR_EXIT("Element %d has no member partition!\n", i);
+ }
- if (recvId > 1)
- ERROR_EXIT("Element %d is member of more than pne partition!\n",
- element->getIndex());
+ if (recvId > 1) {
+ ERROR_EXIT("Element %d is member of more than pne partition!\n", i);
+ }
}
-
- elInfo = stack.traverseNext(elInfo);
}
}
--
GitLab