From 4ff7d475480289ca9e370257890a10ae16c7e978 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Mon, 24 Jan 2011 14:50:10 +0000
Subject: [PATCH] First real Zoltan implementation for parallel mesh
partitioning.
---
AMDiS/bin/Makefile.am | 2 ++
AMDiS/bin/Makefile.in | 16 +++++++++++++++-
AMDiS/other/include/Makefile_AMDiS.mk | 2 ++
AMDiS/src/ElInfo.h | 6 +++---
AMDiS/src/parallel/MeshDistributor.cc | 10 +++++++++-
AMDiS/src/parallel/MeshPartitioner.h | 5 +++++
AMDiS/src/parallel/ParMetisPartitioner.cc | 6 ++++--
AMDiS/src/parallel/ParMetisPartitioner.h | 12 +++++++-----
8 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/AMDiS/bin/Makefile.am b/AMDiS/bin/Makefile.am
index bc35ae7a..7cf79d50 100644
--- a/AMDiS/bin/Makefile.am
+++ b/AMDiS/bin/Makefile.am
@@ -17,6 +17,7 @@ if USE_PARALLEL_DOMAIN_AMDIS
PARALLEL_AMDIS_SOURCES += \
$(SOURCE_DIR)/parallel/StdMpi.h $(SOURCE_DIR)/parallel/StdMpi.cc \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.h $(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
+ $(SOURCE_DIR)/parallel/ZoltanPartitioner.h $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshDistributor.h $(SOURCE_DIR)/parallel/MeshDistributor.cc \
$(SOURCE_DIR)/parallel/MeshPartitioner.h $(SOURCE_DIR)/parallel/MeshPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
@@ -28,6 +29,7 @@ if USE_PARALLEL_DOMAIN_AMDIS
$(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1
AMDIS_INCLUDES += -I$(PETSC_DIR)/include -I$(PETSC_DIR)/$(PETSC_ARCH)/include
+ AMDIS_INCLUDES += -I$(LIB_DIR)/zoltan_build/include
endif
if ENABLE_UMFPACK
diff --git a/AMDiS/bin/Makefile.in b/AMDiS/bin/Makefile.in
index d42f6643..2cb803e6 100644
--- a/AMDiS/bin/Makefile.in
+++ b/AMDiS/bin/Makefile.in
@@ -39,6 +39,7 @@ host_triplet = @host@
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_1 = \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/StdMpi.h $(SOURCE_DIR)/parallel/StdMpi.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ParMetisPartitioner.h $(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ZoltanPartitioner.h $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshDistributor.h $(SOURCE_DIR)/parallel/MeshDistributor.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshPartitioner.h $(SOURCE_DIR)/parallel/MeshPartitioner.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
@@ -50,7 +51,9 @@ host_triplet = @host@
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1
-@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -I$(PETSC_DIR)/include -I$(PETSC_DIR)/$(PETSC_ARCH)/include
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -I$(PETSC_DIR)/include \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ -I$(PETSC_DIR)/$(PETSC_ARCH)/include \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ -I$(LIB_DIR)/zoltan_build/include
@ENABLE_UMFPACK_TRUE@am__append_4 = -DHAVE_UMFPACK=1 -DMTL_HAS_UMFPACK
@ENABLE_UMFPACK_TRUE@am__append_5 = -I$(LIB_DIR)/UFconfig \
@ENABLE_UMFPACK_TRUE@ -I$(LIB_DIR)/AMD/Include \
@@ -84,6 +87,8 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
$(SOURCE_DIR)/parallel/StdMpi.cc \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.h \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
+ $(SOURCE_DIR)/parallel/ZoltanPartitioner.h \
+ $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshDistributor.h \
$(SOURCE_DIR)/parallel/MeshDistributor.cc \
$(SOURCE_DIR)/parallel/MeshPartitioner.h \
@@ -254,6 +259,7 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
$(SOURCE_DIR)/time/RosenbrockMethod.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-StdMpi.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParMetisPartitioner.lo \
+@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ZoltanPartitioner.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshDistributor.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshPartitioner.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshManipulation.lo \
@@ -861,6 +867,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-VtkWriter.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZeroOrderAssembler.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZeroOrderTerm.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZoltanPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CFE_Integration.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CFE_NormAndErrorFcts.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CompositeFEMMethods.Plo@am__quote@
@@ -906,6 +913,13 @@ libamdis_la-ParMetisPartitioner.lo: $(SOURCE_DIR)/parallel/ParMetisPartitioner.c
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParMetisPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc
+libamdis_la-ZoltanPartitioner.lo: $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc
+@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ZoltanPartitioner.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo" -c -o libamdis_la-ZoltanPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc; \
+@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo" "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' object='libamdis_la-ZoltanPartitioner.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ZoltanPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc
+
libamdis_la-MeshDistributor.lo: $(SOURCE_DIR)/parallel/MeshDistributor.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-MeshDistributor.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo" -c -o libamdis_la-MeshDistributor.lo `test -f '$(SOURCE_DIR)/parallel/MeshDistributor.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/MeshDistributor.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo" "$(DEPDIR)/libamdis_la-MeshDistributor.Plo"; else rm -f "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo"; exit 1; fi
diff --git a/AMDiS/other/include/Makefile_AMDiS.mk b/AMDiS/other/include/Makefile_AMDiS.mk
index 6707d314..87e1eae3 100644
--- a/AMDiS/other/include/Makefile_AMDiS.mk
+++ b/AMDiS/other/include/Makefile_AMDiS.mk
@@ -39,6 +39,7 @@ endif
MPCCI_LIB = -L$(MPCCI_DIR)/lib/linux-x86-glibc22 -lmpcci
PARMETIS_LIB = -L$(PARMETIS_DIR) -lparmetis -lmetis
+ZOLTAN_LIB = -L$(AMDIS_DIR)/lib/zoltan_build/lib -lzoltan
LIBS += $(AMDIS_LIB) $(PNG_LIB)
LIBS += -lboost_iostreams -lboost_filesystem -lboost_system -lboost_date_time
@@ -89,6 +90,7 @@ ifeq ($(strip $(USE_PARALLEL_AMDIS)), 1)
CPPFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS
INCLUDES += $(PETSC_INCLUDE)
LIBS += $(PARMETIS_LIB) -lmpi $(PETSC_LIB)
+ LIBS += $(ZOLTAN_LIB)
else
ifeq ($(strip $(USE_COMPILER)), gcc)
COMPILE = g++
diff --git a/AMDiS/src/ElInfo.h b/AMDiS/src/ElInfo.h
index e722b6e8..a811dcda 100644
--- a/AMDiS/src/ElInfo.h
+++ b/AMDiS/src/ElInfo.h
@@ -294,13 +294,13 @@ namespace AMDiS {
}
/// Sets ElInfo's \ref coord[i].
- inline void setCoord(int i,WorldVector<double>& c)
+ inline void setCoord(int i, WorldVector<double>& c)
{
coord[i] = c;
}
/// Sets ElInfo's \ref coord.
- inline void setCoords(FixVec<WorldVector<double>,VERTEX >& c)
+ inline void setCoords(FixVec<WorldVector<double>, VERTEX >& c)
{
coord = c;
}
@@ -412,7 +412,7 @@ namespace AMDiS {
virtual double calcGrdLambda(DimVec<WorldVector<double> >& grd_lam) = 0;
/** \brief
- * calculates a normal of the given side (1d,2d: edge, 3d: face) of \ref element.
+ * calculates a normal of the given side (1d, 2d: edge, 3d: face) of \ref element.
* Returns the absolute value of the determinant of the
* transformation to the reference element.
* pure virtual => must be overriden in sub-class.
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 132595e2..f9b9cedf 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -17,6 +17,7 @@
#include <stdint.h>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
+#include <zoltan_cpp.h>
#include "parallel/MeshDistributor.h"
#include "parallel/MeshManipulation.h"
@@ -24,6 +25,7 @@
#include "parallel/StdMpi.h"
#include "parallel/MeshPartitioner.h"
#include "parallel/ParMetisPartitioner.h"
+#include "parallel/ZoltanPartitioner.h"
#include "parallel/MpiHelper.h"
#include "io/ElementFileWriter.h"
#include "io/MacroInfo.h"
@@ -94,7 +96,8 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->log main rank", "%d", &tmp);
Msg::outputMainRank = (tmp > 0);
- partitioner = new ParMetisPartitioner(&mpiComm);
+ // partitioner = new ParMetisPartitioner(&mpiComm);
+ partitioner = new ZoltanPartitioner(&mpiComm);
}
@@ -108,6 +111,11 @@ namespace AMDiS {
TEST_EXIT(feSpace)("No FE space has been defined for the mesh distributor!\n");
TEST_EXIT(mesh)("No mesh has been defined for the mesh distributor!\n");
+ int a;
+ char *b;
+ float zoltanVersion;
+ Zoltan_Initialize(a, &b, &zoltanVersion);
+
elObjects.setFeSpace(feSpace);
// If the problem has been already read from a file, we need only to set
diff --git a/AMDiS/src/parallel/MeshPartitioner.h b/AMDiS/src/parallel/MeshPartitioner.h
index a288062c..1b410280 100644
--- a/AMDiS/src/parallel/MeshPartitioner.h
+++ b/AMDiS/src/parallel/MeshPartitioner.h
@@ -77,6 +77,11 @@ namespace AMDiS {
mapLocalGlobal = m;
}
+ Mesh *getMesh()
+ {
+ return mesh;
+ }
+
map<int, bool>& getElementInRank()
{
return elementInRank;
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.cc b/AMDiS/src/parallel/ParMetisPartitioner.cc
index c59a85fe..8676a8b1 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.cc
+++ b/AMDiS/src/parallel/ParMetisPartitioner.cc
@@ -11,6 +11,8 @@
#include <queue>
+#include <mpi.h>
+
#include "parallel/ParMetisPartitioner.h"
#include "parallel/MpiHelper.h"
#include "Serializer.h"
@@ -20,7 +22,7 @@
#include "Element.h"
#include "FixVec.h"
#include "DOFVector.h"
-#include "mpi.h"
+
namespace AMDiS {
@@ -542,7 +544,7 @@ namespace AMDiS {
it != elementInRank.end(); ++it)
elementInRank[it->first] = false;
- // Create map which stores for each element index on ther partitioning level
+ // Create map which stores for each element index on macro level
// if the element is in the partition of this rank.
recvElements.clear();
for (int i = 0; i < mpiSize; i++) {
diff --git a/AMDiS/src/parallel/ParMetisPartitioner.h b/AMDiS/src/parallel/ParMetisPartitioner.h
index 3b11c09a..092b3daa 100644
--- a/AMDiS/src/parallel/ParMetisPartitioner.h
+++ b/AMDiS/src/parallel/ParMetisPartitioner.h
@@ -20,16 +20,17 @@
/** \file ParMetisPartitioner.h */
-#ifndef AMDIS_PARMETISADAPTER_H
-#define AMDIS_PARMETISADAPTER_H
+#ifndef AMDIS_PARMETIS_PARTITIONER_H
+#define AMDIS_PARMETIS_PARTITIONER_H
#include <map>
#include <set>
+#include <parmetis.h>
+#include <mpi.h>
+
#include "AMDiS_fwd.h"
#include "Global.h"
#include "parallel/MeshPartitioner.h"
-#include "parmetis.h"
-#include "mpi.h"
namespace AMDiS {
@@ -174,10 +175,11 @@ namespace AMDiS {
~ParMetisPartitioner();
+ /// \ref MeshPartitioner::partition
bool partition(map<int, double> &elemWeights,
PartitionMode mode = INITIAL);
- /// Creates a map which stores for each element the rank that owns this element.
+ /// \ref MeshPartitioner::getPartitionMap
void getPartitionMap(map<int, int> &partitionMap);
void setItr(float value)
--
GitLab