Commit 4ff7d475 authored by Thomas Witkowski's avatar Thomas Witkowski

First real Zoltan implementation for parallel mesh partitioning.

parent 3b5f871b
......@@ -17,6 +17,7 @@ if USE_PARALLEL_DOMAIN_AMDIS
PARALLEL_AMDIS_SOURCES += \
$(SOURCE_DIR)/parallel/StdMpi.h $(SOURCE_DIR)/parallel/StdMpi.cc \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.h $(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
$(SOURCE_DIR)/parallel/ZoltanPartitioner.h $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshDistributor.h $(SOURCE_DIR)/parallel/MeshDistributor.cc \
$(SOURCE_DIR)/parallel/MeshPartitioner.h $(SOURCE_DIR)/parallel/MeshPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
......@@ -28,6 +29,7 @@ if USE_PARALLEL_DOMAIN_AMDIS
$(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1
AMDIS_INCLUDES += -I$(PETSC_DIR)/include -I$(PETSC_DIR)/$(PETSC_ARCH)/include
AMDIS_INCLUDES += -I$(LIB_DIR)/zoltan_build/include
endif
if ENABLE_UMFPACK
......
......@@ -39,6 +39,7 @@ host_triplet = @host@
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_1 = \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/StdMpi.h $(SOURCE_DIR)/parallel/StdMpi.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ParMetisPartitioner.h $(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ZoltanPartitioner.h $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshDistributor.h $(SOURCE_DIR)/parallel/MeshDistributor.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshPartitioner.h $(SOURCE_DIR)/parallel/MeshPartitioner.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/MeshManipulation.h $(SOURCE_DIR)/parallel/MeshManipulation.cc \
......@@ -50,7 +51,9 @@ host_triplet = @host@
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(SOURCE_DIR)/parallel/ElementObjectData.h $(SOURCE_DIR)/parallel/ElementObjectData.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -I$(PETSC_DIR)/include -I$(PETSC_DIR)/$(PETSC_ARCH)/include
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -I$(PETSC_DIR)/include \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ -I$(PETSC_DIR)/$(PETSC_ARCH)/include \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ -I$(LIB_DIR)/zoltan_build/include
@ENABLE_UMFPACK_TRUE@am__append_4 = -DHAVE_UMFPACK=1 -DMTL_HAS_UMFPACK
@ENABLE_UMFPACK_TRUE@am__append_5 = -I$(LIB_DIR)/UFconfig \
@ENABLE_UMFPACK_TRUE@ -I$(LIB_DIR)/AMD/Include \
......@@ -84,6 +87,8 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
$(SOURCE_DIR)/parallel/StdMpi.cc \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.h \
$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc \
$(SOURCE_DIR)/parallel/ZoltanPartitioner.h \
$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc \
$(SOURCE_DIR)/parallel/MeshDistributor.h \
$(SOURCE_DIR)/parallel/MeshDistributor.cc \
$(SOURCE_DIR)/parallel/MeshPartitioner.h \
......@@ -254,6 +259,7 @@ am__libamdis_la_SOURCES_DIST = $(SOURCE_DIR)/parallel/StdMpi.h \
$(SOURCE_DIR)/time/RosenbrockMethod.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-StdMpi.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParMetisPartitioner.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ZoltanPartitioner.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshDistributor.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshPartitioner.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-MeshManipulation.lo \
......@@ -861,6 +867,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-VtkWriter.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZeroOrderAssembler.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZeroOrderTerm.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ZoltanPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CFE_Integration.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CFE_NormAndErrorFcts.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcompositeFEM_la-CompositeFEMMethods.Plo@am__quote@
......@@ -906,6 +913,13 @@ libamdis_la-ParMetisPartitioner.lo: $(SOURCE_DIR)/parallel/ParMetisPartitioner.c
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParMetisPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ParMetisPartitioner.cc
libamdis_la-ZoltanPartitioner.lo: $(SOURCE_DIR)/parallel/ZoltanPartitioner.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ZoltanPartitioner.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo" -c -o libamdis_la-ZoltanPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo" "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ZoltanPartitioner.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' object='libamdis_la-ZoltanPartitioner.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ZoltanPartitioner.lo `test -f '$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/ZoltanPartitioner.cc
libamdis_la-MeshDistributor.lo: $(SOURCE_DIR)/parallel/MeshDistributor.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-MeshDistributor.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo" -c -o libamdis_la-MeshDistributor.lo `test -f '$(SOURCE_DIR)/parallel/MeshDistributor.cc' || echo '$(srcdir)/'`$(SOURCE_DIR)/parallel/MeshDistributor.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo" "$(DEPDIR)/libamdis_la-MeshDistributor.Plo"; else rm -f "$(DEPDIR)/libamdis_la-MeshDistributor.Tpo"; exit 1; fi
......
......@@ -39,6 +39,7 @@ endif
MPCCI_LIB = -L$(MPCCI_DIR)/lib/linux-x86-glibc22 -lmpcci
PARMETIS_LIB = -L$(PARMETIS_DIR) -lparmetis -lmetis
ZOLTAN_LIB = -L$(AMDIS_DIR)/lib/zoltan_build/lib -lzoltan
LIBS += $(AMDIS_LIB) $(PNG_LIB)
LIBS += -lboost_iostreams -lboost_filesystem -lboost_system -lboost_date_time
......@@ -89,6 +90,7 @@ ifeq ($(strip $(USE_PARALLEL_AMDIS)), 1)
CPPFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS
INCLUDES += $(PETSC_INCLUDE)
LIBS += $(PARMETIS_LIB) -lmpi $(PETSC_LIB)
LIBS += $(ZOLTAN_LIB)
else
ifeq ($(strip $(USE_COMPILER)), gcc)
COMPILE = g++
......
......@@ -294,13 +294,13 @@ namespace AMDiS {
}
/// Sets ElInfo's \ref coord[i].
inline void setCoord(int i,WorldVector<double>& c)
inline void setCoord(int i, WorldVector<double>& c)
{
coord[i] = c;
}
/// Sets ElInfo's \ref coord.
inline void setCoords(FixVec<WorldVector<double>,VERTEX >& c)
inline void setCoords(FixVec<WorldVector<double>, VERTEX >& c)
{
coord = c;
}
......@@ -412,7 +412,7 @@ namespace AMDiS {
virtual double calcGrdLambda(DimVec<WorldVector<double> >& grd_lam) = 0;
/** \brief
* calculates a normal of the given side (1d,2d: edge, 3d: face) of \ref element.
* calculates a normal of the given side (1d, 2d: edge, 3d: face) of \ref element.
* Returns the absolute value of the determinant of the
* transformation to the reference element.
* pure virtual => must be overriden in sub-class.
......
......@@ -17,6 +17,7 @@
#include <stdint.h>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <zoltan_cpp.h>
#include "parallel/MeshDistributor.h"
#include "parallel/MeshManipulation.h"
......@@ -24,6 +25,7 @@
#include "parallel/StdMpi.h"
#include "parallel/MeshPartitioner.h"
#include "parallel/ParMetisPartitioner.h"
#include "parallel/ZoltanPartitioner.h"
#include "parallel/MpiHelper.h"
#include "io/ElementFileWriter.h"
#include "io/MacroInfo.h"
......@@ -94,7 +96,8 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->log main rank", "%d", &tmp);
Msg::outputMainRank = (tmp > 0);
partitioner = new ParMetisPartitioner(&mpiComm);
// partitioner = new ParMetisPartitioner(&mpiComm);
partitioner = new ZoltanPartitioner(&mpiComm);
}
......@@ -108,6 +111,11 @@ namespace AMDiS {
TEST_EXIT(feSpace)("No FE space has been defined for the mesh distributor!\n");
TEST_EXIT(mesh)("No mesh has been defined for the mesh distributor!\n");
int a;
char *b;
float zoltanVersion;
Zoltan_Initialize(a, &b, &zoltanVersion);
elObjects.setFeSpace(feSpace);
// If the problem has been already read from a file, we need only to set
......
......@@ -77,6 +77,11 @@ namespace AMDiS {
mapLocalGlobal = m;
}
Mesh *getMesh()
{
return mesh;
}
map<int, bool>& getElementInRank()
{
return elementInRank;
......
......@@ -11,6 +11,8 @@
#include <queue>
#include <mpi.h>
#include "parallel/ParMetisPartitioner.h"
#include "parallel/MpiHelper.h"
#include "Serializer.h"
......@@ -20,7 +22,7 @@
#include "Element.h"
#include "FixVec.h"
#include "DOFVector.h"
#include "mpi.h"
namespace AMDiS {
......@@ -542,7 +544,7 @@ namespace AMDiS {
it != elementInRank.end(); ++it)
elementInRank[it->first] = false;
// Create map which stores for each element index on ther partitioning level
// Create map which stores for each element index on macro level
// if the element is in the partition of this rank.
recvElements.clear();
for (int i = 0; i < mpiSize; i++) {
......
......@@ -20,16 +20,17 @@
/** \file ParMetisPartitioner.h */
#ifndef AMDIS_PARMETISADAPTER_H
#define AMDIS_PARMETISADAPTER_H
#ifndef AMDIS_PARMETIS_PARTITIONER_H
#define AMDIS_PARMETIS_PARTITIONER_H
#include <map>
#include <set>
#include <parmetis.h>
#include <mpi.h>
#include "AMDiS_fwd.h"
#include "Global.h"
#include "parallel/MeshPartitioner.h"
#include "parmetis.h"
#include "mpi.h"
namespace AMDiS {
......@@ -174,10 +175,11 @@ namespace AMDiS {
~ParMetisPartitioner();
/// \ref MeshPartitioner::partition
bool partition(map<int, double> &elemWeights,
PartitionMode mode = INITIAL);
/// Creates a map which stores for each element the rank that owns this element.
/// \ref MeshPartitioner::getPartitionMap
void getPartitionMap(map<int, int> &partitionMap);
void setItr(float value)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment