Commit edb60cbf authored by Thomas Witkowski's avatar Thomas Witkowski

* Make AMDiS more parallel :)

parent 6bd7a092
......@@ -16,6 +16,7 @@ if USE_PARALLEL_AMDIS
$(PARALLEL_DIR)/MeshStructure_ED.h \
$(PARALLEL_DIR)/ParallelError.h $(PARALLEL_DIR)/ParallelError.hh \
$(PARALLEL_DIR)/ParallelProblem.h $(PARALLEL_DIR)/ParallelProblem.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc \
$(PARALLEL_DIR)/ParMetisPartitioner.h $(PARALLEL_DIR)/ParMetisPartitioner.cc \
$(PARALLEL_DIR)/PartitionElementData.h
PARALLEL_INCLUDES = -I$(MPI_DIR)/include -I$(PARMETIS_DIR)
......
......@@ -75,6 +75,8 @@ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ConditionalEstimator.h \
$(PARALLEL_DIR)/ParallelError.hh \
$(PARALLEL_DIR)/ParallelProblem.h \
$(PARALLEL_DIR)/ParallelProblem.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.h \
$(PARALLEL_DIR)/ParallelDomainProblem.cc \
$(PARALLEL_DIR)/ParMetisPartitioner.h \
$(PARALLEL_DIR)/ParMetisPartitioner.cc \
$(PARALLEL_DIR)/PartitionElementData.h \
......@@ -258,6 +260,7 @@ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ConditionalEstimator.h \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-MeshStructure.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParallelProblem.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParallelDomainProblem.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParMetisPartitioner.lo
am_libamdis_la_OBJECTS = $(am__objects_1) \
libamdis_la-MultiGridPreconWrapper.lo \
......@@ -490,6 +493,7 @@ AMDIS_INCLUDES = -I$(SOURCE_DIR) $(am__append_2) $(am__append_5)
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/MeshStructure_ED.h \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelError.h $(PARALLEL_DIR)/ParallelError.hh \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelProblem.h $(PARALLEL_DIR)/ParallelProblem.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParMetisPartitioner.h $(PARALLEL_DIR)/ParMetisPartitioner.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/PartitionElementData.h
......@@ -836,6 +840,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@
......@@ -932,6 +937,13 @@ libamdis_la-ParallelProblem.lo: $(PARALLEL_DIR)/ParallelProblem.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelProblem.lo `test -f '$(PARALLEL_DIR)/ParallelProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelProblem.cc
libamdis_la-ParallelDomainProblem.lo: $(PARALLEL_DIR)/ParallelDomainProblem.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainProblem.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainProblem.cc' object='libamdis_la-ParallelDomainProblem.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc
libamdis_la-ParMetisPartitioner.lo: $(PARALLEL_DIR)/ParMetisPartitioner.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParMetisPartitioner.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo" -c -o libamdis_la-ParMetisPartitioner.lo `test -f '$(PARALLEL_DIR)/ParMetisPartitioner.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParMetisPartitioner.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo" "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo"; exit 1; fi
......
......@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......
#include "ParallelDomainProblem.h"
#include "ProblemScal.h"
#include "ProblemInstat.h"
#include "ParMetisPartitioner.h"
#include "Mesh.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
#include "MacroElement.h"
#include "PartitionElementData.h"
namespace AMDiS {
ParallelDomainProblemBase::ParallelDomainProblemBase(const std::string& name,
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
Mesh *m)
: mesh(m)
{
mpiRank = MPI::COMM_WORLD.Get_rank();
mpiSize = MPI::COMM_WORLD.Get_size();
mpiComm = MPI::COMM_WORLD;
partitioner = new ParMetisPartitioner(mesh, &mpiComm);
}
void ParallelDomainProblemBase::initParallelization(AdaptInfo *adaptInfo)
{
if (mpiSize <= 1)
return;
// create an initial partitioning of the mesh
partitioner->createPartitionData();
// set the element weights, which are 1 at the very first begin
setElemWeights(adaptInfo);
// and now partition the mesh
partitionMesh(adaptInfo);
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
int nLeaves = 0;
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) {
element->setHidden(true);
} else {
nLeaves++;
}
elInfo = stack.traverseNext(elInfo);
}
mesh->setNumberOfLeaves(nLeaves);
}
void ParallelDomainProblemBase::exitParallelization(AdaptInfo *adaptInfo)
{
}
double ParallelDomainProblemBase::setElemWeights(AdaptInfo *adaptInfo)
{
double localWeightSum = 0.0;
int elNum = -1;
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
Element *element = elInfo->getElement();
// get partition data
PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData && partitionData->getPartitionStatus() == IN) {
if (partitionData->getLevel() == 0) {
elNum = element->getIndex();
}
TEST_EXIT(elNum != -1)("invalid element number\n");
if (element->isLeaf()) {
elemWeights[elNum] += 1.0;
localWeightSum += 1.0;
}
}
elInfo = stack.traverseNext(elInfo);
}
return localWeightSum;
}
void ParallelDomainProblemBase::partitionMesh(AdaptInfo *adaptInfo)
{
if (initialPartitionMesh) {
initialPartitionMesh = false;
partitioner->fillCoarsePartitionVec(&oldPartitionVec);
partitioner->partition(&elemWeights, INITIAL);
} else {
oldPartitionVec = partitionVec;
partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
}
partitioner->fillCoarsePartitionVec(&partitionVec);
}
ParallelDomainProblemScal::ParallelDomainProblemScal(const std::string& name,
ProblemScal *problem,
ProblemInstatScal *problemInstat)
: ParallelDomainProblemBase(name, problem, problemInstat, problem->getMesh())
{
}
}
// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// ============================================================================
// == ==
// == crystal growth group ==
// == ==
// == Stiftung caesar ==
// == Ludwig-Erhard-Allee 2 ==
// == 53175 Bonn ==
// == germany ==
// == ==
// ============================================================================
// == ==
// == http://www.caesar.de/cg/AMDiS ==
// == ==
// ============================================================================
/** \file ParallelDomainProblem.h */
#ifndef AMDIS_PARALLELDOMAINPROBLEM_H
#define AMDIS_PARALLELDOMAINPROBLEM_H
#include <map>
#include <vector>
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "AdaptInfo.h"
#include "mpi.h"
namespace AMDiS {
class ProblemScal;
class ProblemInstatScal;
class ParMetisPartitioner;
class Mesh;
class ParallelDomainProblemBase : public ProblemIterationInterface,
public ProblemTimeInterface
{
public:
ParallelDomainProblemBase(const std::string& name,
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
Mesh *mesh);
virtual ~ParallelDomainProblemBase() {}
void initParallelization(AdaptInfo *adaptInfo);
void exitParallelization(AdaptInfo *adaptInfo);
/// Set for each element on the partitioning level the number of leaf elements.
double setElemWeights(AdaptInfo *adaptInfo);
void partitionMesh(AdaptInfo *adaptInfo);
virtual void setTime(AdaptInfo *adaptInfo) {}
virtual void initTimestep(AdaptInfo *adaptInfo) {}
virtual void closeTimestep(AdaptInfo *adaptInfo) {}
virtual void solveInitialProblem(AdaptInfo *adaptInfo) {}
virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}
virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) {}
virtual int getNumProblems() {}
inline virtual const std::string& getName() {
return name;
}
virtual ProblemStatBase *getProblem(int number = 0) {}
virtual void serialize(std::ostream&) {}
virtual void deserialize(std::istream&) {}
protected:
/// The rank of the current process.
int mpiRank;
/// Overall number of processes.
int mpiSize;
/** \brief
* MPI communicator collected all processes, which should
* be used for calculation. The Debug procces is not included
* in this communicator.
*/
MPI::Intracomm mpiComm;
/// Name of the problem (as used in the init files)
std::string name;
/// Mesh of the problem.
Mesh *mesh;
/// Pointer to the paritioner which is used to devide a mesh into partitions.
ParMetisPartitioner *partitioner;
/// Weights for the elements, i.e., the number of leaf elements within this element.
std::map<int, double> elemWeights;
/// Is true, if the mesh was not partitioned before, otherwise it's false.
bool initialPartitionMesh;
/** \brief
* Stores to every coarse element index the number of the partition it
* corresponds to.
*/
std::map<int, int> partitionVec;
/** \brief
* Stores an old partitioning of elements. To every element index the number
* of the parition it corresponds to is stored.
*/
std::map<int, int> oldPartitionVec;
};
class ParallelDomainProblemScal : public ParallelDomainProblemBase
{
public:
ParallelDomainProblemScal(const std::string& name,
ProblemScal *problem,
ProblemInstatScal *problemInstat);
};
}
#endif // AMDIS_PARALLELDOMAINPROBLEM_H
......@@ -190,12 +190,12 @@ namespace AMDiS {
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
std::vector<DOFVector<double>*> vectors,
Mesh *mesh,
Mesh *mesh_,
RefinementManager *rm,
CoarseningManager *cm)
: ParallelProblemBase(name, iterationIF, timeIF),
name_(name),
mesh(mesh),
mesh(mesh_),
refinementManager(rm),
coarseningManager(cm),
repartitionSteps_(1),
......@@ -1053,8 +1053,7 @@ namespace AMDiS {
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh,
-1,
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
Element *element = elInfo->getElement();
......@@ -1359,29 +1358,6 @@ namespace AMDiS {
// and now partition the mesh
partitionMesh(adaptInfo);
#if 0
// 2009-03-31 THOMAS, DO NOT REMOVE THIS BLOCK
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
int nLeaves = 0;
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) {
element->setHidden(true);
} else {
nLeaves++;
}
elInfo = stack.traverseNext(elInfo);
}
mesh->setNumberOfLeaves(nLeaves);
#endif
#if 1
globalRefineOutOfPartition(adaptInfo);
refineOverlap(adaptInfo);
......@@ -1429,8 +1405,6 @@ namespace AMDiS {
std::string(number) + "_");
(*fwIt)->setTraverseProperties(-1, 0, elementInPartition);
}
#endif
}
}
......
......@@ -22,23 +22,18 @@
#ifndef AMDIS_PARALLELPROBLEM_H
#define AMDIS_PARALLELPROBLEM_H
#include <map>
#include <vector>
#include <set>
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "AdaptInfo.h"
#include "mpi.h"
#include <map>
#include <vector>
#include <set>
namespace AMDiS {
// =========================================================================
// ===== class ParallelProblemInterface ====================================
// =========================================================================
/** \brief
* Interface for parallel problems
*/
/// Interface for parallel problems
class ParallelProblemInterface
{
public:
......@@ -47,13 +42,7 @@ namespace AMDiS {
virtual void exitParallelization(AdaptInfo *adaptInfo) = 0;
};
// =========================================================================
// ===== class ParallelProblemBase =========================================
// =========================================================================
/** \brief
* Schablonen Klasse
*/
/// Schablonen Klasse
class ParallelProblemBase : public ParallelProblemInterface,
public ProblemIterationInterface,
public ProblemTimeInterface
......@@ -88,19 +77,10 @@ namespace AMDiS {
virtual void coarsenOutOfPartition(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void synchronizeMeshes(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void exchangeRankSolutions(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void buildGlobalSolution(AdaptInfo *adaptInfo) = 0;
virtual void exitParallelization(AdaptInfo *adaptInfo);
......@@ -160,14 +140,10 @@ namespace AMDiS {
double partitioningTime;
/** \brief
* The rank of the current process.
*/
/// The rank of the current process.
int mpiRank;
/** \brief
* Overall number of processes.
*/
/// Overall number of processes.
int mpiSize;
/** \brief
......@@ -177,16 +153,10 @@ namespace AMDiS {
*/
MPI::Intracomm mpiComm;
/** \brief
* The MPI group, which is used for the communicator
* \ref mpiComm.
*/
/// The MPI group, which is used for the communicator \ref mpiComm.
MPI::Group amdisGroup;
/** \brief
* Defines the debug mode. If it is 1, a debug server will be started
* on rank 0.
*/
/// Defines the debug mode. If it is 1, a debug server will be started on rank 0.
int debugMode;
/** \brief
......@@ -328,74 +298,32 @@ namespace AMDiS {
///
int repartitionSteps_;
/** \brief
*
*/
bool puEveryTimestep_;
/** \brief
*
*/
std::vector<DOFVector<double>*> dofVectors_;
/** \brief
*
*/
double upperPartThreshold_;
/** \brief
*
*/
double lowerPartThreshold_;
/** \brief
*
*/
int globalCoarseGridLevel_;
/** \brief
*
*/
int localCoarseGridLevel_;
/** \brief
*
*/
int globalRefinements_;
/** \brief
*
*/
std::map<Element*, int> overlapDistance_;
/** \brief
*
*/
int adaptiveThresholds_;
/** \brief
*
*/
double thresholdIncFactor_;
/** \brief
*
*/
double thresholdDecFactor_;
/** \brief
*
*/
double repartTimeFactor_;
/** \brief
*
*/
double minUpperTH_;
/** \brief
*
*/
double maxLowerTH_;
};
......@@ -458,19 +386,13 @@ namespace AMDiS {
Marker *oldMarker;
/** \brief
* Vector of all process' solution DOFVectors.
*/
/// Vector of all process' solution DOFVectors.
std::vector<DOFVector<double>*> rankSolution;
/** \brief
*
*/
///
Estimator *usersEstimator;
/** \brief
*
*/
///
Marker *usersMarker;
};
......@@ -527,41 +449,28 @@ namespace AMDiS {
virtual void debugFunction(AdaptInfo *adaptInfo) {}
protected:
///
ProblemVec *problem;
/** \brief
*
*/
///
ProblemInstatVec *problemInstat_;
/** \brief
*
*/
///
std::vector<Estimator*> oldEstimator;
/** \brief
*
*/
///
std::vector<Marker*> oldMarker;
/** \brief
* Vector of all process' solution SystemVectors.
*/
/// Vector of all process' solution SystemVectors.
std::vector<SystemVector*> rankSolution;
/** \brief
*
*/
///
std::vector<Estimator*> usersEstimator;
/** \brief
*
*/
///
std::vector<Marker*> usersMarker;
/** \brief
* Number of components of the vectorial problem.
*/
/// Number of components of the vectorial problem.
int nComponents;
};
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment