Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit edb60cbf authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

* Make AMDiS more parallel :)

parent 6bd7a092
......@@ -16,6 +16,7 @@ if USE_PARALLEL_AMDIS
$(PARALLEL_DIR)/MeshStructure_ED.h \
$(PARALLEL_DIR)/ParallelError.h $(PARALLEL_DIR)/ParallelError.hh \
$(PARALLEL_DIR)/ParallelProblem.h $(PARALLEL_DIR)/ParallelProblem.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc \
$(PARALLEL_DIR)/ParMetisPartitioner.h $(PARALLEL_DIR)/ParMetisPartitioner.cc \
$(PARALLEL_DIR)/PartitionElementData.h
PARALLEL_INCLUDES = -I$(MPI_DIR)/include -I$(PARMETIS_DIR)
......
......@@ -75,6 +75,8 @@ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ConditionalEstimator.h \
$(PARALLEL_DIR)/ParallelError.hh \
$(PARALLEL_DIR)/ParallelProblem.h \
$(PARALLEL_DIR)/ParallelProblem.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.h \
$(PARALLEL_DIR)/ParallelDomainProblem.cc \
$(PARALLEL_DIR)/ParMetisPartitioner.h \
$(PARALLEL_DIR)/ParMetisPartitioner.cc \
$(PARALLEL_DIR)/PartitionElementData.h \
......@@ -258,6 +260,7 @@ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ConditionalEstimator.h \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-MeshStructure.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParallelProblem.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParallelDomainProblem.lo \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ParMetisPartitioner.lo
am_libamdis_la_OBJECTS = $(am__objects_1) \
libamdis_la-MultiGridPreconWrapper.lo \
......@@ -490,6 +493,7 @@ AMDIS_INCLUDES = -I$(SOURCE_DIR) $(am__append_2) $(am__append_5)
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/MeshStructure_ED.h \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelError.h $(PARALLEL_DIR)/ParallelError.hh \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelProblem.h $(PARALLEL_DIR)/ParallelProblem.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/ParMetisPartitioner.h $(PARALLEL_DIR)/ParMetisPartitioner.cc \
@USE_PARALLEL_AMDIS_TRUE@ $(PARALLEL_DIR)/PartitionElementData.h
......@@ -836,6 +840,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@
......@@ -932,6 +937,13 @@ libamdis_la-ParallelProblem.lo: $(PARALLEL_DIR)/ParallelProblem.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelProblem.lo `test -f '$(PARALLEL_DIR)/ParallelProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelProblem.cc
libamdis_la-ParallelDomainProblem.lo: $(PARALLEL_DIR)/ParallelDomainProblem.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainProblem.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainProblem.cc' object='libamdis_la-ParallelDomainProblem.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc
libamdis_la-ParMetisPartitioner.lo: $(PARALLEL_DIR)/ParMetisPartitioner.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParMetisPartitioner.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo" -c -o libamdis_la-ParMetisPartitioner.lo `test -f '$(PARALLEL_DIR)/ParMetisPartitioner.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParMetisPartitioner.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo" "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParMetisPartitioner.Tpo"; exit 1; fi
......
......@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host p1s060:
# Libtool was configured on host p2s148:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......
#include "ParallelDomainProblem.h"
#include "ProblemScal.h"
#include "ProblemInstat.h"
#include "ParMetisPartitioner.h"
#include "Mesh.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
#include "MacroElement.h"
#include "PartitionElementData.h"
namespace AMDiS {
ParallelDomainProblemBase::ParallelDomainProblemBase(const std::string& name,
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
Mesh *m)
: mesh(m)
{
mpiRank = MPI::COMM_WORLD.Get_rank();
mpiSize = MPI::COMM_WORLD.Get_size();
mpiComm = MPI::COMM_WORLD;
partitioner = new ParMetisPartitioner(mesh, &mpiComm);
}
void ParallelDomainProblemBase::initParallelization(AdaptInfo *adaptInfo)
{
if (mpiSize <= 1)
return;
// create an initial partitioning of the mesh
partitioner->createPartitionData();
// set the element weights, which are 1 at the very first begin
setElemWeights(adaptInfo);
// and now partition the mesh
partitionMesh(adaptInfo);
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
int nLeaves = 0;
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) {
element->setHidden(true);
} else {
nLeaves++;
}
elInfo = stack.traverseNext(elInfo);
}
mesh->setNumberOfLeaves(nLeaves);
}
void ParallelDomainProblemBase::exitParallelization(AdaptInfo *adaptInfo)
{
}
double ParallelDomainProblemBase::setElemWeights(AdaptInfo *adaptInfo)
{
double localWeightSum = 0.0;
int elNum = -1;
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
Element *element = elInfo->getElement();
// get partition data
PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData && partitionData->getPartitionStatus() == IN) {
if (partitionData->getLevel() == 0) {
elNum = element->getIndex();
}
TEST_EXIT(elNum != -1)("invalid element number\n");
if (element->isLeaf()) {
elemWeights[elNum] += 1.0;
localWeightSum += 1.0;
}
}
elInfo = stack.traverseNext(elInfo);
}
return localWeightSum;
}
void ParallelDomainProblemBase::partitionMesh(AdaptInfo *adaptInfo)
{
if (initialPartitionMesh) {
initialPartitionMesh = false;
partitioner->fillCoarsePartitionVec(&oldPartitionVec);
partitioner->partition(&elemWeights, INITIAL);
} else {
oldPartitionVec = partitionVec;
partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
}
partitioner->fillCoarsePartitionVec(&partitionVec);
}
ParallelDomainProblemScal::ParallelDomainProblemScal(const std::string& name,
ProblemScal *problem,
ProblemInstatScal *problemInstat)
: ParallelDomainProblemBase(name, problem, problemInstat, problem->getMesh())
{
}
}
// ============================================================================
// == ==
// == AMDiS - Adaptive multidimensional simulations ==
// == ==
// ============================================================================
// == ==
// == crystal growth group ==
// == ==
// == Stiftung caesar ==
// == Ludwig-Erhard-Allee 2 ==
// == 53175 Bonn ==
// == germany ==
// == ==
// ============================================================================
// == ==
// == http://www.caesar.de/cg/AMDiS ==
// == ==
// ============================================================================
/** \file ParallelDomainProblem.h */
#ifndef AMDIS_PARALLELDOMAINPROBLEM_H
#define AMDIS_PARALLELDOMAINPROBLEM_H
#include <map>
#include <vector>
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "AdaptInfo.h"
#include "mpi.h"
namespace AMDiS {
class ProblemScal;
class ProblemInstatScal;
class ParMetisPartitioner;
class Mesh;
class ParallelDomainProblemBase : public ProblemIterationInterface,
public ProblemTimeInterface
{
public:
ParallelDomainProblemBase(const std::string& name,
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
Mesh *mesh);
virtual ~ParallelDomainProblemBase() {}
void initParallelization(AdaptInfo *adaptInfo);
void exitParallelization(AdaptInfo *adaptInfo);
/// Set for each element on the partitioning level the number of leaf elements.
double setElemWeights(AdaptInfo *adaptInfo);
void partitionMesh(AdaptInfo *adaptInfo);
virtual void setTime(AdaptInfo *adaptInfo) {}
virtual void initTimestep(AdaptInfo *adaptInfo) {}
virtual void closeTimestep(AdaptInfo *adaptInfo) {}
virtual void solveInitialProblem(AdaptInfo *adaptInfo) {}
virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}
virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) {}
virtual int getNumProblems() {}
inline virtual const std::string& getName() {
return name;
}
virtual ProblemStatBase *getProblem(int number = 0) {}
virtual void serialize(std::ostream&) {}
virtual void deserialize(std::istream&) {}
protected:
/// The rank of the current process.
int mpiRank;
/// Overall number of processes.
int mpiSize;
/** \brief
* MPI communicator collected all processes, which should
* be used for calculation. The Debug procces is not included
* in this communicator.
*/
MPI::Intracomm mpiComm;
/// Name of the problem (as used in the init files)
std::string name;
/// Mesh of the problem.
Mesh *mesh;
/// Pointer to the paritioner which is used to devide a mesh into partitions.
ParMetisPartitioner *partitioner;
/// Weights for the elements, i.e., the number of leaf elements within this element.
std::map<int, double> elemWeights;
/// Is true, if the mesh was not partitioned before, otherwise it's false.
bool initialPartitionMesh;
/** \brief
* Stores to every coarse element index the number of the partition it
* corresponds to.
*/
std::map<int, int> partitionVec;
/** \brief
* Stores an old partitioning of elements. To every element index the number
* of the parition it corresponds to is stored.
*/
std::map<int, int> oldPartitionVec;
};
class ParallelDomainProblemScal : public ParallelDomainProblemBase
{
public:
ParallelDomainProblemScal(const std::string& name,
ProblemScal *problem,
ProblemInstatScal *problemInstat);
};
}
#endif // AMDIS_PARALLELDOMAINPROBLEM_H
......@@ -190,12 +190,12 @@ namespace AMDiS {
ProblemIterationInterface *iterationIF,
ProblemTimeInterface *timeIF,
std::vector<DOFVector<double>*> vectors,
Mesh *mesh,
Mesh *mesh_,
RefinementManager *rm,
CoarseningManager *cm)
: ParallelProblemBase(name, iterationIF, timeIF),
name_(name),
mesh(mesh),
mesh(mesh_),
refinementManager(rm),
coarseningManager(cm),
repartitionSteps_(1),
......@@ -1053,8 +1053,7 @@ namespace AMDiS {
elemWeights.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh,
-1,
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
Element *element = elInfo->getElement();
......@@ -1359,29 +1358,6 @@ namespace AMDiS {
// and now partition the mesh
partitionMesh(adaptInfo);
#if 0
// 2009-03-31 THOMAS, DO NOT REMOVE THIS BLOCK
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
int nLeaves = 0;
while (elInfo) {
Element *element = elInfo->getElement();
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
(element->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) {
element->setHidden(true);
} else {
nLeaves++;
}
elInfo = stack.traverseNext(elInfo);
}
mesh->setNumberOfLeaves(nLeaves);
#endif
#if 1
globalRefineOutOfPartition(adaptInfo);
refineOverlap(adaptInfo);
......@@ -1429,8 +1405,6 @@ namespace AMDiS {
std::string(number) + "_");
(*fwIt)->setTraverseProperties(-1, 0, elementInPartition);
}
#endif
}
}
......
......@@ -22,23 +22,18 @@
#ifndef AMDIS_PARALLELPROBLEM_H
#define AMDIS_PARALLELPROBLEM_H
#include <map>
#include <vector>
#include <set>
#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
#include "AdaptInfo.h"
#include "mpi.h"
#include <map>
#include <vector>
#include <set>
namespace AMDiS {
// =========================================================================
// ===== class ParallelProblemInterface ====================================
// =========================================================================
/** \brief
* Interface for parallel problems
*/
/// Interface for parallel problems
class ParallelProblemInterface
{
public:
......@@ -47,13 +42,7 @@ namespace AMDiS {
virtual void exitParallelization(AdaptInfo *adaptInfo) = 0;
};
// =========================================================================
// ===== class ParallelProblemBase =========================================
// =========================================================================
/** \brief
* Schablonen Klasse
*/
/// Schablonen Klasse
class ParallelProblemBase : public ParallelProblemInterface,
public ProblemIterationInterface,
public ProblemTimeInterface
......@@ -88,19 +77,10 @@ namespace AMDiS {
virtual void coarsenOutOfPartition(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void synchronizeMeshes(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void exchangeRankSolutions(AdaptInfo *adaptInfo) = 0;
/** \brief
*
*/
virtual void buildGlobalSolution(AdaptInfo *adaptInfo) = 0;
virtual void exitParallelization(AdaptInfo *adaptInfo);
......@@ -160,14 +140,10 @@ namespace AMDiS {
double partitioningTime;
/** \brief
* The rank of the current process.
*/
/// The rank of the current process.
int mpiRank;
/** \brief
* Overall number of processes.
*/
/// Overall number of processes.
int mpiSize;
/** \brief
......@@ -177,16 +153,10 @@ namespace AMDiS {
*/
MPI::Intracomm mpiComm;
/** \brief
* The MPI group, which is used for the communicator
* \ref mpiComm.
*/
/// The MPI group, which is used for the communicator \ref mpiComm.
MPI::Group amdisGroup;
/** \brief
* Defines the debug mode. If it is 1, a debug server will be started
* on rank 0.
*/
/// Defines the debug mode. If it is 1, a debug server will be started on rank 0.
int debugMode;
/** \brief
......@@ -328,74 +298,32 @@ namespace AMDiS {
///
int repartitionSteps_;
/** \brief
*
*/
bool puEveryTimestep_;
/** \brief
*
*/
std::vector<DOFVector<double>*> dofVectors_;
/** \brief
*
*/
double upperPartThreshold_;
/** \brief
*
*/
double lowerPartThreshold_;
/** \brief
*
*/
int globalCoarseGridLevel_;
/** \brief
*
*/
int localCoarseGridLevel_;
/** \brief
*
*/
int globalRefinements_;
/** \brief
*
*/
std::map<Element*, int> overlapDistance_;
/** \brief
*
*/
int adaptiveThresholds_;
/** \brief
*
*/
double thresholdIncFactor_;
/** \brief
*
*/
double thresholdDecFactor_;
/** \brief
*
*/
double repartTimeFactor_;
/** \brief
*
*/
double minUpperTH_;
/** \brief
*
*/
double maxLowerTH_;
};
......@@ -458,19 +386,13 @@ namespace AMDiS {
Marker *oldMarker;
/** \brief
* Vector of all process' solution DOFVectors.
*/
/// Vector of all process' solution DOFVectors.
std::vector<DOFVector<double>*> rankSolution;
/** \brief
*
*/
///
Estimator *usersEstimator;
/** \brief
*
*/
///
Marker *usersMarker;
};
......@@ -527,41 +449,28 @@ namespace AMDiS {
virtual void debugFunction(AdaptInfo *adaptInfo) {}
protected:
///
ProblemVec *problem;
/** \brief
*
*/
///
ProblemInstatVec *problemInstat_;
/** \brief
*
*/
///
std::vector<Estimator*> oldEstimator;
/** \brief
*
*/
///
std::vector<Marker*> oldMarker;
/** \brief
* Vector of all process' solution SystemVectors.
*/
/// Vector of all process' solution SystemVectors.
std::vector<SystemVector*> rankSolution;
/** \brief
*
*/
///
std::vector<Estimator*> usersEstimator;