Commit 77275071 authored by Thomas Witkowski's avatar Thomas Witkowski

Code refactoring of parallelization code.

parent 23eb48b5
...@@ -29,7 +29,8 @@ endif ...@@ -29,7 +29,8 @@ endif
if USE_PARALLEL_DOMAIN_AMDIS if USE_PARALLEL_DOMAIN_AMDIS
PARALLEL_AMDIS_SOURCES += \ PARALLEL_AMDIS_SOURCES += \
$(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc $(PARALLEL_DIR)/ParallelDomainBase.h $(PARALLEL_DIR)/ParallelDomainBase.cc \
$(PARALLEL_DIR)/ParallelDomainScal.h $(PARALLEL_DIR)/ParallelDomainScal.cc
libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1 libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1
AMDIS_INCLUDES += -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include AMDIS_INCLUDES += -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include
endif endif
......
...@@ -38,7 +38,8 @@ build_triplet = @build@ ...@@ -38,7 +38,8 @@ build_triplet = @build@
host_triplet = @host@ host_triplet = @host@
@USE_PARALLEL_AMDIS_TRUE@am__append_1 = -DHAVE_PARALLEL_AMDIS=1 @USE_PARALLEL_AMDIS_TRUE@am__append_1 = -DHAVE_PARALLEL_AMDIS=1
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = \ @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc @USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainBase.h $(PARALLEL_DIR)/ParallelDomainBase.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainScal.h $(PARALLEL_DIR)/ParallelDomainScal.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1 @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_4 = -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_4 = -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include
...@@ -71,9 +72,10 @@ am__installdirs = "$(DESTDIR)$(libdir)" ...@@ -71,9 +72,10 @@ am__installdirs = "$(DESTDIR)$(libdir)"
libLTLIBRARIES_INSTALL = $(INSTALL) libLTLIBRARIES_INSTALL = $(INSTALL)
LTLIBRARIES = $(lib_LTLIBRARIES) LTLIBRARIES = $(lib_LTLIBRARIES)
libamdis_la_LIBADD = libamdis_la_LIBADD =
am__libamdis_la_SOURCES_DIST = \ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ParallelDomainBase.h \
$(PARALLEL_DIR)/ParallelDomainProblem.h \ $(PARALLEL_DIR)/ParallelDomainBase.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.cc \ $(PARALLEL_DIR)/ParallelDomainScal.h \
$(PARALLEL_DIR)/ParallelDomainScal.cc \
$(PARALLEL_DIR)/ConditionalEstimator.h \ $(PARALLEL_DIR)/ConditionalEstimator.h \
$(PARALLEL_DIR)/ConditionalEstimator.cc \ $(PARALLEL_DIR)/ConditionalEstimator.cc \
$(PARALLEL_DIR)/ConditionalMarker.h \ $(PARALLEL_DIR)/ConditionalMarker.h \
...@@ -224,7 +226,8 @@ am__libamdis_la_SOURCES_DIST = \ ...@@ -224,7 +226,8 @@ am__libamdis_la_SOURCES_DIST = \
$(SOURCE_DIR)/parareal/ProblemBase.h \ $(SOURCE_DIR)/parareal/ProblemBase.h \
$(SOURCE_DIR)/parareal/AdaptParaReal.h \ $(SOURCE_DIR)/parareal/AdaptParaReal.h \
$(SOURCE_DIR)/parareal/AdaptParaReal.cc $(SOURCE_DIR)/parareal/AdaptParaReal.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-ParallelDomainProblem.lo @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-ParallelDomainBase.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParallelDomainScal.lo
@USE_PARALLEL_AMDIS_FALSE@am__objects_2 = $(am__objects_1) @USE_PARALLEL_AMDIS_FALSE@am__objects_2 = $(am__objects_1)
@USE_PARALLEL_AMDIS_TRUE@am__objects_2 = \ @USE_PARALLEL_AMDIS_TRUE@am__objects_2 = \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \ @USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \
...@@ -754,7 +757,8 @@ distclean-compile: ...@@ -754,7 +757,8 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainBase.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainScal.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@
...@@ -825,12 +829,19 @@ distclean-compile: ...@@ -825,12 +829,19 @@ distclean-compile:
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< @am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
libamdis_la-ParallelDomainProblem.lo: $(PARALLEL_DIR)/ParallelDomainProblem.cc libamdis_la-ParallelDomainBase.lo: $(PARALLEL_DIR)/ParallelDomainBase.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainProblem.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc; \ @am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainBase.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo" -c -o libamdis_la-ParallelDomainBase.lo `test -f '$(PARALLEL_DIR)/ParallelDomainBase.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainBase.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo"; exit 1; fi @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainBase.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainProblem.cc' object='libamdis_la-ParallelDomainProblem.lo' libtool=yes @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainBase.cc' object='libamdis_la-ParallelDomainBase.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc @am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainBase.lo `test -f '$(PARALLEL_DIR)/ParallelDomainBase.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainBase.cc
libamdis_la-ParallelDomainScal.lo: $(PARALLEL_DIR)/ParallelDomainScal.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainScal.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo" -c -o libamdis_la-ParallelDomainScal.lo `test -f '$(PARALLEL_DIR)/ParallelDomainScal.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainScal.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainScal.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainScal.cc' object='libamdis_la-ParallelDomainScal.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainScal.lo `test -f '$(PARALLEL_DIR)/ParallelDomainScal.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainScal.cc
libamdis_la-ConditionalEstimator.lo: $(PARALLEL_DIR)/ConditionalEstimator.cc libamdis_la-ConditionalEstimator.lo: $(PARALLEL_DIR)/ConditionalEstimator.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ConditionalEstimator.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ConditionalEstimator.Tpo" -c -o libamdis_la-ConditionalEstimator.lo `test -f '$(PARALLEL_DIR)/ConditionalEstimator.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ConditionalEstimator.cc; \ @am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ConditionalEstimator.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ConditionalEstimator.Tpo" -c -o libamdis_la-ConditionalEstimator.lo `test -f '$(PARALLEL_DIR)/ConditionalEstimator.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ConditionalEstimator.cc; \
......
#include <boost/lambda/lambda.hpp>
#include <algorithm> #include <algorithm>
#include "ParallelDomainProblem.h" #include "ParallelDomainBase.h"
#include "ProblemScal.h"
#include "ProblemInstat.h"
#include "ParMetisPartitioner.h" #include "ParMetisPartitioner.h"
#include "Mesh.h" #include "Mesh.h"
#include "Traverse.h" #include "Traverse.h"
...@@ -19,8 +16,6 @@ ...@@ -19,8 +16,6 @@
#include "petscksp.h" #include "petscksp.h"
namespace AMDiS { namespace AMDiS {
using namespace boost::lambda;
PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *) PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
{ {
...@@ -30,6 +25,11 @@ namespace AMDiS { ...@@ -30,6 +25,11 @@ namespace AMDiS {
return 0; return 0;
} }
inline bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2)
{
return (*dof1 < *dof2);
}
ParallelDomainBase::ParallelDomainBase(const std::string& name, ParallelDomainBase::ParallelDomainBase(const std::string& name,
ProblemIterationInterface *iIF, ProblemIterationInterface *iIF,
ProblemTimeInterface *tIF, ProblemTimeInterface *tIF,
...@@ -88,11 +88,11 @@ namespace AMDiS { ...@@ -88,11 +88,11 @@ namespace AMDiS {
// Number of all DOFs in the macro mesh. // Number of all DOFs in the macro mesh.
int nOverallDOFs = 0; int nOverallDOFs = 0;
createLocalGlobalNumbering(rankDOFs, boundaryDOFs, nRankDOFs, nOverallDOFs); createLocalGlobalNumbering(rankDOFs, nRankDOFs, nOverallDOFs);
// === Create interior boundary information === // === Create interior boundary information ===
createInteriorBoundaryInfo(rankDOFs, boundaryDOFs); createInteriorBoundaryInfo(rankDOFs);
// === Remove all macro elements that are not part of the rank partition. === // === Remove all macro elements that are not part of the rank partition. ===
...@@ -369,8 +369,7 @@ namespace AMDiS { ...@@ -369,8 +369,7 @@ namespace AMDiS {
} }
void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs, void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs)
DofToRank& boundaryDOFs)
{ {
FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()"); FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()");
...@@ -532,7 +531,6 @@ namespace AMDiS { ...@@ -532,7 +531,6 @@ namespace AMDiS {
void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs, void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs,
DofToRank& boundaryDOFs,
int& nRankDOFs, int& nRankDOFs,
int& nOverallDOFs) int& nOverallDOFs)
{ {
...@@ -543,8 +541,9 @@ namespace AMDiS { ...@@ -543,8 +541,9 @@ namespace AMDiS {
// Stores to each DOF pointer the set of ranks the DOF is part of. // Stores to each DOF pointer the set of ranks the DOF is part of.
std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs; std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs;
DofContainer rankAllDofs; DofContainer rankAllDofs;
DofToRank boundaryDofs;
createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDOFs); createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDofs);
nRankDOFs = rankDOFs.size(); nRankDOFs = rankDOFs.size();
nOverallDOFs = partitionDOFs.size(); nOverallDOFs = partitionDOFs.size();
...@@ -604,7 +603,7 @@ namespace AMDiS { ...@@ -604,7 +603,7 @@ namespace AMDiS {
// another rank. // another rank.
std::map<int, int> recvNewDofs; std::map<int, int> recvNewDofs;
for (DofToRank::iterator it = boundaryDOFs.begin(); it != boundaryDOFs.end(); ++it) { for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
if (it->second == mpiRank) { if (it->second == mpiRank) {
// If the boundary dof is a rank dof, it must be send to other ranks. // If the boundary dof is a rank dof, it must be send to other ranks.
...@@ -685,21 +684,17 @@ namespace AMDiS { ...@@ -685,21 +684,17 @@ namespace AMDiS {
delete [] sendBuffers[j]; delete [] sendBuffers[j];
// === Change dof indices for rank partition. ===
mapLocalGlobalDOFs.clear();
// === Change dof indices at boundary from other ranks. === // === Change dof indices at boundary from other ranks. ===
// Within this small data structure we track which dof index was already changed. // Within this small data structure we track which dof index was already changed.
// This is used to avoid the following situation: Assume, there are two dof indices // This is used to avoid the following situation: Assume, there are two dof indices
// a and b in boundaryDOFs. Then we have to change index a to b and b to c. When // a and b in boundaryDofs. Then we have to change index a to b and b to c. When
// the second rule applies, we have to avoid that not the first b, resulted from // the second rule applies, we have to avoid that not the first b, resulted from
// changing a to b, is set to c, but the second one. Therefore, after the first // changing a to b, is set to c, but the second one. Therefore, after the first
// rule was applied, the dof pointer is set to false in this data structure and // rule was applied, the dof pointer is set to false in this data structure and
// is not allowed to be changed anymore. // is not allowed to be changed anymore.
std::map<const DegreeOfFreedom*, bool> dofChanged; std::map<const DegreeOfFreedom*, bool> dofChanged;
for (DofToRank::iterator dofIt = boundaryDOFs.begin(); dofIt != boundaryDOFs.end(); for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
++dofIt) ++dofIt)
dofChanged[dofIt->first] = false; dofChanged[dofIt->first] = false;
...@@ -717,8 +712,8 @@ namespace AMDiS { ...@@ -717,8 +712,8 @@ namespace AMDiS {
// Iterate over all boundary dofs to find the dof, which index we have to change. // Iterate over all boundary dofs to find the dof, which index we have to change.
for (DofToRank::iterator dofIt = boundaryDOFs.begin(); for (DofToRank::iterator dofIt = boundaryDofs.begin();
dofIt != boundaryDOFs.end(); ++dofIt) { dofIt != boundaryDofs.end(); ++dofIt) {
if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) { if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true; dofChanged[dofIt->first] = true;
...@@ -738,22 +733,10 @@ namespace AMDiS { ...@@ -738,22 +733,10 @@ namespace AMDiS {
delete [] recvBuffers[i]; delete [] recvBuffers[i];
} }
// === Create now the local to global index and local to dof index mappings. ===
// === Create now the local to global index, and vice verse, mappings. ===
for (DofIndexMap::iterator dofIt = rankDofsNewLocalIndex.begin(); createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
dofIt != rankDofsNewLocalIndex.end(); ++dofIt) { rankDofsNewGlobalIndex);
DegreeOfFreedom localDof = dofIt->second;
DegreeOfFreedom globalDof = rankDofsNewGlobalIndex[dofIt->first];
*const_cast<DegreeOfFreedom*>(dofIt->first) = localDof;
mapLocalGlobalDOFs[localDof] = globalDof;
}
mapLocalToDofIndex.clear();
for (DofIndexMap::iterator dofIt = rankOwnedDofsNewLocalIndex.begin();
dofIt != rankOwnedDofsNewLocalIndex.end(); ++dofIt)
mapLocalToDofIndex[dofIt->second] = *(dofIt->first);
} }
...@@ -789,9 +772,8 @@ namespace AMDiS { ...@@ -789,9 +772,8 @@ namespace AMDiS {
// === Traverse on interior boundaries and move all not ranked owned DOFs from === // === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDOFs to boundaryDOFs. === // === rankDOFs to boundaryDOFs. ===
DofToRank newBoundaryDOFs; sendDofs.clear();
RankToDofContainer sendNewDofs; recvDofs.clear();
RankToDofContainer recvNewDofs;
for (RankToBoundMap::iterator it = myIntBoundary.boundary.begin(); for (RankToBoundMap::iterator it = myIntBoundary.boundary.begin();
it != myIntBoundary.boundary.end(); ++it) { it != myIntBoundary.boundary.end(); ++it) {
...@@ -799,51 +781,38 @@ namespace AMDiS { ...@@ -799,51 +781,38 @@ namespace AMDiS {
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
const DegreeOfFreedom *dof1, *dof2; DofContainer dofs;
DofContainer &dofsToSend = sendDofs[it->first];
switch (boundIt->rankObject.ithObjAtBoundary) { switch (boundIt->rankObject.ithObjAtBoundary) {
case 0: case 0:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 1: case 1:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 2: case 2:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
break; break;
default: default:
ERROR_EXIT("Should never happen!\n"); ERROR_EXIT("Should never happen!\n");
} }
TEST_EXIT_DBG(boundaryDOFs.find(dof1) != boundaryDOFs.end()) for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt) {
("Should never happen!\n"); if (find(dofsToSend.begin(), dofsToSend.end(), *dofIt) == dofsToSend.end())
TEST_EXIT_DBG(boundaryDOFs.find(dof2) != boundaryDOFs.end()) dofsToSend.push_back(*dofIt);
("Should never happen!\n"); }
newBoundaryDOFs[dof1] = boundaryDOFs[dof1];
newBoundaryDOFs[dof2] = boundaryDOFs[dof2];
DofContainer &dofsToSend = sendNewDofs[it->first];
if (find(dofsToSend.begin(), dofsToSend.end(), dof1) == dofsToSend.end())
dofsToSend.push_back(dof1);
if (find(dofsToSend.begin(), dofsToSend.end(), dof2) == dofsToSend.end())
dofsToSend.push_back(dof2);
DofContainer boundDOFs;
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) { dofs.clear();
newBoundaryDOFs[boundDOFs[i]] = mpiRank; addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofsToSend.push_back(boundDOFs[i]); dofs);
addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofs);
for (int i = 0; i < static_cast<int>(dofs.size()); i++) {
dofsToSend.push_back(dofs[i]);
} }
} }
...@@ -855,64 +824,49 @@ namespace AMDiS { ...@@ -855,64 +824,49 @@ namespace AMDiS {
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
const DegreeOfFreedom *dof1, *dof2; DofContainer dofs;
DofContainer &dofsToRecv = recvDofs[it->first];
switch (boundIt->rankObject.ithObjAtBoundary) { switch (boundIt->rankObject.ithObjAtBoundary) {
case 0: case 0:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 1: case 1:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 2: case 2:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
break; break;
default: default:
ERROR_EXIT("Should never happen!\n"); ERROR_EXIT("Should never happen!\n");
} }
TEST_EXIT_DBG(boundaryDOFs.find(dof1) != boundaryDOFs.end()) for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt) {
("Should never happen!\n"); DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), *dofIt);
TEST_EXIT_DBG(boundaryDOFs.find(dof2) != boundaryDOFs.end()) if (eraseIt != rankDOFs.end())
("Should never happen!\n"); rankDOFs.erase(eraseIt);
if (find(dofsToRecv.begin(), dofsToRecv.end(), *dofIt) == dofsToRecv.end())
DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dof1); dofsToRecv.push_back(*dofIt);
if (eraseIt != rankDOFs.end()) }
rankDOFs.erase(eraseIt);
eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dof2); dofs.clear();
if (eraseIt != rankDOFs.end()) addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
rankDOFs.erase(eraseIt); dofs);
addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
newBoundaryDOFs[dof1] = boundaryDOFs[dof1]; dofs);
newBoundaryDOFs[dof2] = boundaryDOFs[dof2];
for (int i = static_cast<int>(dofs.size()) - 1; i >= 0; i--) {
DofContainer &dofsToRecv = recvNewDofs[it->first]; TEST_EXIT_DBG(find(rankDOFs.begin(), rankDOFs.end(), dofs[i]) != rankDOFs.end())
if (find(dofsToRecv.begin(), dofsToRecv.end(), dof1) == dofsToRecv.end())
dofsToRecv.push_back(dof1);
if (find(dofsToRecv.begin(), dofsToRecv.end(), dof2) == dofsToRecv.end())
dofsToRecv.push_back(dof2);
DofContainer boundDOFs;
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
for (int i = static_cast<int>(boundDOFs.size()) - 1; i >= 0; i--) {
TEST_EXIT_DBG(find(rankDOFs.begin(), rankDOFs.end(), boundDOFs[i]) != rankDOFs.end())
("Should never happen!\n"); ("Should never happen!\n");
eraseIt = find(rankDOFs.begin(), rankDOFs.end(), boundDOFs[i]); DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dofs[i]);
if (eraseIt != rankDOFs.end()) if (eraseIt != rankDOFs.end())
rankDOFs.erase(eraseIt); rankDOFs.erase(eraseIt);
newBoundaryDOFs[boundDOFs[i]] = it->first; dofsToRecv.push_back(dofs[i]);
dofsToRecv.push_back(boundDOFs[i]);
} }
} }
} }
...@@ -961,15 +915,15 @@ namespace AMDiS { ...@@ -961,15 +915,15 @@ namespace AMDiS {
// === Send new DOF indices. === // === Send new DOF indices. ===
std::vector<int*> sendBuffers(sendNewDofs.size()); std::vector<int*> sendBuffers(sendDofs.size());
std::vector<int*> recvBuffers(recvNewDofs.size()); std::vector<int*> recvBuffers(recvDofs.size());
MPI::Request request[sendNewDofs.size() + recvNewDofs.size()]; MPI::Request request[sendDofs.size() + recvDofs.size()];
int requestCounter = 0; int requestCounter = 0;
i = 0; i = 0;
for (RankToDofContainer::iterator sendIt = sendNewDofs.begin(); for (RankToDofContainer::iterator sendIt = sendDofs.begin();
sendIt != sendNewDofs.end(); ++sendIt, i++) { sendIt != sendDofs.end(); ++sendIt, i++) {
int nSendDofs = sendIt->second.size(); int nSendDofs = sendIt->second.size();
sendBuffers[i] = new int[nSendDofs]; sendBuffers[i] = new int[nSendDofs];
int c = 0; int c = 0;
...@@ -982,8 +936,8 @@ namespace AMDiS { ...@@ -982,8 +936,8 @@ namespace AMDiS {
} }
i = 0; i = 0;
for (RankToDofContainer::iterator recvIt = recvNewDofs.begin(); for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt, i++) { recvIt != recvDofs.end(); ++recvIt, i++) {
int nRecvDofs = recvIt->second.size(); int nRecvDofs = recvIt->second.size();
recvBuffers[i] = new int[nRecvDofs]; recvBuffers[i] = new int[nRecvDofs];
...@@ -997,8 +951,8 @@ namespace AMDiS { ...@@ -997,8 +951,8 @@ namespace AMDiS {
delete [] sendBuffers[j]; delete [] sendBuffers[j];
i = 0; i = 0;
for (RankToDofContainer::iterator recvIt = recvNewDofs.begin(); for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt) { recvIt != recvDofs.end(); ++recvIt) {
int j = 0; int j = 0;
for (DofContainer::iterator dofIt = recvIt->second.begin(); for (DofContainer::iterator dofIt = recvIt->second.begin();
dofIt != recvIt->second.end(); ++dofIt) { dofIt != recvIt->second.end(); ++dofIt) {
...@@ -1012,14 +966,16 @@ namespace AMDiS { ...@@ -1012,14 +966,16 @@ namespace AMDiS {
} }
// === Update list of dofs that must be communicated for solution exchange. === // === Create now the local to global index and local to dof index mappings. ===
sendDofs = sendNewDofs; createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
recvDofs = recvNewDofs; rankDofsNewGlobalIndex);
}
// === Create now the local to global index, and vice verse, mappings. ===
void ParallelDomainBase::createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
DofIndexMap &rankOwnedDofsNewLocalIndex,
DofIndexMap &rankDofsNewGlobalIndex)
{
mapLocalGlobalDOFs.clear(); mapLocalGlobalDOFs.clear();
mapLocalToDofIndex.clear(); mapLocalToDofIndex.clear();
...@@ -1032,13 +988,11 @@ namespace AMDiS { ...@@ -1032,13 +988,11 @@ namespace AMDiS {
mapLocalGlobalDOFs[localDof] = globalDof; mapLocalGlobalDOFs[localDof] = globalDof;