Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit 77275071 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Code refactoring of parallelization code.

parent 23eb48b5
...@@ -29,7 +29,8 @@ endif ...@@ -29,7 +29,8 @@ endif
if USE_PARALLEL_DOMAIN_AMDIS if USE_PARALLEL_DOMAIN_AMDIS
PARALLEL_AMDIS_SOURCES += \ PARALLEL_AMDIS_SOURCES += \
$(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc $(PARALLEL_DIR)/ParallelDomainBase.h $(PARALLEL_DIR)/ParallelDomainBase.cc \
$(PARALLEL_DIR)/ParallelDomainScal.h $(PARALLEL_DIR)/ParallelDomainScal.cc
libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1 libamdis_la_CXXFLAGS += -DHAVE_PARALLEL_DOMAIN_AMDIS=1
AMDIS_INCLUDES += -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include AMDIS_INCLUDES += -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include
endif endif
......
...@@ -38,7 +38,8 @@ build_triplet = @build@ ...@@ -38,7 +38,8 @@ build_triplet = @build@
host_triplet = @host@ host_triplet = @host@
@USE_PARALLEL_AMDIS_TRUE@am__append_1 = -DHAVE_PARALLEL_AMDIS=1 @USE_PARALLEL_AMDIS_TRUE@am__append_1 = -DHAVE_PARALLEL_AMDIS=1
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = \ @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_2 = \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainProblem.h $(PARALLEL_DIR)/ParallelDomainProblem.cc @USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainBase.h $(PARALLEL_DIR)/ParallelDomainBase.cc \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ $(PARALLEL_DIR)/ParallelDomainScal.h $(PARALLEL_DIR)/ParallelDomainScal.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1 @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_3 = -DHAVE_PARALLEL_DOMAIN_AMDIS=1
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_4 = -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__append_4 = -I/u/witkowski/local/petsc-3.0.0-p4/include -I/u/witkowski/local/petsc-3.0.0-p4/linux-gnu-c-debug/include
...@@ -71,9 +72,10 @@ am__installdirs = "$(DESTDIR)$(libdir)" ...@@ -71,9 +72,10 @@ am__installdirs = "$(DESTDIR)$(libdir)"
libLTLIBRARIES_INSTALL = $(INSTALL) libLTLIBRARIES_INSTALL = $(INSTALL)
LTLIBRARIES = $(lib_LTLIBRARIES) LTLIBRARIES = $(lib_LTLIBRARIES)
libamdis_la_LIBADD = libamdis_la_LIBADD =
am__libamdis_la_SOURCES_DIST = \ am__libamdis_la_SOURCES_DIST = $(PARALLEL_DIR)/ParallelDomainBase.h \
$(PARALLEL_DIR)/ParallelDomainProblem.h \ $(PARALLEL_DIR)/ParallelDomainBase.cc \
$(PARALLEL_DIR)/ParallelDomainProblem.cc \ $(PARALLEL_DIR)/ParallelDomainScal.h \
$(PARALLEL_DIR)/ParallelDomainScal.cc \
$(PARALLEL_DIR)/ConditionalEstimator.h \ $(PARALLEL_DIR)/ConditionalEstimator.h \
$(PARALLEL_DIR)/ConditionalEstimator.cc \ $(PARALLEL_DIR)/ConditionalEstimator.cc \
$(PARALLEL_DIR)/ConditionalMarker.h \ $(PARALLEL_DIR)/ConditionalMarker.h \
...@@ -224,7 +226,8 @@ am__libamdis_la_SOURCES_DIST = \ ...@@ -224,7 +226,8 @@ am__libamdis_la_SOURCES_DIST = \
$(SOURCE_DIR)/parareal/ProblemBase.h \ $(SOURCE_DIR)/parareal/ProblemBase.h \
$(SOURCE_DIR)/parareal/AdaptParaReal.h \ $(SOURCE_DIR)/parareal/AdaptParaReal.h \
$(SOURCE_DIR)/parareal/AdaptParaReal.cc $(SOURCE_DIR)/parareal/AdaptParaReal.cc
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-ParallelDomainProblem.lo @USE_PARALLEL_DOMAIN_AMDIS_TRUE@am__objects_1 = libamdis_la-ParallelDomainBase.lo \
@USE_PARALLEL_DOMAIN_AMDIS_TRUE@ libamdis_la-ParallelDomainScal.lo
@USE_PARALLEL_AMDIS_FALSE@am__objects_2 = $(am__objects_1) @USE_PARALLEL_AMDIS_FALSE@am__objects_2 = $(am__objects_1)
@USE_PARALLEL_AMDIS_TRUE@am__objects_2 = \ @USE_PARALLEL_AMDIS_TRUE@am__objects_2 = \
@USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \ @USE_PARALLEL_AMDIS_TRUE@ libamdis_la-ConditionalEstimator.lo \
...@@ -754,7 +757,8 @@ distclean-compile: ...@@ -754,7 +757,8 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-NonLinUpdater.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Operator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParMetisPartitioner.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainBase.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelDomainScal.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-ParallelProblem.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parameters.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libamdis_la-Parametric.Plo@am__quote@
...@@ -825,12 +829,19 @@ distclean-compile: ...@@ -825,12 +829,19 @@ distclean-compile:
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< @am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
libamdis_la-ParallelDomainProblem.lo: $(PARALLEL_DIR)/ParallelDomainProblem.cc libamdis_la-ParallelDomainBase.lo: $(PARALLEL_DIR)/ParallelDomainBase.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainProblem.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc; \ @am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainBase.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo" -c -o libamdis_la-ParallelDomainBase.lo `test -f '$(PARALLEL_DIR)/ParallelDomainBase.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainBase.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainProblem.Tpo"; exit 1; fi @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainBase.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainBase.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainProblem.cc' object='libamdis_la-ParallelDomainProblem.lo' libtool=yes @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainBase.cc' object='libamdis_la-ParallelDomainBase.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainProblem.lo `test -f '$(PARALLEL_DIR)/ParallelDomainProblem.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainProblem.cc @am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainBase.lo `test -f '$(PARALLEL_DIR)/ParallelDomainBase.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainBase.cc
libamdis_la-ParallelDomainScal.lo: $(PARALLEL_DIR)/ParallelDomainScal.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ParallelDomainScal.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo" -c -o libamdis_la-ParallelDomainScal.lo `test -f '$(PARALLEL_DIR)/ParallelDomainScal.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainScal.cc; \
@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo" "$(DEPDIR)/libamdis_la-ParallelDomainScal.Plo"; else rm -f "$(DEPDIR)/libamdis_la-ParallelDomainScal.Tpo"; exit 1; fi
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$(PARALLEL_DIR)/ParallelDomainScal.cc' object='libamdis_la-ParallelDomainScal.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -c -o libamdis_la-ParallelDomainScal.lo `test -f '$(PARALLEL_DIR)/ParallelDomainScal.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ParallelDomainScal.cc
libamdis_la-ConditionalEstimator.lo: $(PARALLEL_DIR)/ConditionalEstimator.cc libamdis_la-ConditionalEstimator.lo: $(PARALLEL_DIR)/ConditionalEstimator.cc
@am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ConditionalEstimator.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ConditionalEstimator.Tpo" -c -o libamdis_la-ConditionalEstimator.lo `test -f '$(PARALLEL_DIR)/ConditionalEstimator.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ConditionalEstimator.cc; \ @am__fastdepCXX_TRUE@ if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libamdis_la_CXXFLAGS) $(CXXFLAGS) -MT libamdis_la-ConditionalEstimator.lo -MD -MP -MF "$(DEPDIR)/libamdis_la-ConditionalEstimator.Tpo" -c -o libamdis_la-ConditionalEstimator.lo `test -f '$(PARALLEL_DIR)/ConditionalEstimator.cc' || echo '$(srcdir)/'`$(PARALLEL_DIR)/ConditionalEstimator.cc; \
......
#include <boost/lambda/lambda.hpp>
#include <algorithm> #include <algorithm>
#include "ParallelDomainProblem.h" #include "ParallelDomainBase.h"
#include "ProblemScal.h"
#include "ProblemInstat.h"
#include "ParMetisPartitioner.h" #include "ParMetisPartitioner.h"
#include "Mesh.h" #include "Mesh.h"
#include "Traverse.h" #include "Traverse.h"
...@@ -20,8 +17,6 @@ ...@@ -20,8 +17,6 @@
namespace AMDiS { namespace AMDiS {
using namespace boost::lambda;
PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *) PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
{ {
if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0) if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0)
...@@ -30,6 +25,11 @@ namespace AMDiS { ...@@ -30,6 +25,11 @@ namespace AMDiS {
return 0; return 0;
} }
inline bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2)
{
return (*dof1 < *dof2);
}
ParallelDomainBase::ParallelDomainBase(const std::string& name, ParallelDomainBase::ParallelDomainBase(const std::string& name,
ProblemIterationInterface *iIF, ProblemIterationInterface *iIF,
ProblemTimeInterface *tIF, ProblemTimeInterface *tIF,
...@@ -88,11 +88,11 @@ namespace AMDiS { ...@@ -88,11 +88,11 @@ namespace AMDiS {
// Number of all DOFs in the macro mesh. // Number of all DOFs in the macro mesh.
int nOverallDOFs = 0; int nOverallDOFs = 0;
createLocalGlobalNumbering(rankDOFs, boundaryDOFs, nRankDOFs, nOverallDOFs); createLocalGlobalNumbering(rankDOFs, nRankDOFs, nOverallDOFs);
// === Create interior boundary information === // === Create interior boundary information ===
createInteriorBoundaryInfo(rankDOFs, boundaryDOFs); createInteriorBoundaryInfo(rankDOFs);
// === Remove all macro elements that are not part of the rank partition. === // === Remove all macro elements that are not part of the rank partition. ===
...@@ -369,8 +369,7 @@ namespace AMDiS { ...@@ -369,8 +369,7 @@ namespace AMDiS {
} }
void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs, void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs)
DofToRank& boundaryDOFs)
{ {
FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()"); FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()");
...@@ -532,7 +531,6 @@ namespace AMDiS { ...@@ -532,7 +531,6 @@ namespace AMDiS {
void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs, void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs,
DofToRank& boundaryDOFs,
int& nRankDOFs, int& nRankDOFs,
int& nOverallDOFs) int& nOverallDOFs)
{ {
...@@ -543,8 +541,9 @@ namespace AMDiS { ...@@ -543,8 +541,9 @@ namespace AMDiS {
// Stores to each DOF pointer the set of ranks the DOF is part of. // Stores to each DOF pointer the set of ranks the DOF is part of.
std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs; std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs;
DofContainer rankAllDofs; DofContainer rankAllDofs;
DofToRank boundaryDofs;
createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDOFs); createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDofs);
nRankDOFs = rankDOFs.size(); nRankDOFs = rankDOFs.size();
nOverallDOFs = partitionDOFs.size(); nOverallDOFs = partitionDOFs.size();
...@@ -604,7 +603,7 @@ namespace AMDiS { ...@@ -604,7 +603,7 @@ namespace AMDiS {
// another rank. // another rank.
std::map<int, int> recvNewDofs; std::map<int, int> recvNewDofs;
for (DofToRank::iterator it = boundaryDOFs.begin(); it != boundaryDOFs.end(); ++it) { for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
if (it->second == mpiRank) { if (it->second == mpiRank) {
// If the boundary dof is a rank dof, it must be send to other ranks. // If the boundary dof is a rank dof, it must be send to other ranks.
...@@ -685,21 +684,17 @@ namespace AMDiS { ...@@ -685,21 +684,17 @@ namespace AMDiS {
delete [] sendBuffers[j]; delete [] sendBuffers[j];
// === Change dof indices for rank partition. ===
mapLocalGlobalDOFs.clear();
// === Change dof indices at boundary from other ranks. === // === Change dof indices at boundary from other ranks. ===
// Within this small data structure we track which dof index was already changed. // Within this small data structure we track which dof index was already changed.
// This is used to avoid the following situation: Assume, there are two dof indices // This is used to avoid the following situation: Assume, there are two dof indices
// a and b in boundaryDOFs. Then we have to change index a to b and b to c. When // a and b in boundaryDofs. Then we have to change index a to b and b to c. When
// the second rule applies, we have to avoid that not the first b, resulted from // the second rule applies, we have to avoid that not the first b, resulted from
// changing a to b, is set to c, but the second one. Therefore, after the first // changing a to b, is set to c, but the second one. Therefore, after the first
// rule was applied, the dof pointer is set to false in this data structure and // rule was applied, the dof pointer is set to false in this data structure and
// is not allowed to be changed anymore. // is not allowed to be changed anymore.
std::map<const DegreeOfFreedom*, bool> dofChanged; std::map<const DegreeOfFreedom*, bool> dofChanged;
for (DofToRank::iterator dofIt = boundaryDOFs.begin(); dofIt != boundaryDOFs.end(); for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
++dofIt) ++dofIt)
dofChanged[dofIt->first] = false; dofChanged[dofIt->first] = false;
...@@ -717,8 +712,8 @@ namespace AMDiS { ...@@ -717,8 +712,8 @@ namespace AMDiS {
// Iterate over all boundary dofs to find the dof, which index we have to change. // Iterate over all boundary dofs to find the dof, which index we have to change.
for (DofToRank::iterator dofIt = boundaryDOFs.begin(); for (DofToRank::iterator dofIt = boundaryDofs.begin();
dofIt != boundaryDOFs.end(); ++dofIt) { dofIt != boundaryDofs.end(); ++dofIt) {
if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) { if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true; dofChanged[dofIt->first] = true;
...@@ -738,22 +733,10 @@ namespace AMDiS { ...@@ -738,22 +733,10 @@ namespace AMDiS {
delete [] recvBuffers[i]; delete [] recvBuffers[i];
} }
// === Create now the local to global index and local to dof index mappings. ===
// === Create now the local to global index, and vice verse, mappings. === createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
rankDofsNewGlobalIndex);
for (DofIndexMap::iterator dofIt = rankDofsNewLocalIndex.begin();
dofIt != rankDofsNewLocalIndex.end(); ++dofIt) {
DegreeOfFreedom localDof = dofIt->second;
DegreeOfFreedom globalDof = rankDofsNewGlobalIndex[dofIt->first];
*const_cast<DegreeOfFreedom*>(dofIt->first) = localDof;
mapLocalGlobalDOFs[localDof] = globalDof;
}
mapLocalToDofIndex.clear();
for (DofIndexMap::iterator dofIt = rankOwnedDofsNewLocalIndex.begin();
dofIt != rankOwnedDofsNewLocalIndex.end(); ++dofIt)
mapLocalToDofIndex[dofIt->second] = *(dofIt->first);
} }
...@@ -789,9 +772,8 @@ namespace AMDiS { ...@@ -789,9 +772,8 @@ namespace AMDiS {
// === Traverse on interior boundaries and move all not ranked owned DOFs from === // === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDOFs to boundaryDOFs. === // === rankDOFs to boundaryDOFs. ===
DofToRank newBoundaryDOFs; sendDofs.clear();
RankToDofContainer sendNewDofs; recvDofs.clear();
RankToDofContainer recvNewDofs;
for (RankToBoundMap::iterator it = myIntBoundary.boundary.begin(); for (RankToBoundMap::iterator it = myIntBoundary.boundary.begin();
it != myIntBoundary.boundary.end(); ++it) { it != myIntBoundary.boundary.end(); ++it) {
...@@ -799,51 +781,38 @@ namespace AMDiS { ...@@ -799,51 +781,38 @@ namespace AMDiS {
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
const DegreeOfFreedom *dof1, *dof2; DofContainer dofs;
DofContainer &dofsToSend = sendDofs[it->first];
switch (boundIt->rankObject.ithObjAtBoundary) { switch (boundIt->rankObject.ithObjAtBoundary) {
case 0: case 0:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 1: case 1:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 2: case 2:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
break; break;
default: default:
ERROR_EXIT("Should never happen!\n"); ERROR_EXIT("Should never happen!\n");
} }
TEST_EXIT_DBG(boundaryDOFs.find(dof1) != boundaryDOFs.end()) for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt) {
("Should never happen!\n"); if (find(dofsToSend.begin(), dofsToSend.end(), *dofIt) == dofsToSend.end())
TEST_EXIT_DBG(boundaryDOFs.find(dof2) != boundaryDOFs.end()) dofsToSend.push_back(*dofIt);
("Should never happen!\n"); }
newBoundaryDOFs[dof1] = boundaryDOFs[dof1];
newBoundaryDOFs[dof2] = boundaryDOFs[dof2];
DofContainer &dofsToSend = sendNewDofs[it->first];
if (find(dofsToSend.begin(), dofsToSend.end(), dof1) == dofsToSend.end())
dofsToSend.push_back(dof1);
if (find(dofsToSend.begin(), dofsToSend.end(), dof2) == dofsToSend.end())
dofsToSend.push_back(dof2);
DofContainer boundDOFs;
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) { dofs.clear();
newBoundaryDOFs[boundDOFs[i]] = mpiRank; addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofsToSend.push_back(boundDOFs[i]); dofs);
addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofs);
for (int i = 0; i < static_cast<int>(dofs.size()); i++) {
dofsToSend.push_back(dofs[i]);
} }
} }
...@@ -855,64 +824,49 @@ namespace AMDiS { ...@@ -855,64 +824,49 @@ namespace AMDiS {
for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin(); for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
boundIt != it->second.end(); ++boundIt) { boundIt != it->second.end(); ++boundIt) {
const DegreeOfFreedom *dof1, *dof2; DofContainer dofs;
DofContainer &dofsToRecv = recvDofs[it->first];
switch (boundIt->rankObject.ithObjAtBoundary) { switch (boundIt->rankObject.ithObjAtBoundary) {
case 0: case 0:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 1: case 1:
dof1 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
dof2 = boundIt->rankObject.el->getDOF(2); dofs.push_back(boundIt->rankObject.el->getDOF(2));
break; break;
case 2: case 2:
dof1 = boundIt->rankObject.el->getDOF(1); dofs.push_back(boundIt->rankObject.el->getDOF(1));
dof2 = boundIt->rankObject.el->getDOF(0); dofs.push_back(boundIt->rankObject.el->getDOF(0));
break; break;
default: default:
ERROR_EXIT("Should never happen!\n"); ERROR_EXIT("Should never happen!\n");
} }
TEST_EXIT_DBG(boundaryDOFs.find(dof1) != boundaryDOFs.end()) for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt) {
("Should never happen!\n"); DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), *dofIt);
TEST_EXIT_DBG(boundaryDOFs.find(dof2) != boundaryDOFs.end())
("Should never happen!\n");
DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dof1);
if (eraseIt != rankDOFs.end())
rankDOFs.erase(eraseIt);
eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dof2);
if (eraseIt != rankDOFs.end()) if (eraseIt != rankDOFs.end())
rankDOFs.erase(eraseIt); rankDOFs.erase(eraseIt);
if (find(dofsToRecv.begin(), dofsToRecv.end(), *dofIt) == dofsToRecv.end())
dofsToRecv.push_back(*dofIt);
}
newBoundaryDOFs[dof1] = boundaryDOFs[dof1]; dofs.clear();
newBoundaryDOFs[dof2] = boundaryDOFs[dof2]; addAllEdgeDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
dofs);
DofContainer &dofsToRecv = recvNewDofs[it->first]; addAllVertexDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary,
if (find(dofsToRecv.begin(), dofsToRecv.end(), dof1) == dofsToRecv.end()) dofs);
dofsToRecv.push_back(dof1);
if (find(dofsToRecv.begin(), dofsToRecv.end(), dof2) == dofsToRecv.end()) for (int i = static_cast<int>(dofs.size()) - 1; i >= 0; i--) {
dofsToRecv.push_back(dof2); TEST_EXIT_DBG(find(rankDOFs.begin(), rankDOFs.end(), dofs[i]) != rankDOFs.end())
DofContainer boundDOFs;
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
for (int i = static_cast<int>(boundDOFs.size()) - 1; i >= 0; i--) {
TEST_EXIT_DBG(find(rankDOFs.begin(), rankDOFs.end(), boundDOFs[i]) != rankDOFs.end())
("Should never happen!\n"); ("Should never happen!\n");
eraseIt = find(rankDOFs.begin(), rankDOFs.end(), boundDOFs[i]); DofContainer::iterator eraseIt = find(rankDOFs.begin(), rankDOFs.end(), dofs[i]);
if (eraseIt != rankDOFs.end()) if (eraseIt != rankDOFs.end())
rankDOFs.erase(eraseIt); rankDOFs.erase(eraseIt);
newBoundaryDOFs[boundDOFs[i]] = it->first; dofsToRecv.push_back(dofs[i]);
dofsToRecv.push_back(boundDOFs[i]);
} }
} }
} }
...@@ -961,15 +915,15 @@ namespace AMDiS { ...@@ -961,15 +915,15 @@ namespace AMDiS {
// === Send new DOF indices. === // === Send new DOF indices. ===
std::vector<int*> sendBuffers(sendNewDofs.size()); std::vector<int*> sendBuffers(sendDofs.size());
std::vector<int*> recvBuffers(recvNewDofs.size()); std::vector<int*> recvBuffers(recvDofs.size());
MPI::Request request[sendNewDofs.size() + recvNewDofs.size()]; MPI::Request request[sendDofs.size() + recvDofs.size()];
int requestCounter = 0; int requestCounter = 0;
i = 0; i = 0;
for (RankToDofContainer::iterator sendIt = sendNewDofs.begin(); for (RankToDofContainer::iterator sendIt = sendDofs.begin();
sendIt != sendNewDofs.end(); ++sendIt, i++) { sendIt != sendDofs.end(); ++sendIt, i++) {
int nSendDofs = sendIt->second.size(); int nSendDofs = sendIt->second.size();
sendBuffers[i] = new int[nSendDofs]; sendBuffers[i] = new int[nSendDofs];
int c = 0; int c = 0;
...@@ -982,8 +936,8 @@ namespace AMDiS { ...@@ -982,8 +936,8 @@ namespace AMDiS {
} }
i = 0; i = 0;
for (RankToDofContainer::iterator recvIt = recvNewDofs.begin(); for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt, i++) { recvIt != recvDofs.end(); ++recvIt, i++) {
int nRecvDofs = recvIt->second.size(); int nRecvDofs = recvIt->second.size();
recvBuffers[i] = new int[nRecvDofs]; recvBuffers[i] = new int[nRecvDofs];
...@@ -997,8 +951,8 @@ namespace AMDiS { ...@@ -997,8 +951,8 @@ namespace AMDiS {
delete [] sendBuffers[j]; delete [] sendBuffers[j];
i = 0; i = 0;
for (RankToDofContainer::iterator recvIt = recvNewDofs.begin(); for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt != recvNewDofs.end(); ++recvIt) { recvIt != recvDofs.end(); ++recvIt) {
int j = 0; int j = 0;
for (DofContainer::iterator dofIt = recvIt->second.begin(); for (DofContainer::iterator dofIt = recvIt->second.begin();
dofIt != recvIt->second.end(); ++dofIt) { dofIt != recvIt->second.end(); ++dofIt) {
...@@ -1012,14 +966,16 @@ namespace AMDiS { ...@@ -1012,14 +966,16 @@ namespace AMDiS {
} }
// === Update list of dofs that must be communicated for solution exchange. === // === Create now the local to global index and local to dof index mappings. ===
sendDofs = sendNewDofs;
recvDofs = recvNewDofs;
// === Create now the local to global index, and vice verse, mappings. === createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
rankDofsNewGlobalIndex);
}
void ParallelDomainBase::createLocalMappings(DofIndexMap &rankDofsNewLocalIndex,
DofIndexMap &rankOwnedDofsNewLocalIndex,
DofIndexMap &rankDofsNewGlobalIndex)
{
mapLocalGlobalDOFs.clear(); mapLocalGlobalDOFs.clear();
mapLocalToDofIndex.clear(); mapLocalToDofIndex.clear();
...@@ -1032,13 +988,11 @@ namespace AMDiS { ...@@ -1032