Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit ab09fcda authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed serialization problem for parallel computations, work on periodic bc in parallel code.

parent 9355f49c
...@@ -2074,7 +2074,7 @@ fi; ...@@ -2074,7 +2074,7 @@ fi;
MPI_DIR=$mpidir MPI_DIR=$mpidir
if test $mpidir != no ; then if test $mpidir != no ; then
CXX=$mpidir/bin/mpiCC CXX=$mpidir/bin/mpicxx
CC=$mpidir/bin/mpicc CC=$mpidir/bin/mpicc
......
...@@ -50,7 +50,7 @@ fi ...@@ -50,7 +50,7 @@ fi
AC_ARG_WITH(mpi, [ --with-mpi=MPI_DIR], mpidir=$withval, mpidir=no) AC_ARG_WITH(mpi, [ --with-mpi=MPI_DIR], mpidir=$withval, mpidir=no)
AC_SUBST(MPI_DIR, $mpidir) AC_SUBST(MPI_DIR, $mpidir)
if test $mpidir != no ; then if test $mpidir != no ; then
AC_SUBST(CXX, $mpidir/bin/mpiCC) AC_SUBST(CXX, $mpidir/bin/mpicxx)
AC_SUBST(CC, $mpidir/bin/mpicc) AC_SUBST(CC, $mpidir/bin/mpicc)
fi fi
......
...@@ -44,7 +44,7 @@ available_tags=" CXX F77" ...@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG # ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host p2q084: # Libtool was configured on host p1q020:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac` ...@@ -6760,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End: # End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX # ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host p2q084: # Libtool was configured on host p1q020:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -6804,7 +6804,7 @@ LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc" ...@@ -6804,7 +6804,7 @@ LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -7065,7 +7065,7 @@ include_expsyms="" ...@@ -7065,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77 # ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host p2q084: # Libtool was configured on host p1q020:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
......
...@@ -63,6 +63,8 @@ namespace AMDiS { ...@@ -63,6 +63,8 @@ namespace AMDiS {
Element *el = bound.el; Element *el = bound.el;
TEST_EXIT_DBG(el)("No element!\n");
clear(); clear();
int s1 = el->getSubObjOfChild(0, bound.subObj, bound.ithObj, bound.elType); int s1 = el->getSubObjOfChild(0, bound.subObj, bound.ithObj, bound.elType);
......
...@@ -33,7 +33,7 @@ namespace AMDiS { ...@@ -33,7 +33,7 @@ namespace AMDiS {
if (Parameters::singlett->paramInfo) { if (Parameters::singlett->paramInfo) {
if (Parameters::singlett->paramInfo > 1) if (Parameters::singlett->paramInfo > 1)
info = max(info, Parameters::singlett->paramInfo-1); info = max(info, Parameters::singlett->paramInfo - 1);
} else { } else {
info = 0; info = 0;
} }
...@@ -47,7 +47,8 @@ namespace AMDiS { ...@@ -47,7 +47,8 @@ namespace AMDiS {
} }
} }
i = std::find(Parameters::singlett->allParam.begin(),Parameters::singlett->allParam.end(),tParam); i = std::find(Parameters::singlett->allParam.begin(),
Parameters::singlett->allParam.end(), tParam);
if (i == Parameters::singlett->allParam.end()) { if (i == Parameters::singlett->allParam.end()) {
if (funcName != funcName2) { if (funcName != funcName2) {
...@@ -186,6 +187,7 @@ namespace AMDiS { ...@@ -186,6 +187,7 @@ namespace AMDiS {
return count; return count;
} }
int Parameters::getGlobalParameter(int flag, const std::string& key, int Parameters::getGlobalParameter(int flag, const std::string& key,
std::string* param) std::string* param)
{ {
...@@ -197,6 +199,7 @@ namespace AMDiS { ...@@ -197,6 +199,7 @@ namespace AMDiS {
return result; return result;
} }
void Parameters::read(const std::string& aFilename, void Parameters::read(const std::string& aFilename,
const std::string& maxKey) const std::string& maxKey)
{ {
...@@ -238,6 +241,7 @@ namespace AMDiS { ...@@ -238,6 +241,7 @@ namespace AMDiS {
singlett->inputFile.close(); singlett->inputFile.close();
} }
const std::string& Parameters::getActFile(const std::string& aFilename) const std::string& Parameters::getActFile(const std::string& aFilename)
{ {
FUNCNAME("Parameters::getActFile()"); FUNCNAME("Parameters::getActFile()");
...@@ -255,6 +259,7 @@ namespace AMDiS { ...@@ -255,6 +259,7 @@ namespace AMDiS {
} }
} }
void Parameters::swap(int i, int j) void Parameters::swap(int i, int j)
{ {
param tmp(allParam[i]); param tmp(allParam[i]);
...@@ -262,6 +267,7 @@ namespace AMDiS { ...@@ -262,6 +267,7 @@ namespace AMDiS {
allParam[j] = tmp; allParam[j] = tmp;
} }
void Parameters::qsort(int left, int right) void Parameters::qsort(int left, int right)
{ {
if (left >= right) if (left >= right)
......
...@@ -171,9 +171,9 @@ namespace AMDiS { ...@@ -171,9 +171,9 @@ namespace AMDiS {
* specified in a parameter file. * specified in a parameter file.
* \see GET_PARAMETER * \see GET_PARAMETER
*/ */
static int getGlobalParameter(int flag, static int getGlobalParameter(int flag,
const std::string& key, const std::string& key,
const char *format, const char *format,
...); ...);
/// Like getGlobalParameter(flag, key, "%s", param->c_str()). /// Like getGlobalParameter(flag, key, "%s", param->c_str()).
...@@ -245,7 +245,7 @@ namespace AMDiS { ...@@ -245,7 +245,7 @@ namespace AMDiS {
const std::string& nparameters, const std::string& nparameters,
const std::string& nfilename, const std::string& nfilename,
const std::string& nfuncName, const std::string& nfuncName,
int line) int line)
: key(nkey), : key(nkey),
parameters(nparameters), parameters(nparameters),
filename(nfilename), filename(nfilename),
...@@ -327,15 +327,14 @@ namespace AMDiS { ...@@ -327,15 +327,14 @@ namespace AMDiS {
private: private:
std::string cppFlags; std::string cppFlags;
std::ifstream inputFile; std::ifstream inputFile;
std::list< std::string> filenames; std::list<std::string> filenames;
size_t maxFiles; size_t maxFiles;
std::string filename; std::string filename;
int paramInfo; int paramInfo;
int msgInfo; int msgInfo;
int msgWait; int msgWait;
std::vector<param> allParam; std::vector<param> allParam;
};
};
/** \brief /** \brief
......
...@@ -184,6 +184,8 @@ namespace AMDiS { ...@@ -184,6 +184,8 @@ namespace AMDiS {
MSG("Deserialization from file: %s\n", serializationFilename.c_str()); MSG("Deserialization from file: %s\n", serializationFilename.c_str());
#endif #endif
deserialized = true;
} else { } else {
int globalRefinements = 0; int globalRefinements = 0;
...@@ -476,12 +478,8 @@ namespace AMDiS { ...@@ -476,12 +478,8 @@ namespace AMDiS {
GET_PARAMETER(0, name + "->output->write serialization", "%d", &writeSerialization); GET_PARAMETER(0, name + "->output->write serialization", "%d", &writeSerialization);
// Serialization is not allowed to be done by the problem, if its part of a parallel
// problem definition. Than, the parallel problem object must be serialized.
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
if (writeSerialization) if (writeSerialization)
fileWriters.push_back(new Serializer<ProblemVec>(this)); fileWriters.push_back(new Serializer<ProblemVec>(this));
#endif
} }
......
...@@ -63,6 +63,7 @@ namespace AMDiS { ...@@ -63,6 +63,7 @@ namespace AMDiS {
systemMatrix(NULL), systemMatrix(NULL),
useGetBound(true), useGetBound(true),
info(10), info(10),
deserialized(false),
computeExactError(false), computeExactError(false),
boundaryConditionSet(false), boundaryConditionSet(false),
writeAsmInfo(false) writeAsmInfo(false)
...@@ -411,6 +412,12 @@ namespace AMDiS { ...@@ -411,6 +412,12 @@ namespace AMDiS {
return info; return info;
} }
/// Returns \ref deserialized;
bool isDeserialized()
{
return deserialized;
}
/** \} */ /** \} */
/** \name setting methods /** \name setting methods
...@@ -601,6 +608,9 @@ namespace AMDiS { ...@@ -601,6 +608,9 @@ namespace AMDiS {
/// Info level. /// Info level.
int info; int info;
/// If true, the stationary problem was deserialized from some serialization file.
bool deserialized;
/** \brief /** \brief
* This vectors stores pointers to functions defining the exact solution of * This vectors stores pointers to functions defining the exact solution of
* the problem. This may be used to compute the real error of the computed * the problem. This may be used to compute the real error of the computed
......
...@@ -46,13 +46,30 @@ namespace AMDiS { ...@@ -46,13 +46,30 @@ namespace AMDiS {
GET_PARAMETER(0, problem->getName() + "->output->serialization filename", &name); GET_PARAMETER(0, problem->getName() + "->output->serialization filename", &name);
GET_PARAMETER(0, problem->getName() + "->output->write every i-th timestep", GET_PARAMETER(0, problem->getName() + "->output->write every i-th timestep",
"%d", &tsModulo); "%d", &tsModulo);
TEST_EXIT(name != "")("no filename\n"); TEST_EXIT(name != "")("No filename!\n");
#if HAVE_PARALLEL_DOMAIN_AMDIS #if HAVE_PARALLEL_DOMAIN_AMDIS
name += ".p" + boost::lexical_cast<std::string>(MPI::COMM_WORLD.Get_rank()); name += ".p" + boost::lexical_cast<std::string>(MPI::COMM_WORLD.Get_rank());
#endif #endif
} }
Serializer(ProblemType *prob, std::string filename, int writeEveryIth)
: name(filename),
problem(prob),
tsModulo(writeEveryIth),
timestepNumber(-1)
{
FUNCNAME("Serializer::Serializer()");
TEST_EXIT(name != "")("No filename!\n");
#if HAVE_PARALLEL_DOMAIN_AMDIS
name += ".p" + boost::lexical_cast<std::string>(MPI::COMM_WORLD.Get_rank());
#endif
}
virtual ~Serializer() {} virtual ~Serializer() {}
virtual void writeFiles(AdaptInfo *adaptInfo, virtual void writeFiles(AdaptInfo *adaptInfo,
......
...@@ -120,6 +120,8 @@ namespace AMDiS { ...@@ -120,6 +120,8 @@ namespace AMDiS {
void InteriorBoundary::deserialize(std::istream &in, void InteriorBoundary::deserialize(std::istream &in,
std::map<int, Element*> &elIndexMap) std::map<int, Element*> &elIndexMap)
{ {
FUNCNAME("InteriorBoundary::deserialize()");
int mSize = 0; int mSize = 0;
SerUtil::deserialize(in, mSize); SerUtil::deserialize(in, mSize);
for (int i = 0; i < mSize; i++) { for (int i = 0; i < mSize; i++) {
...@@ -146,6 +148,10 @@ namespace AMDiS { ...@@ -146,6 +148,10 @@ namespace AMDiS {
SerUtil::deserialize(in, bound.neighObj.reverseMode); SerUtil::deserialize(in, bound.neighObj.reverseMode);
deserializeExcludeList(in, bound.neighObj.excludedSubstructures); deserializeExcludeList(in, bound.neighObj.excludedSubstructures);
TEST_EXIT_DBG(elIndexMap.count(bound.rankObj.elIndex) == 1)
("Cannot find element with index %d for deserialization!\n",
bound.rankObj.elIndex);
bound.rankObj.el = elIndexMap[bound.rankObj.elIndex]; bound.rankObj.el = elIndexMap[bound.rankObj.elIndex];
bound.neighObj.el = NULL; bound.neighObj.el = NULL;
} }
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
#include <boost/lexical_cast.hpp> #include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include "parallel/ParallelDomainBase.h" #include "parallel/ParallelDomainBase.h"
#include "parallel/ParallelDomainDbg.h" #include "parallel/ParallelDomainDbg.h"
#include "parallel/StdMpi.h" #include "parallel/StdMpi.h"
...@@ -29,6 +31,7 @@ ...@@ -29,6 +31,7 @@
namespace AMDiS { namespace AMDiS {
using boost::lexical_cast; using boost::lexical_cast;
using namespace boost::filesystem;
inline bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2) inline bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2)
{ {
...@@ -49,6 +52,7 @@ namespace AMDiS { ...@@ -49,6 +52,7 @@ namespace AMDiS {
nOverallDofs(0), nOverallDofs(0),
rstart(0), rstart(0),
deserialized(false), deserialized(false),
writeSerializationFile(false),
lastMeshChangeIndex(0), lastMeshChangeIndex(0),
macroElementStructureConsisten(false) macroElementStructureConsisten(false)
{ {
...@@ -253,23 +257,49 @@ namespace AMDiS { ...@@ -253,23 +257,49 @@ namespace AMDiS {
// Create parallel serialization file writer, if needed. // Create parallel serialization file writer, if needed.
int writeSerialization = 0; int writeSerialization = 0;
GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d", &writeSerialization); GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d",
if (writeSerialization) &writeSerialization);
probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this)); if (writeSerialization && !writeSerializationFile) {
std::string f = "";
GET_PARAMETER(0, probVec->getName() + "->output->serialization filename", &f);
path myPath(f);
std::string meshFilename =
myPath.parent_path().directory_string() + "/meshDistributor.ser";
int tsModulo = -1;
GET_PARAMETER(0, probVec->getName() + "->output->write every i-th timestep",
"%d", &tsModulo);
probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, meshFilename, tsModulo));
writeSerializationFile = true;
}
int readSerialization = 0; int readSerialization = 0;
GET_PARAMETER(0, probVec->getName() + "->input->read serialization", "%d", &readSerialization); GET_PARAMETER(0, probVec->getName() + "->input->read serialization", "%d",
&readSerialization);
if (readSerialization) { if (readSerialization) {
ERROR_EXIT("Must be reimplemented!\n");
#if 0
std::string filename = ""; std::string filename = "";
GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename); GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename);
filename += ".p" + lexical_cast<std::string>(mpiRank); filename += ".p" + lexical_cast<std::string>(mpiRank);
MSG("Start serialization with %s\n", filename.c_str()); MSG("Start deserialization with %s\n", filename.c_str());
std::ifstream in(filename.c_str()); std::ifstream in(filename.c_str());
deserialize(in); probVec->deserialize(in);
in.close(); in.close();
#endif MSG("Deserialization from file: %s\n", filename.c_str());
if (!deserialized) {
GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename);
path myPath(filename);
std::string meshFilename =
myPath.parent_path().directory_string() + "/meshDistributor.ser.p" +
lexical_cast<std::string>(mpiRank);
std::ifstream in(meshFilename.c_str());
deserialize(in);
in.close();
MSG("Deserializtion of mesh distributor from file: %s\n", meshFilename.c_str());
deserialized = true;
}
} }
probStat.push_back(probVec); probStat.push_back(probVec);
...@@ -475,6 +505,10 @@ namespace AMDiS { ...@@ -475,6 +505,10 @@ namespace AMDiS {
mesh->dofCompress(); mesh->dofCompress();
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
// === Update periodic mapping, if there are periodic boundaries. ===
createPeriodicMap();
} }
...@@ -990,10 +1024,14 @@ namespace AMDiS { ...@@ -990,10 +1024,14 @@ namespace AMDiS {
// === to \ref otherIntBoundary. It dependes on which rank is respon- === // === to \ref otherIntBoundary. It dependes on which rank is respon- ===
// === sible for this boundary. === // === sible for this boundary. ===
if (BoundaryManager::isBoundaryPeriodic(elInfo->getBoundary(subObj, i))) { if (BoundaryManager::isBoundaryPeriodic(elInfo->getBoundary(subObj, i))) {
// We have found an element that is at an interior, periodic boundary. // We have found an element that is at an interior, periodic boundary.
AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank); AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank);
b = bound; b = bound;
if (mpiRank > otherElementRank)
b.rankObj.setReverseMode(b.neighObj, feSpace);
else
b.neighObj.setReverseMode(b.rankObj, feSpace);
} else { } else {
// We have found an element that is at an interior, non-periodic boundary. // We have found an element that is at an interior, non-periodic boundary.
if (mpiRank > otherElementRank) { if (mpiRank > otherElementRank) {
...@@ -1927,7 +1965,7 @@ namespace AMDiS { ...@@ -1927,7 +1965,7 @@ namespace AMDiS {
// Send the global indices to the rank on the other side. // Send the global indices to the rank on the other side.
stdMpi.getSendData(it->first).reserve(dofs.size()); stdMpi.getSendData(it->first).reserve(dofs.size());
for (int i = 0; i < static_cast<int>(dofs.size()); i++) for (unsigned int i = 0; i < dofs.size(); i++)
stdMpi.getSendData(it->first).push_back(mapLocalGlobalDofs[*(dofs[i])]); stdMpi.getSendData(it->first).push_back(mapLocalGlobalDofs[*(dofs[i])]);
// Receive from this rank the same number of dofs. // Receive from this rank the same number of dofs.
...@@ -1955,14 +1993,14 @@ namespace AMDiS { ...@@ -1955,14 +1993,14 @@ namespace AMDiS {
boundIt->rankObj.el->getNonVertexDofs(feSpace, boundIt->rankObj, tmpdofs); boundIt->rankObj.el->getNonVertexDofs(feSpace, boundIt->rankObj, tmpdofs);
boundIt->rankObj.el->getVertexDofs(feSpace, boundIt->rankObj, tmpdofs); boundIt->rankObj.el->getVertexDofs(feSpace, boundIt->rankObj, tmpdofs);
for (int j = static_cast<int>(tmpdofs.size()) - 1; j >= 0; j--) for (unsigned int i = 0; i < tmpdofs.size(); i++)
dofs.push_back(tmpdofs[j]); dofs.push_back(tmpdofs[i]);
} }
// Added the received dofs to the mapping. // Added the received dofs to the mapping.
for (int j = 0; j < static_cast<int>(dofs.size()); j++) { for (int i = 0; i < static_cast<int>(dofs.size()); i++) {
int globalDofIndex = mapLocalGlobalDofs[*(dofs[j])]; int globalDofIndex = mapLocalGlobalDofs[*(dofs[i])];
periodicDof[globalDofIndex].insert(stdMpi.getRecvData(it->first)[j]); periodicDof[globalDofIndex].insert(stdMpi.getRecvData(it->first)[i]);
dofFromRank[globalDofIndex].insert(it->first); dofFromRank[globalDofIndex].insert(it->first);
} }
} }
...@@ -2028,9 +2066,11 @@ namespace AMDiS { ...@@ -2028,9 +2066,11 @@ namespace AMDiS {
SerUtil::serialize(out, partitionVec); SerUtil::serialize(out, partitionVec);
SerUtil::serialize(out, oldPartitionVec); SerUtil::serialize(out, oldPartitionVec);
SerUtil::serialize(out, nRankDofs); SerUtil::serialize(out, nRankDofs);
SerUtil::serialize(out, nOverallDofs);
myIntBoundary.serialize(out); myIntBoundary.serialize(out);
otherIntBoundary.serialize(out); otherIntBoundary.serialize(out);
periodicBoundary.serialize(out);
serialize(out, sendDofs); serialize(out, sendDofs);
serialize(out, recvDofs); serialize(out, recvDofs);
...@@ -2043,6 +2083,7 @@ namespace AMDiS { ...@@ -2043,6 +2083,7 @@ namespace AMDiS {
serialize(out, periodicDof); serialize(out, periodicDof);
SerUtil::serialize(out, rstart); SerUtil::serialize(out, rstart);
SerUtil::serialize(out, macroElementStructureConsisten);
} }
...@@ -2053,6 +2094,7 @@ namespace AMDiS { ...@@ -2053,6 +2094,7 @@ namespace AMDiS {
SerUtil::deserialize(in, partitionVec);