Commit cf698b4a authored by Thomas Witkowski's avatar Thomas Witkowski

Work on serialization of parallel problems.

parent 4c2d3483
...@@ -8,4 +8,11 @@ namespace AMDiS { ...@@ -8,4 +8,11 @@ namespace AMDiS {
return boundary[rank][boundary[rank].size() - 1]; return boundary[rank][boundary[rank].size() - 1];
} }
void InteriorBoundary::serialize(std::ostream &out)
{
}
void InteriorBoundary::deserialize(std::istream &in)
{
}
} }
...@@ -81,6 +81,10 @@ namespace AMDiS { ...@@ -81,6 +81,10 @@ namespace AMDiS {
AtomicBoundary& getNewAtomicBoundary(int rank); AtomicBoundary& getNewAtomicBoundary(int rank);
void serialize(std::ostream &out);
void deserialize(std::istream &in);
public: public:
RankToBoundMap boundary; RankToBoundMap boundary;
}; };
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include "ProblemStatBase.h" #include "ProblemStatBase.h"
#include "StandardProblemIteration.h" #include "StandardProblemIteration.h"
#include "ElementFileWriter.h" #include "ElementFileWriter.h"
#include "Serializer.h"
#include "petscksp.h" #include "petscksp.h"
...@@ -45,7 +46,7 @@ namespace AMDiS { ...@@ -45,7 +46,7 @@ namespace AMDiS {
mesh(fe->getMesh()), mesh(fe->getMesh()),
refinementManager(refineManager), refinementManager(refineManager),
initialPartitionMesh(true), initialPartitionMesh(true),
nRankDOFs(0), nRankDofs(0),
rstart(0), rstart(0),
nComponents(1) nComponents(1)
{ {
...@@ -92,11 +93,11 @@ namespace AMDiS { ...@@ -92,11 +93,11 @@ namespace AMDiS {
// Set of all DOFs of the rank. // Set of all DOFs of the rank.
std::vector<const DegreeOfFreedom*> rankDOFs; std::vector<const DegreeOfFreedom*> rankDOFs;
// Number of DOFs in ranks partition that are owned by the rank. // Number of DOFs in ranks partition that are owned by the rank.
nRankDOFs = 0; nRankDofs = 0;
// Number of all DOFs in the macro mesh. // Number of all DOFs in the macro mesh.
int nOverallDOFs = 0; int nOverallDOFs = 0;
createLocalGlobalNumbering(rankDOFs, nRankDOFs, nOverallDOFs); createLocalGlobalNumbering(rankDOFs, nRankDofs, nOverallDOFs);
// === Create interior boundary information === // === Create interior boundary information ===
...@@ -130,7 +131,7 @@ namespace AMDiS { ...@@ -130,7 +131,7 @@ namespace AMDiS {
DbgCreateElementMap(elMap); DbgCreateElementMap(elMap);
#endif #endif
updateLocalGlobalNumbering(nRankDOFs, nOverallDOFs); updateLocalGlobalNumbering(nRankDofs, nOverallDOFs);
updateDofAdmins(); updateDofAdmins();
...@@ -144,7 +145,7 @@ namespace AMDiS { ...@@ -144,7 +145,7 @@ namespace AMDiS {
DbgTestCommonDofs(true); DbgTestCommonDofs(true);
#endif #endif
nRankRows = nRankDOFs * nComponents; nRankRows = nRankDofs * nComponents;
nOverallRows = nOverallDOFs * nComponents; nOverallRows = nOverallDOFs * nComponents;
} }
...@@ -535,7 +536,7 @@ namespace AMDiS { ...@@ -535,7 +536,7 @@ namespace AMDiS {
PetscScalar *vecPointer; PetscScalar *vecPointer;
VecGetArray(petscSolVec, &vecPointer); VecGetArray(petscSolVec, &vecPointer);
for (int i = 0; i < nRankDOFs; i++) for (int i = 0; i < nRankDofs; i++)
(*vec)[mapLocalToDofIndex[i]] = vecPointer[i]; (*vec)[mapLocalToDofIndex[i]] = vecPointer[i];
VecRestoreArray(petscSolVec, &vecPointer); VecRestoreArray(petscSolVec, &vecPointer);
...@@ -613,7 +614,7 @@ namespace AMDiS { ...@@ -613,7 +614,7 @@ namespace AMDiS {
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < nComponents; i++) {
DOFVector<double> *dofvec = vec.getDOFVector(i); DOFVector<double> *dofvec = vec.getDOFVector(i);
for (int j = 0; j < nRankDOFs; j++) for (int j = 0; j < nRankDofs; j++)
(*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i]; (*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
} }
...@@ -895,7 +896,7 @@ namespace AMDiS { ...@@ -895,7 +896,7 @@ namespace AMDiS {
void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs, void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs,
int& nRankDOFs, int& nRankDofs,
int& nOverallDOFs) int& nOverallDOFs)
{ {
FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()");
...@@ -909,15 +910,15 @@ namespace AMDiS { ...@@ -909,15 +910,15 @@ namespace AMDiS {
createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDofs, vertexDof); createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDofs, vertexDof);
nRankDOFs = rankDOFs.size(); nRankDofs = rankDOFs.size();
nOverallDOFs = partitionDOFs.size(); nOverallDOFs = partitionDOFs.size();
// === Get starting position for global rank dof ordering. ==== // === Get starting position for global rank dof ordering. ====
rstart = 0; rstart = 0;
mpiComm.Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM); mpiComm.Scan(&nRankDofs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDOFs; rstart -= nRankDofs;
// === Create for all dofs in rank new indices. The new index must start at === // === Create for all dofs in rank new indices. The new index must start at ===
...@@ -1115,7 +1116,7 @@ namespace AMDiS { ...@@ -1115,7 +1116,7 @@ namespace AMDiS {
} }
void ParallelDomainBase::updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs) void ParallelDomainBase::updateLocalGlobalNumbering(int& nRankDofs, int& nOverallDOFs)
{ {
FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()");
...@@ -1220,17 +1221,17 @@ namespace AMDiS { ...@@ -1220,17 +1221,17 @@ namespace AMDiS {
} }
} }
nRankDOFs = rankDOFs.size(); nRankDofs = rankDOFs.size();
// === Get starting position for global rank dof ordering. ==== // === Get starting position for global rank dof ordering. ====
mpiComm.Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM); mpiComm.Scan(&nRankDofs, &rstart, 1, MPI_INT, MPI_SUM);
rstart -= nRankDOFs; rstart -= nRankDofs;
// === Calculate number of overall DOFs of all partitions. === // === Calculate number of overall DOFs of all partitions. ===
mpiComm.Allreduce(&nRankDOFs, &nOverallDOFs, 1, MPI_INT, MPI_SUM); mpiComm.Allreduce(&nRankDofs, &nOverallDOFs, 1, MPI_INT, MPI_SUM);
// Do not change the indices now, but create a new indexing a store it here. // Do not change the indices now, but create a new indexing a store it here.
...@@ -1510,6 +1511,57 @@ namespace AMDiS { ...@@ -1510,6 +1511,57 @@ namespace AMDiS {
} }
Flag ParallelDomainBase::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
{
FUNCNAME("ParallelDomainBase::oneIteration()");
Flag flag = dynamic_cast<StandardProblemIteration*>(iterationIF)->
buildAndAdapt(adaptInfo, toDo);
if (toDo.isSet(SOLVE))
solve();
if (toDo.isSet(SOLVE_RHS))
ERROR_EXIT("Not yet implemented!\n");
if (toDo.isSet(ESTIMATE))
iterationIF->getProblem()->estimate(adaptInfo);
return flag;
}
void ParallelDomainBase::serialize(std::ostream &out)
{
SerUtil::serialize(out, elemWeights);
SerUtil::serialize(out, &initialPartitionMesh);
SerUtil::serialize(out, partitionVec);
SerUtil::serialize(out, oldPartitionVec);
SerUtil::serialize(out, &nRankDofs);
myIntBoundary.serialize(out);
otherIntBoundary.serialize(out);
// SerUtil::serialize(out, sendDofs);
// SerUtil::serialize(out, recvDofs);
SerUtil::serialize(out, mapLocalGlobalDOFs);
SerUtil::serialize(out, mapLocalToDofIndex);
SerUtil::serialize(out, isRankDof);
// SerUtil::serialize(out, vertexDof);
SerUtil::serialize(out, &rstart);
SerUtil::serialize(out, &nRankRows);
SerUtil::serialize(out, &nOverallRows);
}
void ParallelDomainBase::deserialize(std::istream &in)
{
}
void ParallelDomainBase::DbgCreateElementMap(ElementIdxToDofs &elMap) void ParallelDomainBase::DbgCreateElementMap(ElementIdxToDofs &elMap)
{ {
FUNCNAME("ParallelDomainBase::DbgCreateElementMap()"); FUNCNAME("ParallelDomainBase::DbgCreateElementMap()");
...@@ -1793,24 +1845,4 @@ namespace AMDiS { ...@@ -1793,24 +1845,4 @@ namespace AMDiS {
ElementFileWriter::writeFile(vec, feSpace, filename); ElementFileWriter::writeFile(vec, feSpace, filename);
} }
Flag ParallelDomainBase::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
{
FUNCNAME("ParallelDomainBase::oneIteration()");
Flag flag = dynamic_cast<StandardProblemIteration*>(iterationIF)->
buildAndAdapt(adaptInfo, toDo);
if (toDo.isSet(SOLVE))
solve();
if (toDo.isSet(SOLVE_RHS))
ERROR_EXIT("Not yet implemented!\n");
if (toDo.isSet(ESTIMATE))
iterationIF->getProblem()->estimate(adaptInfo);
return flag;
}
} }
...@@ -163,9 +163,9 @@ namespace AMDiS { ...@@ -163,9 +163,9 @@ namespace AMDiS {
} }
/// Returns \ref nRankDOFs, the number of DOFs in the rank mesh. /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
int getNumberRankDOFs() int getNumberRankDofs()
{ {
return nRankDOFs; return nRankDofs;
} }
void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec); void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec);
...@@ -181,10 +181,9 @@ namespace AMDiS { ...@@ -181,10 +181,9 @@ namespace AMDiS {
return NULL; return NULL;
} }
virtual void serialize(std::ostream&) {} virtual void serialize(std::ostream &out);
virtual void deserialize(std::istream&) {}
virtual void deserialize(std::istream &in);
protected: protected:
/** \brief /** \brief
...@@ -376,16 +375,17 @@ namespace AMDiS { ...@@ -376,16 +375,17 @@ namespace AMDiS {
*/ */
std::map<int, int> oldPartitionVec; std::map<int, int> oldPartitionVec;
/// Petsc's matrix structure.
Mat petscMatrix; Mat petscMatrix;
Vec petscRhsVec; /** \brief
* Petsc's vector structures for the rhs vector, the solution vector and a
* temporary vector for calculating the final residuum.
*/
Vec petscRhsVec, petscSolVec, petscTmpVec;
Vec petscSolVec;
Vec petscTmpVec;
/// Number of DOFs in the rank mesh. /// Number of DOFs in the rank mesh.
int nRankDOFs; int nRankDofs;
/** \brief /** \brief
* Defines the interior boundaries of the domain that result from partitioning * Defines the interior boundaries of the domain that result from partitioning
...@@ -430,12 +430,19 @@ namespace AMDiS { ...@@ -430,12 +430,19 @@ namespace AMDiS {
DofToBool vertexDof; DofToBool vertexDof;
/// Is the index of the first row of the linear system, which is owned by the rank.
int rstart; int rstart;
/** \brief
* Number of components of the equation. Is used to calculate the exact number
* of rows in the the overall linear system.
*/
int nComponents; int nComponents;
/// Number of rows of the whole linear system that are stored on this rank.
int nRankRows; int nRankRows;
/// Overall number of the rows in the lineary system.
int nOverallRows; int nOverallRows;
}; };
......
...@@ -1309,7 +1309,7 @@ namespace AMDiS { ...@@ -1309,7 +1309,7 @@ namespace AMDiS {
{ {
FUNCNAME("ProblemVec::serialize()"); FUNCNAME("ProblemVec::serialize()");
SerializerUtil::serializeBool(out, &allowFirstRef); SerUtil::serialize(out, &allowFirstRef);
for (int i = 0; i < static_cast<int>(meshes.size()); i++) for (int i = 0; i < static_cast<int>(meshes.size()); i++)
meshes[i]->serialize(out); meshes[i]->serialize(out);
...@@ -1321,7 +1321,7 @@ namespace AMDiS { ...@@ -1321,7 +1321,7 @@ namespace AMDiS {
{ {
FUNCNAME("ProblemVec::deserialize()"); FUNCNAME("ProblemVec::deserialize()");
SerializerUtil::deserializeBool(in, &allowFirstRef); SerUtil::deserialize(in, &allowFirstRef);
for (int i = 0; i < static_cast<int>(meshes.size()); i++) for (int i = 0; i < static_cast<int>(meshes.size()); i++)
meshes[i]->deserialize(in); meshes[i]->deserialize(in);
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#ifndef AMDIS_SERIALIZER_H #ifndef AMDIS_SERIALIZER_H
#define AMDIS_SERIALIZER_H #define AMDIS_SERIALIZER_H
#include <map>
#include "FileWriter.h" #include "FileWriter.h"
#include "Parameters.h" #include "Parameters.h"
#include "AdaptInfo.h" #include "AdaptInfo.h"
...@@ -33,17 +34,17 @@ namespace AMDiS { ...@@ -33,17 +34,17 @@ namespace AMDiS {
class Serializer : public FileWriterInterface class Serializer : public FileWriterInterface
{ {
public: public:
Serializer(ProblemType *problem) Serializer(ProblemType *prob)
: name_(""), : name(""),
problem_(problem), problem(prob),
tsModulo_(1), tsModulo(1),
timestepNumber_(-1) timestepNumber(-1)
{ {
GET_PARAMETER(0, problem_->getName() + "->output->serialization filename", GET_PARAMETER(0, problem->getName() + "->output->serialization filename",
&name_); &name);
GET_PARAMETER(0, problem_->getName() + "->output->write every i-th timestep", GET_PARAMETER(0, problem->getName() + "->output->write every i-th timestep",
"%d", &tsModulo_); "%d", &tsModulo);
TEST_EXIT(name_ != "")("no filename\n"); TEST_EXIT(name != "")("no filename\n");
} }
virtual ~Serializer() {} virtual ~Serializer() {}
...@@ -56,76 +57,65 @@ namespace AMDiS { ...@@ -56,76 +57,65 @@ namespace AMDiS {
{ {
FUNCNAME("Serializer::writeFiles()"); FUNCNAME("Serializer::writeFiles()");
timestepNumber_++; timestepNumber++;
timestepNumber_ %= tsModulo_; timestepNumber %= tsModulo;
if ((timestepNumber_ != 0) && !force) if ((timestepNumber != 0) && !force)
return; return;
TEST_EXIT(adaptInfo)("No AdaptInfo\n"); TEST_EXIT(adaptInfo)("No AdaptInfo\n");
std::ofstream out(name_.c_str()); std::ofstream out(name.c_str());
TEST_EXIT(out.is_open())("Cannot open serialization file!\n"); TEST_EXIT(out.is_open())("Cannot open serialization file!\n");
problem_->serialize(out); problem->serialize(out);
adaptInfo->serialize(out); adaptInfo->serialize(out);
out.close(); out.close();
MSG("problem serialized to %s \n", name_.c_str()); MSG("problem serialized to %s \n", name.c_str());
}
void writeDelayedFiles() {}
bool isWritingDelayed()
{
return false;
} }
protected: protected:
/// Name of file to which the problem is serialized. /// Name of file to which the problem is serialized.
std::string name_; std::string name;
/// Pointer to the problem. /// Pointer to the problem.
ProblemType *problem_; ProblemType *problem;
/// The problem is serialized every tsModulo-th timestep. /// The problem is serialized every tsModulo-th timestep.
int tsModulo_; int tsModulo;
/// Current timestep number. /// Current timestep number.
int timestepNumber_; int timestepNumber;
}; };
namespace SerUtil {
class SerializerUtil { template<typename T>
public: void serialize(std::ostream &out, T *data)
static void serializeInt(std::ostream &out, int* ptrInt)
{
out.write(reinterpret_cast<const char*>(ptrInt), sizeof(int));
}
static void serializeDouble(std::ostream &out, double* ptrInt)
{
out.write(reinterpret_cast<const char*>(ptrInt), sizeof(double));
}
static void serializeBool(std::ostream &out, bool* ptrBool)
{
out.write(reinterpret_cast<const char*>(ptrBool), sizeof(bool));
}
static void deserializeInt(std::istream &in, int* ptrInt)
{ {
in.read(reinterpret_cast<char*>(ptrInt), sizeof(int)); out.write(reinterpret_cast<const char*>(data), sizeof(T));
} }
static void deserializeDouble(std::istream &in, double* ptrInt) template<typename T>
void deserialize(std::istream &in, T *data)
{ {
in.read(reinterpret_cast<char*>(ptrInt), sizeof(double)); in.read(reinterpret_cast<char*>(data), sizeof(T));
} }
static void deserializeBool(std::istream &in, bool* ptrBool) template<typename T1, typename T2>
void serialize(std::ostream &out, std::map<T1, T2> &data)
{ {
in.read(reinterpret_cast<char*>(ptrBool), sizeof(bool)); int mapSize = data.size();
serialize(out, &mapSize);
for (typename std::map<T1,T2>::iterator it = data.begin();
it != data.end(); ++it) {
T1 v1 = it->first;
T2 v2 = it->second;
serialize(out, &v1);
serialize(out, &v2);
}
} }
}
};
} }
#endif #endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment