Commit c52b43fe authored by Thomas Witkowski's avatar Thomas Witkowski

Some comments in parallel code, changed code of parallel serialization-deserialization.

parent 0f61523c
...@@ -416,7 +416,9 @@ namespace AMDiS { ...@@ -416,7 +416,9 @@ namespace AMDiS {
MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows, MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
0, d_nnz, 0, o_nnz, &petscMatrix); 0, d_nnz, 0, o_nnz, &petscMatrix);
#if (DEBUG != 0)
INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock())); INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock()));
#endif
#if (DEBUG != 0) #if (DEBUG != 0)
int a, b; int a, b;
...@@ -435,8 +437,9 @@ namespace AMDiS { ...@@ -435,8 +437,9 @@ namespace AMDiS {
if ((*mat)[i][j]) if ((*mat)[i][j])
setDofMatrix((*mat)[i][j], nComponents, i, j); setDofMatrix((*mat)[i][j], nComponents, i, j);
#if (DEBUG != 0)
INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock())); INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock()));
#endif
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
......
...@@ -106,6 +106,8 @@ namespace AMDiS { ...@@ -106,6 +106,8 @@ namespace AMDiS {
}; };
typedef GlobalMatrixSolver ParallelProblemStat;
} //namespace AMDiS } //namespace AMDiS
#endif #endif
...@@ -238,18 +238,17 @@ namespace AMDiS { ...@@ -238,18 +238,17 @@ namespace AMDiS {
GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d", GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d",
&writeSerialization); &writeSerialization);
if (writeSerialization && !writeSerializationFile) { if (writeSerialization && !writeSerializationFile) {
std::string f = ""; std::string filename = "";
GET_PARAMETER(0, probVec->getName() + "->output->serialization filename", &f); GET_PARAMETER(0, name + "->output->serialization filename", &filename);
path myPath(f);
std::string meshFilename =
myPath.parent_path().directory_string() + "/meshDistributor.ser";
TEST_EXIT(filename != "")
("No filename defined for parallel serialization file!\n");
int tsModulo = -1; int tsModulo = -1;
GET_PARAMETER(0, probVec->getName() + "->output->write every i-th timestep", GET_PARAMETER(0, probVec->getName() + "->output->write every i-th timestep",
"%d", &tsModulo); "%d", &tsModulo);
probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, meshFilename, tsModulo)); probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, filename, tsModulo));
writeSerializationFile = true; writeSerializationFile = true;
} }
...@@ -262,20 +261,30 @@ namespace AMDiS { ...@@ -262,20 +261,30 @@ namespace AMDiS {
filename += ".p" + lexical_cast<std::string>(mpiRank); filename += ".p" + lexical_cast<std::string>(mpiRank);
MSG("Start deserialization with %s\n", filename.c_str()); MSG("Start deserialization with %s\n", filename.c_str());
std::ifstream in(filename.c_str()); std::ifstream in(filename.c_str());
TEST_EXIT(!in.fail())("Could not open deserialization file: %s\n",
filename.c_str());
probVec->deserialize(in); probVec->deserialize(in);
in.close(); in.close();
MSG("Deserialization from file: %s\n", filename.c_str()); MSG("Deserialization from file: %s\n", filename.c_str());
if (!deserialized) { if (!deserialized) {
GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename); filename = "";
path myPath(filename); GET_PARAMETER(0, name + "->input->serialization filename", &filename);
std::string meshFilename =
myPath.parent_path().directory_string() + "/meshDistributor.ser.p" + TEST_EXIT(filename != "")
lexical_cast<std::string>(mpiRank); ("No filename defined for parallel deserialization file!\n");
std::ifstream in(meshFilename.c_str());
std::string rankFilename = filename + ".p" + lexical_cast<std::string>(mpiRank);
std::ifstream in(rankFilename.c_str());
TEST_EXIT(!in.fail())("Could not open parallel deserialization file: %s\n",
filename.c_str());
deserialize(in); deserialize(in);
in.close(); in.close();
MSG("Deserializtion of mesh distributor from file: %s\n", meshFilename.c_str()); MSG("Deserializtion of mesh distributor from file: %s\n", rankFilename.c_str());
deserialized = true; deserialized = true;
} }
} }
...@@ -1961,7 +1970,7 @@ namespace AMDiS { ...@@ -1961,7 +1970,7 @@ namespace AMDiS {
// Clear all periodic dof mappings calculated before. We do it from scratch. // Clear all periodic dof mappings calculated before. We do it from scratch.
periodicDof.clear(); periodicDof.clear();
perDofAssociations.clear(); periodicDofAssociations.clear();
StdMpi<std::vector<int> > stdMpi(mpiComm, false); StdMpi<std::vector<int> > stdMpi(mpiComm, false);
...@@ -2037,7 +2046,7 @@ namespace AMDiS { ...@@ -2037,7 +2046,7 @@ namespace AMDiS {
BoundaryType type = types[i]; BoundaryType type = types[i];
periodicDof[type][globalDofIndex] = mapGlobalDofIndex; periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
perDofAssociations[globalDofIndex].insert(type); periodicDofAssociations[globalDofIndex].insert(type);
dofFromRank[globalDofIndex].insert(it->first); dofFromRank[globalDofIndex].insert(it->first);
} }
} }
...@@ -2054,11 +2063,11 @@ namespace AMDiS { ...@@ -2054,11 +2063,11 @@ namespace AMDiS {
for (std::map<DegreeOfFreedom, std::set<int> >::iterator it = dofFromRank.begin(); for (std::map<DegreeOfFreedom, std::set<int> >::iterator it = dofFromRank.begin();
it != dofFromRank.end(); ++it) { it != dofFromRank.end(); ++it) {
if (it->second.size() == 2) { if (it->second.size() == 2) {
TEST_EXIT_DBG(perDofAssociations[it->first].size() == 2) TEST_EXIT_DBG(periodicDofAssociations[it->first].size() == 2)
("Missing periodic dof!\n"); ("Missing periodic dof!\n");
int type0 = *(perDofAssociations[it->first].begin()); int type0 = *(periodicDofAssociations[it->first].begin());
int type1 = *(++(perDofAssociations[it->first].begin())); int type1 = *(++(periodicDofAssociations[it->first].begin()));
int *sendbuf = new int[2]; int *sendbuf = new int[2];
sendbuf[0] = periodicDof[type0][it->first]; sendbuf[0] = periodicDof[type0][it->first];
...@@ -2098,7 +2107,7 @@ namespace AMDiS { ...@@ -2098,7 +2107,7 @@ namespace AMDiS {
int type = 3; int type = 3;
periodicDof[type][globalDofIndex] = mapGlobalDofIndex; periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
perDofAssociations[globalDofIndex].insert(type); periodicDofAssociations[globalDofIndex].insert(type);
} }
i++; i++;
...@@ -2112,8 +2121,8 @@ namespace AMDiS { ...@@ -2112,8 +2121,8 @@ namespace AMDiS {
delete [] recvBuffers[i]; delete [] recvBuffers[i];
#if (DEBUG != 0) #if (DEBUG != 0)
for (std::map<int, std::set<int> >::iterator it = perDofAssociations.begin(); for (std::map<int, std::set<BoundaryType> >::iterator it = periodicDofAssociations.begin();
it != perDofAssociations.end(); ++it) { it != periodicDofAssociations.end(); ++it) {
int nAssoc = it->second.size(); int nAssoc = it->second.size();
TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3) TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3)
("Should not happen! DOF %d has %d periodic associations!\n", ("Should not happen! DOF %d has %d periodic associations!\n",
...@@ -2146,7 +2155,7 @@ namespace AMDiS { ...@@ -2146,7 +2155,7 @@ namespace AMDiS {
serialize(out, vertexDof); serialize(out, vertexDof);
serialize(out, periodicDof); serialize(out, periodicDof);
serialize(out, perDofAssociations); serialize(out, periodicDofAssociations);
SerUtil::serialize(out, rstart); SerUtil::serialize(out, rstart);
SerUtil::serialize(out, macroElementStructureConsisten); SerUtil::serialize(out, macroElementStructureConsisten);
...@@ -2196,7 +2205,7 @@ namespace AMDiS { ...@@ -2196,7 +2205,7 @@ namespace AMDiS {
deserialize(in, vertexDof, dofMap); deserialize(in, vertexDof, dofMap);
deserialize(in, periodicDof); deserialize(in, periodicDof);
deserialize(in, perDofAssociations); deserialize(in, periodicDofAssociations);
SerUtil::deserialize(in, rstart); SerUtil::deserialize(in, rstart);
SerUtil::deserialize(in, macroElementStructureConsisten); SerUtil::deserialize(in, macroElementStructureConsisten);
......
...@@ -68,7 +68,9 @@ namespace AMDiS { ...@@ -68,7 +68,9 @@ namespace AMDiS {
typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap; typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;
typedef std::map<int, DofMapping> PeriodicDofMap; /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic
/// dof mapping.
typedef std::map<BoundaryType, DofMapping> PeriodicDofMap;
typedef std::vector<MeshStructure> MeshCodeVec; typedef std::vector<MeshStructure> MeshCodeVec;
...@@ -118,22 +120,26 @@ namespace AMDiS { ...@@ -118,22 +120,26 @@ namespace AMDiS {
return nRankDofs; return nRankDofs;
} }
/// Returns \ref nOverallDofs, the global number of DOFs.
inline int getNumberOverallDofs() inline int getNumberOverallDofs()
{ {
return nOverallDofs; return nOverallDofs;
} }
/// Maps a local dof to its global index.
inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof) inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
{ {
return mapLocalGlobalDofs[dof]; return mapLocalGlobalDofs[dof];
} }
/// Maps a local dof to its local index.
inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof) inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
{ {
return mapLocalDofIndex[dof]; return mapLocalDofIndex[dof];
} }
inline int getPeriodicMapping(int type, int globalDofIndex) /// Returns for a global dof index its periodic mapping for a given boundary type.
inline int getPeriodicMapping(BoundaryType type, int globalDofIndex)
{ {
TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1) TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
("Should not happen!\n"); ("Should not happen!\n");
...@@ -141,21 +147,28 @@ namespace AMDiS { ...@@ -141,21 +147,28 @@ namespace AMDiS {
return periodicDof[type][globalDofIndex]; return periodicDof[type][globalDofIndex];
} }
inline std::set<int>& getPerDofAssociations(int globalDofIndex) /// For a given global DOF index, this function returns the set of periodic
/// associations, i.e., the boundary types the DOF is associated to, for this DOF.
inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
{ {
return perDofAssociations[globalDofIndex]; return periodicDofAssociations[globalDofIndex];
} }
/// Returns true, if the DOF (global index) is a periodic DOF.
inline bool isPeriodicDof(int globalDofIndex) inline bool isPeriodicDof(int globalDofIndex)
{ {
return (perDofAssociations.count(globalDofIndex) > 0); return (periodicDofAssociations.count(globalDofIndex) > 0);
} }
inline bool isPeriodicDof(int globalDofIndex, int type) /// Returns true, if the DOF (global index) is a periodic DOF for the given
/// boundary type.
inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
{ {
return (periodicDof[type].count(globalDofIndex) > 0); return (periodicDof[type].count(globalDofIndex) > 0);
} }
/// Return true, if the given DOF is owned by the rank. If false, the DOF is in
/// rank's partition, but is owned by some other rank.
inline bool getIsRankDof(DegreeOfFreedom dof) inline bool getIsRankDof(DegreeOfFreedom dof)
{ {
return isRankDof[dof]; return isRankDof[dof];
...@@ -512,13 +525,20 @@ namespace AMDiS { ...@@ -512,13 +525,20 @@ namespace AMDiS {
DofToBool vertexDof; DofToBool vertexDof;
/** \brief /** \brief
* If periodic boundaries are used, this map stores to each dof in rank's * If periodic boundaries are used, this map stores, for each periodic boundary
* partition, that is on periodic boundaries, the corresponding periodic dofs. * type, for all DOFs in rank's partition (that are on periodic boundaries), the
* The mapping is defined by using global dof indices. * corresponding mapped periodic DOFs. The mapping is defined by using global
* dof indices.
*/ */
PeriodicDofMap periodicDof; PeriodicDofMap periodicDof;
std::map<int, std::set<int> > perDofAssociations; /** \brief
* If periodic boundaries are used, this map stores to each periodic DOF in rank's
* partition the set of periodic boundaries the DOF is associated to. In 2D, most
* DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
* being periodic, the for corners are associated by two different boundaries.
*/
std::map<int, std::set<BoundaryType> > periodicDofAssociations;
/// Is the index of the first row of the linear system, which is owned by the rank. /// Is the index of the first row of the linear system, which is owned by the rank.
int rstart; int rstart;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment