From c52b43fe338222212eb96261282a67c3641220a5 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Fri, 9 Jul 2010 09:34:17 +0000
Subject: [PATCH] Some comments in parallel code, changed code of parallel
serialization-deserialization.
---
AMDiS/src/parallel/GlobalMatrixSolver.cc | 5 ++-
AMDiS/src/parallel/GlobalMatrixSolver.h | 2 +
AMDiS/src/parallel/MeshDistributor.cc | 57 ++++++++++++++----------
AMDiS/src/parallel/MeshDistributor.h | 42 ++++++++++++-----
4 files changed, 70 insertions(+), 36 deletions(-)
diff --git a/AMDiS/src/parallel/GlobalMatrixSolver.cc b/AMDiS/src/parallel/GlobalMatrixSolver.cc
index 7d98b16c..05bdb6e6 100644
--- a/AMDiS/src/parallel/GlobalMatrixSolver.cc
+++ b/AMDiS/src/parallel/GlobalMatrixSolver.cc
@@ -416,7 +416,9 @@ namespace AMDiS {
MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
0, d_nnz, 0, o_nnz, &petscMatrix);
+#if (DEBUG != 0)
INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock()));
+#endif
#if (DEBUG != 0)
int a, b;
@@ -435,8 +437,9 @@ namespace AMDiS {
if ((*mat)[i][j])
setDofMatrix((*mat)[i][j], nComponents, i, j);
-
+#if (DEBUG != 0)
INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock()));
+#endif
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
diff --git a/AMDiS/src/parallel/GlobalMatrixSolver.h b/AMDiS/src/parallel/GlobalMatrixSolver.h
index d6ca31e4..d133e695 100644
--- a/AMDiS/src/parallel/GlobalMatrixSolver.h
+++ b/AMDiS/src/parallel/GlobalMatrixSolver.h
@@ -106,6 +106,8 @@ namespace AMDiS {
};
+ typedef GlobalMatrixSolver ParallelProblemStat;
+
} //namespace AMDiS
#endif
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 70f76c7a..2a22efb2 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -238,18 +238,17 @@ namespace AMDiS {
GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d",
&writeSerialization);
if (writeSerialization && !writeSerializationFile) {
- std::string f = "";
- GET_PARAMETER(0, probVec->getName() + "->output->serialization filename", &f);
- path myPath(f);
- std::string meshFilename =
- myPath.parent_path().directory_string() + "/meshDistributor.ser";
-
+ std::string filename = "";
+ GET_PARAMETER(0, name + "->output->serialization filename", &filename);
+
+ TEST_EXIT(filename != "")
+ ("No filename defined for parallel serialization file!\n");
int tsModulo = -1;
GET_PARAMETER(0, probVec->getName() + "->output->write every i-th timestep",
"%d", &tsModulo);
- probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, meshFilename, tsModulo));
+ probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, filename, tsModulo));
writeSerializationFile = true;
}
@@ -262,20 +261,30 @@ namespace AMDiS {
filename += ".p" + lexical_cast<std::string>(mpiRank);
MSG("Start deserialization with %s\n", filename.c_str());
std::ifstream in(filename.c_str());
+
+ TEST_EXIT(!in.fail())("Could not open deserialization file: %s\n",
+ filename.c_str());
+
probVec->deserialize(in);
in.close();
MSG("Deserialization from file: %s\n", filename.c_str());
if (!deserialized) {
- GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename);
- path myPath(filename);
- std::string meshFilename =
- myPath.parent_path().directory_string() + "/meshDistributor.ser.p" +
- lexical_cast<std::string>(mpiRank);
- std::ifstream in(meshFilename.c_str());
+ filename = "";
+ GET_PARAMETER(0, name + "->input->serialization filename", &filename);
+
+ TEST_EXIT(filename != "")
+ ("No filename defined for parallel deserialization file!\n");
+
+ std::string rankFilename = filename + ".p" + lexical_cast<std::string>(mpiRank);
+ std::ifstream in(rankFilename.c_str());
+
+ TEST_EXIT(!in.fail())("Could not open parallel deserialization file: %s\n",
+ filename.c_str());
+
deserialize(in);
in.close();
- MSG("Deserializtion of mesh distributor from file: %s\n", meshFilename.c_str());
+ MSG("Deserializtion of mesh distributor from file: %s\n", rankFilename.c_str());
deserialized = true;
}
}
@@ -1961,7 +1970,7 @@ namespace AMDiS {
// Clear all periodic dof mappings calculated before. We do it from scratch.
periodicDof.clear();
- perDofAssociations.clear();
+ periodicDofAssociations.clear();
StdMpi<std::vector<int> > stdMpi(mpiComm, false);
@@ -2037,7 +2046,7 @@ namespace AMDiS {
BoundaryType type = types[i];
periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
- perDofAssociations[globalDofIndex].insert(type);
+ periodicDofAssociations[globalDofIndex].insert(type);
dofFromRank[globalDofIndex].insert(it->first);
}
}
@@ -2054,11 +2063,11 @@ namespace AMDiS {
for (std::map<DegreeOfFreedom, std::set<int> >::iterator it = dofFromRank.begin();
it != dofFromRank.end(); ++it) {
if (it->second.size() == 2) {
- TEST_EXIT_DBG(perDofAssociations[it->first].size() == 2)
+ TEST_EXIT_DBG(periodicDofAssociations[it->first].size() == 2)
("Missing periodic dof!\n");
- int type0 = *(perDofAssociations[it->first].begin());
- int type1 = *(++(perDofAssociations[it->first].begin()));
+ int type0 = *(periodicDofAssociations[it->first].begin());
+ int type1 = *(++(periodicDofAssociations[it->first].begin()));
int *sendbuf = new int[2];
sendbuf[0] = periodicDof[type0][it->first];
@@ -2098,7 +2107,7 @@ namespace AMDiS {
int type = 3;
periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
- perDofAssociations[globalDofIndex].insert(type);
+ periodicDofAssociations[globalDofIndex].insert(type);
}
i++;
@@ -2112,8 +2121,8 @@ namespace AMDiS {
delete [] recvBuffers[i];
#if (DEBUG != 0)
- for (std::map<int, std::set<int> >::iterator it = perDofAssociations.begin();
- it != perDofAssociations.end(); ++it) {
+ for (std::map<int, std::set<BoundaryType> >::iterator it = periodicDofAssociations.begin();
+ it != periodicDofAssociations.end(); ++it) {
int nAssoc = it->second.size();
TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3)
("Should not happen! DOF %d has %d periodic associations!\n",
@@ -2146,7 +2155,7 @@ namespace AMDiS {
serialize(out, vertexDof);
serialize(out, periodicDof);
- serialize(out, perDofAssociations);
+ serialize(out, periodicDofAssociations);
SerUtil::serialize(out, rstart);
SerUtil::serialize(out, macroElementStructureConsisten);
@@ -2196,7 +2205,7 @@ namespace AMDiS {
deserialize(in, vertexDof, dofMap);
deserialize(in, periodicDof);
- deserialize(in, perDofAssociations);
+ deserialize(in, periodicDofAssociations);
SerUtil::deserialize(in, rstart);
SerUtil::deserialize(in, macroElementStructureConsisten);
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index bf16242e..e171f73f 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -68,7 +68,9 @@ namespace AMDiS {
typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;
- typedef std::map<int, DofMapping> PeriodicDofMap;
+ /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic
+ /// dof mapping.
+ typedef std::map<BoundaryType, DofMapping> PeriodicDofMap;
typedef std::vector<MeshStructure> MeshCodeVec;
@@ -118,22 +120,26 @@ namespace AMDiS {
return nRankDofs;
}
+ /// Returns \ref nOverallDofs, the global number of DOFs.
inline int getNumberOverallDofs()
{
return nOverallDofs;
}
+ /// Maps a local dof to its global index.
inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
{
return mapLocalGlobalDofs[dof];
}
+ /// Maps a local dof to its local index.
inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
{
return mapLocalDofIndex[dof];
}
- inline int getPeriodicMapping(int type, int globalDofIndex)
+ /// Returns for a global dof index its periodic mapping for a given boundary type.
+ inline int getPeriodicMapping(BoundaryType type, int globalDofIndex)
{
TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
("Should not happen!\n");
@@ -141,21 +147,28 @@ namespace AMDiS {
return periodicDof[type][globalDofIndex];
}
- inline std::set<int>& getPerDofAssociations(int globalDofIndex)
+ /// For a given global DOF index, this function returns the set of periodic
+ /// associations, i.e., the boundary types the DOF is associated to, for this DOF.
+ inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
{
- return perDofAssociations[globalDofIndex];
+ return periodicDofAssociations[globalDofIndex];
}
+ /// Returns true, if the DOF (global index) is a periodic DOF.
inline bool isPeriodicDof(int globalDofIndex)
{
- return (perDofAssociations.count(globalDofIndex) > 0);
+ return (periodicDofAssociations.count(globalDofIndex) > 0);
}
- inline bool isPeriodicDof(int globalDofIndex, int type)
+ /// Returns true, if the DOF (global index) is a periodic DOF for the given
+ /// boundary type.
+ inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
{
return (periodicDof[type].count(globalDofIndex) > 0);
}
+ /// Return true, if the given DOF is owned by the rank. If false, the DOF is in
+ /// rank's partition, but is owned by some other rank.
inline bool getIsRankDof(DegreeOfFreedom dof)
{
return isRankDof[dof];
@@ -512,13 +525,20 @@ namespace AMDiS {
DofToBool vertexDof;
/** \brief
- * If periodic boundaries are used, this map stores to each dof in rank's
- * partition, that is on periodic boundaries, the corresponding periodic dofs.
- * The mapping is defined by using global dof indices.
+ * If periodic boundaries are used, this map stores, for each periodic boundary
+ * type, for all DOFs in rank's partition (that are on periodic boundaries), the
+ * corresponding mapped periodic DOFs. The mapping is defined by using global
+ * dof indices.
*/
PeriodicDofMap periodicDof;
-
- std::map<int, std::set<int> > perDofAssociations;
+
+ /** \brief
+ * If periodic boundaries are used, this map stores to each periodic DOF in rank's
+ * partition the set of periodic boundaries the DOF is associated to. In 2D, most
+ * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
+ * being periodic, the for corners are associated by two different boundaries.
+ */
+ std::map<int, std::set<BoundaryType> > periodicDofAssociations;
/// Is the index of the first row of the linear system, which is owned by the rank.
int rstart;
--
GitLab