Some comments in parallel code, changed code of parallel serialization-deserialization.

AMDiS/src/parallel/GlobalMatrixSolver.cc

@@ -416,7 +416,9 @@ namespace AMDiS {
     MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
   		    0, d_nnz, 0, o_nnz, &petscMatrix);
     
+#if (DEBUG != 0)
     INFO(info, 8)("Fill petsc matrix 1 needed %.5f seconds\n", TIME_USED(first, clock()));
+#endif
 
 #if (DEBUG != 0)
     int a, b;
@@ -435,8 +437,9 @@ namespace AMDiS {
 	if ((*mat)[i][j])
 	  setDofMatrix((*mat)[i][j], nComponents, i, j);	
 	
-
+#if (DEBUG != 0)
     INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", TIME_USED(first, clock()));
+#endif
 
     MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
     MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);

AMDiS/src/parallel/GlobalMatrixSolver.h

@@ -106,6 +106,8 @@ namespace AMDiS {
   };
 
 
+  typedef GlobalMatrixSolver ParallelProblemStat;
+
 } //namespace AMDiS
 
 #endif

AMDiS/src/parallel/MeshDistributor.cc

@@ -238,18 +238,17 @@ namespace AMDiS {
     GET_PARAMETER(0, probVec->getName() + "->output->write serialization", "%d", 
 		  &writeSerialization);
     if (writeSerialization && !writeSerializationFile) {
-      std::string f = "";
-      GET_PARAMETER(0, probVec->getName() + "->output->serialization filename", &f);
-      path myPath(f);
-      std::string meshFilename = 
-	myPath.parent_path().directory_string() + "/meshDistributor.ser";
-
+      std::string filename = "";
+      GET_PARAMETER(0, name + "->output->serialization filename", &filename);
+      
+      TEST_EXIT(filename != "")
+	("No filename defined for parallel serialization file!\n");
 
       int tsModulo = -1;
       GET_PARAMETER(0, probVec->getName() + "->output->write every i-th timestep", 
 		    "%d", &tsModulo);
       
-      probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, meshFilename, tsModulo));
+      probVec->getFileWriterList().push_back(new Serializer<MeshDistributor>(this, filename, tsModulo));
       writeSerializationFile = true;
     }    
 
@@ -262,20 +261,30 @@ namespace AMDiS {
       filename += ".p" + lexical_cast<std::string>(mpiRank);
       MSG("Start deserialization with %s\n", filename.c_str());
       std::ifstream in(filename.c_str());
+
+      TEST_EXIT(!in.fail())("Could not open deserialization file: %s\n",
+			    filename.c_str());
+
       probVec->deserialize(in);
       in.close();
       MSG("Deserialization from file: %s\n", filename.c_str());
 
       if (!deserialized) {
-	GET_PARAMETER(0, probVec->getName() + "->input->serialization filename", &filename);
-	path myPath(filename);
-	std::string meshFilename = 
-	  myPath.parent_path().directory_string() + "/meshDistributor.ser.p" + 
-	  lexical_cast<std::string>(mpiRank);
-	std::ifstream in(meshFilename.c_str());
+	filename = "";
+	GET_PARAMETER(0, name + "->input->serialization filename", &filename);
+
+	TEST_EXIT(filename != "")
+	  ("No filename defined for parallel deserialization file!\n");
+
+	std::string rankFilename = filename + ".p" + lexical_cast<std::string>(mpiRank);
+	std::ifstream in(rankFilename.c_str());
+
+	TEST_EXIT(!in.fail())("Could not open parallel deserialization file: %s\n",
+			      filename.c_str());
+
 	deserialize(in);
 	in.close();
-	MSG("Deserializtion of mesh distributor from file: %s\n", meshFilename.c_str());
+	MSG("Deserializtion of mesh distributor from file: %s\n", rankFilename.c_str());
 	deserialized = true;
       }
     }
@@ -1961,7 +1970,7 @@ namespace AMDiS {
 
     // Clear all periodic dof mappings calculated before. We do it from scratch.
     periodicDof.clear();
-    perDofAssociations.clear();
+    periodicDofAssociations.clear();
 
     StdMpi<std::vector<int> > stdMpi(mpiComm, false);
 
@@ -2037,7 +2046,7 @@ namespace AMDiS {
 	BoundaryType type = types[i];
 
 	periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
-	perDofAssociations[globalDofIndex].insert(type);
+	periodicDofAssociations[globalDofIndex].insert(type);
 	dofFromRank[globalDofIndex].insert(it->first);
        }
     }
@@ -2054,11 +2063,11 @@ namespace AMDiS {
     for (std::map<DegreeOfFreedom, std::set<int> >::iterator it = dofFromRank.begin();
 	 it != dofFromRank.end(); ++it) {
       if (it->second.size() == 2) {
-	TEST_EXIT_DBG(perDofAssociations[it->first].size() == 2)
+	TEST_EXIT_DBG(periodicDofAssociations[it->first].size() == 2)
 	  ("Missing periodic dof!\n");
 
-	int type0 = *(perDofAssociations[it->first].begin());
-	int type1 = *(++(perDofAssociations[it->first].begin()));
+	int type0 = *(periodicDofAssociations[it->first].begin());
+	int type1 = *(++(periodicDofAssociations[it->first].begin()));
 
 	int *sendbuf = new int[2];
 	sendbuf[0] = periodicDof[type0][it->first];
@@ -2098,7 +2107,7 @@ namespace AMDiS {
 	      int type = 3;
 	
 	      periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
-	      perDofAssociations[globalDofIndex].insert(type);
+	      periodicDofAssociations[globalDofIndex].insert(type);
 	    }
 
 	i++;
@@ -2112,8 +2121,8 @@ namespace AMDiS {
       delete [] recvBuffers[i];
 
 #if (DEBUG != 0)
-    for (std::map<int, std::set<int> >::iterator it = perDofAssociations.begin();
-	 it != perDofAssociations.end(); ++it) {
+    for (std::map<int, std::set<BoundaryType> >::iterator it = periodicDofAssociations.begin();
+	 it != periodicDofAssociations.end(); ++it) {
       int nAssoc = it->second.size();
       TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3)
  	("Should not happen! DOF %d has %d periodic associations!\n", 
@@ -2146,7 +2155,7 @@ namespace AMDiS {
 
     serialize(out, vertexDof);
     serialize(out, periodicDof);
-    serialize(out, perDofAssociations);
+    serialize(out, periodicDofAssociations);
 
     SerUtil::serialize(out, rstart);
     SerUtil::serialize(out, macroElementStructureConsisten);
@@ -2196,7 +2205,7 @@ namespace AMDiS {
 
     deserialize(in, vertexDof, dofMap);
     deserialize(in, periodicDof);
-    deserialize(in, perDofAssociations);
+    deserialize(in, periodicDofAssociations);
 
     SerUtil::deserialize(in, rstart);
     SerUtil::deserialize(in, macroElementStructureConsisten);

AMDiS/src/parallel/MeshDistributor.h

@@ -68,7 +68,9 @@ namespace AMDiS {
 
     typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;
 
-    typedef std::map<int, DofMapping> PeriodicDofMap;
+    /// Mapps a boundar type, i.e., a boundary identifier index, to a periodic 
+    /// dof mapping.
+    typedef std::map<BoundaryType, DofMapping> PeriodicDofMap;
     
     typedef std::vector<MeshStructure> MeshCodeVec;
 
@@ -118,22 +120,26 @@ namespace AMDiS {
       return nRankDofs;
     }
 
+    /// Returns \ref nOverallDofs, the global number of DOFs.
     inline int getNumberOverallDofs()
     {
       return nOverallDofs;
     }
 
+    /// Maps a local dof to its global index.
     inline DegreeOfFreedom mapLocalToGlobal(DegreeOfFreedom dof)
     {
       return mapLocalGlobalDofs[dof];
     }
 
+    /// Maps a local dof to its local index.
     inline DegreeOfFreedom mapLocalToDofIndex(DegreeOfFreedom dof)
     {
       return mapLocalDofIndex[dof];
     }
 
-    inline int getPeriodicMapping(int type, int globalDofIndex)
+    /// Returns for a global dof index its periodic mapping for a given boundary type.
+    inline int getPeriodicMapping(BoundaryType type, int globalDofIndex)
     {
       TEST_EXIT_DBG(periodicDof[type].count(globalDofIndex) == 1)
 	("Should not happen!\n");
@@ -141,21 +147,28 @@ namespace AMDiS {
       return periodicDof[type][globalDofIndex];
     }
 
-    inline std::set<int>& getPerDofAssociations(int globalDofIndex)
+    /// For a given global DOF index, this function returns the set of periodic
+    /// associations, i.e., the boundary types the DOF is associated to, for this DOF.
+    inline std::set<BoundaryType>& getPerDofAssociations(int globalDofIndex)
     {      
-      return perDofAssociations[globalDofIndex];
+      return periodicDofAssociations[globalDofIndex];
     }
 
+    /// Returns true, if the DOF (global index) is a periodic DOF.
     inline bool isPeriodicDof(int globalDofIndex)
     {
-      return (perDofAssociations.count(globalDofIndex) > 0);
+      return (periodicDofAssociations.count(globalDofIndex) > 0);
     }
 
-    inline bool isPeriodicDof(int globalDofIndex, int type)
+    /// Returns true, if the DOF (global index) is a periodic DOF for the given
+    /// boundary type.
+    inline bool isPeriodicDof(int globalDofIndex, BoundaryType type)
     {
       return (periodicDof[type].count(globalDofIndex) > 0);
     }
 
+    /// Return true, if the given DOF is owned by the rank. If false, the DOF is in
+    /// rank's partition, but is owned by some other rank.
     inline bool getIsRankDof(DegreeOfFreedom dof)
     {
       return isRankDof[dof];
@@ -512,13 +525,20 @@ namespace AMDiS {
     DofToBool vertexDof;
 
     /** \brief
-     * If periodic boundaries are used, this map stores to each dof in rank's 
-     * partition, that is on periodic boundaries, the corresponding periodic dofs.
-     * The mapping is defined by using global dof indices.
+     * If periodic boundaries are used, this map stores, for each periodic boundary
+     * type, for all DOFs in rank's partition (that are on periodic boundaries), the 
+     * corresponding mapped periodic DOFs. The mapping is defined by using global 
+     * dof indices.
      */
     PeriodicDofMap periodicDof;
-
-    std::map<int, std::set<int> > perDofAssociations;
+    
+    /** \brief
+     * If periodic boundaries are used, this map stores to each periodic DOF in rank's
+     * partition the set of periodic boundaries the DOF is associated to. In 2D, most
+     * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
+     * being periodic, the for corners are associated by two different boundaries.     
+     */
+    std::map<int, std::set<BoundaryType> > periodicDofAssociations;
 
     /// Is the index of the first row of the linear system, which is owned by the rank.
     int rstart;