And more changes on the DofCommunicator.

AMDiS/src/parallel/DofComm.cc

@@ -18,26 +18,15 @@ namespace AMDiS {
 
   using namespace std;
 
-  int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace)
+  void DofComm::create(InteriorBoundary &boundary)
   {
-    FUNCNAME("DofComm::getNumberDofs()");
-
-    TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
-
-    DofContainerSet dofs;
-
-    for (DataIter rankIt = data[level].begin(); 
-	 rankIt != data[level].end(); ++rankIt)
-      for (FeMapIter feIt = rankIt->second.begin();
-	   feIt != rankIt->second.end(); ++feIt)
-	if (feIt->first == feSpace)
-	  dofs.insert(feIt->second.begin(), feIt->second.end());
-
-    return static_cast<int>(dofs.size());
+    createContainer(boundary.getOwn(), sendDofs);
+    createContainer(boundary.getOther(), recvDofs);
   }
 
 
-  void DofComm::create(RankToBoundMap &boundary)
+  void DofComm::createContainer(RankToBoundMap &boundary,
+				LevelDataType &data)
   {
     // === Fill data. ===
 
@@ -72,11 +61,32 @@ namespace AMDiS {
   }
 
 
+  int DofComm::getNumberDofs(LevelDataType &data, 
+			     int level, 
+			     const FiniteElemSpace *feSpace)
+  {
+    FUNCNAME("DofComm::getNumberDofs()");
+
+    TEST_EXIT_DBG(level < data.size())("Should not happen!\n");
+
+    DofContainerSet dofs;
+
+    for (DataIter rankIt = data[level].begin(); 
+	 rankIt != data[level].end(); ++rankIt)
+      for (FeMapIter feIt = rankIt->second.begin();
+	   feIt != rankIt->second.end(); ++feIt)
+	if (feIt->first == feSpace)
+	  dofs.insert(feIt->second.begin(), feIt->second.end());
+
+    return static_cast<int>(dofs.size());
+  }
+  
+
   bool DofComm::Iterator::setNextFeMap()
   {
     FUNCNAME("DofComm::Iterator::setNextFeMap()");
 
-    if (dataIter != dofComm.data[traverseLevel].end()) {
+    if (dataIter != data[traverseLevel].end()) {
       TEST_EXIT_DBG(dataIter->second.size())("Should not happen!\n");
 
       feMapIter = dataIter->second.begin();

AMDiS/src/parallel/DofComm.h

@@ -36,7 +36,9 @@ namespace AMDiS {
   {
   public:
     DofComm() 
-      : data(1)
+      : recvDofs(1),
+	sendDofs(1),
+	periodicDofs(0)
     {}
     
     typedef map<const FiniteElemSpace*, DofContainer> FeMapType;
@@ -46,33 +48,76 @@ namespace AMDiS {
     // meshLevel: map[rank -> map[feSpace -> DofContainer]]
     typedef vector<DataType> LevelDataType;
 
-    inline DofContainer& getDofContainer(int rank, 
-					 const FiniteElemSpace *feSpace, 
-					 int level = 0)
-    {
-      return data[level][rank][feSpace];
-    }
-
     void init(int n, vector<const FiniteElemSpace*> &fe)
     {
+      FUNCNAME("DofComm::init()");
+
+      TEST_EXIT_DBG(n >= 1)("Should not happen!\n");
+
       nLevel = n;
       feSpaces = fe;
 
-      data.clear();
-      data.resize(nLevel);
+      sendDofs.clear();
+      recvDofs.clear();
+      periodicDofs.clear();
+
+      sendDofs.resize(nLevel);
+      recvDofs.resize(nLevel);
+      periodicDofs.resize(nLevel);
     }
 
-    DataType& getData(int level = 0)
+    void create(InteriorBoundary &boundary);
+
+    LevelDataType& getSendDofs()
     {
-      return data[level];
+      return sendDofs;
     }
 
-    int getNumberDofs(int level, const FiniteElemSpace *feSpace);
+    LevelDataType& getRecvDofs()
+    {
+      return recvDofs;
+    }
 
-    void create(RankToBoundMap &boundary);
+    LevelDataType& getPeriodicDofs()
+    {
+      return periodicDofs;
+    }
+
+    // Writes all data of this object to an output stream.
+    void serialize(ostream &out)
+    {
+      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
+    }
+
+    // Reads the object data from an input stream.
+    void deserialize(istream &in, 
+		     map<const FiniteElemSpace*, map<int, const DegreeOfFreedom*> > dofIndexMap)
+    {
+      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
+    }
+
+    int getNumberDofs(LevelDataType &data, 
+		      int level, 
+		      const FiniteElemSpace *feSpace);
+
+  protected:
+    void createContainer(RankToBoundMap &boundary, LevelDataType &data);
 
   protected:
-    LevelDataType data;
+    /// This map contains for each rank the list of DOFs the current rank must 
+    /// end to exchange solution DOFs at the interior boundaries.
+    LevelDataType sendDofs;
+
+    /// This map contains on each rank the list of DOFs from which the current 
+    /// rank will receive DOF values (i.e., this are all DOFs at an interior 
+    /// boundary). The DOF indices are given in rank's local numbering.
+    LevelDataType recvDofs;
+
+    /// This map contains on each rank a list of DOFs along the interior bound-
+    /// aries to communicate with other ranks. The DOF indices are given in rank's
+    /// local numbering. Periodic boundaries within one subdomain are not 
+    /// considered here. 
+    LevelDataType periodicDofs;
 
     int nLevel;
 
@@ -84,9 +129,9 @@ namespace AMDiS {
     class Iterator
     {
     public:
-      Iterator(DofComm &dc,
+      Iterator(LevelDataType &d,
 	       const FiniteElemSpace *fe = NULL)
-	: dofComm(dc),
+	: data(d),
 	  dofCounter(-1),
 	  traverseFeSpace(fe),
 	  traverseLevel(0)
@@ -94,10 +139,10 @@ namespace AMDiS {
 	goFirst();
       }
 
-      Iterator(DofComm &dc,
+      Iterator(LevelDataType &d,
 	       int level,
 	       const FiniteElemSpace *fe = NULL)
-	: dofComm(dc),
+	: data(d),
 	  dofCounter(-1),
 	  traverseFeSpace(fe),
 	  traverseLevel(level)
@@ -107,7 +152,7 @@ namespace AMDiS {
 
       inline bool end()
       {
-	return (dataIter == dofComm.data[traverseLevel].end());
+	return (dataIter == data[traverseLevel].end());
       }
       
       inline void nextRank()
@@ -200,7 +245,7 @@ namespace AMDiS {
     protected:
       void goFirst()
       {
-	dataIter = dofComm.data[traverseLevel].begin();
+	dataIter = data[traverseLevel].begin();
 
 	while (setNextFeMap() == false)
 	  ++dataIter;
@@ -209,7 +254,7 @@ namespace AMDiS {
       bool setNextFeMap();
 
     protected:
-      DofComm &dofComm;
+      LevelDataType &data;
       
       DofComm::DataIter dataIter;
       

AMDiS/src/parallel/MeshDistributor.cc

@@ -511,7 +511,8 @@ namespace AMDiS {
 
     StdMpi<vector<double> > stdMpi(mpiComm);
 
-    for (DofComm::Iterator it(sendDofs); !it.end(); it.nextRank()) {
+    for (DofComm::Iterator it(dofComm.getSendDofs()); 
+	 !it.end(); it.nextRank()) {
       vector<double> dofs;
 
       for (int i = 0; i < vec.getSize(); i++) {
@@ -524,12 +525,12 @@ namespace AMDiS {
       stdMpi.send(it.getRank(), dofs);
     }
 	   
-    for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank())
       stdMpi.recv(it.getRank());
 
     stdMpi.startCommunication();
 
-    for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank()) {
+    for (DofComm::Iterator it(dofComm.getRecvDofs()); !it.end(); it.nextRank()) {
       int counter = 0;
 
       for (int i = 0; i < vec.getSize(); i++) {
@@ -659,9 +660,11 @@ namespace AMDiS {
     FUNCNAME("MeshDistributor::getAllBoundaryDofs()");
 
     DofContainerSet dofSet;
-    for (DofComm::Iterator it(sendDofs, level, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(dofComm.getSendDofs(), level, feSpace); 
+	 !it.end(); it.nextRank())
       dofSet.insert(it.getDofs().begin(), it.getDofs().end());
-    for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace); 
+	 !it.end(); it.nextRank())
       dofSet.insert(it.getDofs().begin(), it.getDofs().end());
 
     dofs.clear();
@@ -1533,14 +1536,8 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::createBoundaryDofs()");
 
-    int nLevels = levelData.getLevelNumber();
-    TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
-
-    sendDofs.init(nLevels, feSpaces);
-    sendDofs.create(intBoundary.getOwn());
-
-    recvDofs.init(nLevels, feSpaces);
-    recvDofs.create(intBoundary.getOther());
+    dofComm.init(levelData.getLevelNumber(), feSpaces);
+    dofComm.create(intBoundary);
 
     createBoundaryDofInfo();
   }
@@ -1625,7 +1622,7 @@ namespace AMDiS {
     TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
 
     dofMap.init(levelData, feSpaces, feSpaces, true, true);
-    dofMap.setDofComm(sendDofs, recvDofs);
+    dofMap.setDofComm(dofComm);
     dofMap.clear();
 
     createBoundaryDofs();
@@ -1684,7 +1681,8 @@ namespace AMDiS {
     int nLevels = levelData.getLevelNumber();
     for (int level = 0; level < nLevels; level++) {
       DofContainerSet nonRankDofs;
-      for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
+      for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace); 
+	   !it.end(); it.nextRank())
 	for (; !it.endDofIter(); it.nextDof())
 	  nonRankDofs.insert(it.getDof());
       
@@ -1692,7 +1690,8 @@ namespace AMDiS {
 	if (nonRankDofs.count(rankDofs[i]) == 0)
 	  dofMap[feSpace].insertRankDof(level, *(rankDofs[i]));
       
-      for (DofComm::Iterator it(recvDofs, level, feSpace); !it.end(); it.nextRank())
+      for (DofComm::Iterator it(dofComm.getRecvDofs(), level, feSpace); 
+	   !it.end(); it.nextRank())
 	for (; !it.endDofIter(); it.nextDof())
 	  dofMap[feSpace].insertNonRankDof(level, it.getDofIndex());
     }
@@ -1702,9 +1701,7 @@ namespace AMDiS {
     lastMeshChangeIndex = mesh->getChangeIndex();
 
 #if (DEBUG != 0)
-    ParallelDebug::testDofContainerCommunication(*this, 
-						 sendDofs.getData(), 
-						 recvDofs.getData());
+    ParallelDebug::testDofContainerCommunication(*this);
 #endif
   }
 
@@ -1714,13 +1711,9 @@ namespace AMDiS {
     FUNCNAME("MeshDistributor::createPeriodicMap()");
 
     // Clear all periodic DOF mappings calculated before. We do it from scratch.
-    periodicDofs.init(levelData.getLevelNumber(), feSpaces);
     periodicMap.clear();
 
-    // If there are no periodic boundaries, return. Note that periodicDofs and
-    // periodicMap must be still cleared before: if we do repartitioning and
-    // there were periodic boundaries in subdomain before and after repartitioning
-    // there are no more periodic boundaries.
+    // If there are no periodic boundaries, return. 
     if (!intBoundary.hasPeriodic())
       return;
 
@@ -1736,6 +1729,8 @@ namespace AMDiS {
   {
     FUNCNAME("MeshDistributor::createPeriodicMap()");
 
+    DofComm::LevelDataType &periodicDofs = dofComm.getPeriodicDofs();
+
     StdMpi<vector<int> > stdMpi(mpiComm, false);
 
     // === Each rank traverse its periodic boundaries and sends the DOF      ===
@@ -1781,7 +1776,7 @@ namespace AMDiS {
 	// Here we have a periodic boundary between two ranks.
 
 	// Create DOF indices on the boundary. 
-	DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace);
+	DofContainer& dofs = periodicDofs[0][it->first][feSpace];
 	for (vector<AtomicBoundary>::iterator boundIt = it->second.begin();
 	     boundIt != it->second.end(); ++boundIt) {
 
@@ -1814,7 +1809,7 @@ namespace AMDiS {
 
     for (RankToBoundMap::iterator it = intBoundary.getPeriodic().begin();
 	 it != intBoundary.getPeriodic().end(); ++it) {
-      DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace);
+      DofContainer& dofs = periodicDofs[0][it->first][feSpace];
       vector<int>& types = rankToDofType[it->first];
 
       TEST_EXIT_DBG(dofs.size() == types.size())("Should not happen!\n");
@@ -1934,8 +1929,7 @@ namespace AMDiS {
 
     intBoundary.serialize(out);
 
-    serialize(out, sendDofs.getData());
-    serialize(out, recvDofs.getData());
+    dofComm.serialize(out);
 
     // === Serialieze FE space dependent data ===
 
@@ -1993,8 +1987,7 @@ namespace AMDiS {
    
     intBoundary.deserialize(in, elIndexMap);
 
-    deserialize(in, sendDofs.getData(), dofIndexMap);
-    deserialize(in, recvDofs.getData(), dofIndexMap);
+    dofComm.deserialize(in, dofIndexMap);
 
     // === Deerialieze FE space dependent data ===
     

AMDiS/src/parallel/MeshDistributor.h

@@ -157,19 +157,9 @@ namespace AMDiS {
       return periodicMap;
     }
 
-    DofComm& getSendDofs()
+    DofComm& getDofComm()
     {
-      return sendDofs;
-    }
-
-    DofComm& getRecvDofs()
-    {
-      return recvDofs;
-    }
-
-    DofComm& getPeriodicDofs()
-    {
-      return periodicDofs;
+      return dofComm;
     }
 
     inline long getLastMeshChangeIndex()
@@ -220,7 +210,8 @@ namespace AMDiS {
 
       const FiniteElemSpace *fe = vec.getFeSpace();
 
-      for (DofComm::Iterator it(sendDofs, fe); !it.end(); it.nextRank()) {
+      for (DofComm::Iterator it(dofComm.getSendDofs(), fe); 
+	   !it.end(); it.nextRank()) {
 	vector<T> dofs;
 	dofs.reserve(it.getDofs().size());
 	
@@ -230,12 +221,14 @@ namespace AMDiS {
 	stdMpi.send(it.getRank(), dofs);
       }
 	     
-      for (DofComm::Iterator it(recvDofs); !it.end(); it.nextRank())
+      for (DofComm::Iterator it(dofComm.getRecvDofs()); 
+	   !it.end(); it.nextRank())
         stdMpi.recv(it.getRank());
 	     
       stdMpi.startCommunication();
 
-      for (DofComm::Iterator it(recvDofs, fe); !it.end(); it.nextRank())
+      for (DofComm::Iterator it(dofComm.getRecvDofs(), fe); 
+	   !it.end(); it.nextRank())
 	for (; !it.endDofIter(); it.nextDof())
 	  vec[it.getDofIndex()] = 
 	     stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
@@ -477,20 +470,7 @@ namespace AMDiS {
     /// partitioning the whole mesh. 
     InteriorBoundary intBoundary;
 
-    /// This map contains for each rank the list of DOFs the current rank must 
-    /// end to exchange solution DOFs at the interior boundaries.
-    DofComm sendDofs;
-
-    /// This map contains on each rank the list of DOFs from which the current 
-    /// rank will receive DOF values (i.e., this are all DOFs at an interior 
-    /// boundary). The DOF indices are given in rank's local numbering.
-    DofComm recvDofs;
-
-    /// This map contains on each rank a list of DOFs along the interior bound-
-    /// aries to communicate with other ranks. The DOF indices are given in rank's
-    /// local numbering. Periodic boundaries within one subdomain are not 
-    /// considered here. 
-    DofComm periodicDofs;
+    DofComm dofComm;
 
     PeriodicMap periodicMap;
 

AMDiS/src/parallel/ParallelDebug.cc

@@ -354,11 +354,13 @@ namespace AMDiS {
     DOFVector<WorldVector<double> > coords(feSpace, "dofCorrds");
     pdb.mesh->getDofIndexCoords(feSpace, coords);
 
-    for (DofComm::Iterator it(pdb.sendDofs, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(pdb.dofComm.getSendDofs(), feSpace); 
+	 !it.end(); it.nextRank())
       for (; !it.endDofIter(); it.nextDof())
 	sendCoords[it.getRank()].push_back(coords[it.getDofIndex()]);
 
-    for (DofComm::Iterator it(pdb.recvDofs, feSpace); !it.end(); it.nextRank())
+    for (DofComm::Iterator it(pdb.dofComm.getRecvDofs(), feSpace);