check the overall number of macro elements with the one in parallel arh2 files...

check the overall number of macro elements with the one in parallel arh2 files after restarting program

AMDiS/src/io/Arh2Reader.cc

@@ -353,8 +353,8 @@ namespace AMDiS { namespace io {
 
     int readMetaFromArh(std::string filename,
 			std::map<int, int> &elInRank,
-			std::map<int, int> &elCodeSize){
-    
+			std::map<int, int> &elCodeSize)
+    {
 #ifdef HAVE_PARALLEL_DOMAIN_AMDIS      
       int nProcs = MPI::COMM_WORLD.Get_size();
 #else

AMDiS/src/io/Arh2Reader.h

@@ -46,11 +46,12 @@ namespace AMDiS { namespace io {
      * if the value is set to true, in Parallel Model, the behavior depends on \ref nProcs. 
      *   But in Sequential Model, an error will be thrown.
      * \param nProcs
-     * the default value is -1. But it only affects when \ref writeParallel is set to true.
+     * It only affects when \ref writeParallel is set to true.
      * If nProcs is -1, and it's in Parallel Model, every processor reads their own filename (with processor
      *   numbers). But in Sequential Model, an error will be thrown.
-     * And you can also set nProcs to a value which is smaller than the number of Processors. Then every 
-     *   processor will read all the files with names from -p0- to -p[nProcs]-.
+     * And you can also set nProcs to the number of the arh files created last time. Then every 
+     *   processor will read all the files to search their own data block, which is useful for restart from
+     *   a different partition.
      */
     void readFile(std::string filename,
                      SystemVector* sysVec,
@@ -193,6 +194,9 @@ namespace AMDiS { namespace io {
     int readMetaData(std::string filename);
 		    
     
+    /// Please make sure the number of processors matches the the number of parallel arh files
+    /// when calling the following functions.
+    
     /// Returns the number of value vectors in the file.
     int readNumOfValueVectors(std::string filename, bool writeParallel = WRITE_PARALLEL);
 

AMDiS/src/io/detail/Arh2Reader.cc

@@ -7,6 +7,7 @@
 #include "Debug.h"
 #include "../Arh2Reader.h"
 
+#include <boost/filesystem.hpp>
 #include <boost/iostreams/filtering_streambuf.hpp>
 #include <boost/iostreams/copy.hpp>
 #ifdef HAVE_COMPRESSION
@@ -466,14 +467,50 @@ namespace AMDiS { namespace io {
 	  string name = filename.substr(0, sPos);
 
 	  if (nProcs == -1) {
-    #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
 	    string procFilename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.arh";
 	    read(procFilename, mesh, vecs, byName);
 	    MSG("ARH file read from: %s\n", procFilename.c_str());
-    #else
+#else
 	    ERROR_EXIT("Reading parallel ARH files in sequential computations requires to specify the number of nodes on which the ARH file was created!\n");
-    #endif
+#endif
 	  } else {
+	    
+	    // check if there are no more or less files as nProcs
+	    int n = 0;
+	    for (; n < nProcs + 1; n++) {
+	      string fn = name + "-p" + boost::lexical_cast<string>(n) + "-.arh";
+	      if(!boost::filesystem::exists(fn)) break;
+	    }
+	    TEST_EXIT(n == nProcs)
+	      ("Number of arh files doesn't match number of processors \n");
+	      
+	    //
+	    // Some test should be checked. This is because of the variation of the
+	    // number of macro elements per processor:
+	    // 
+	    //    There should be at least 10 macro Elements per processor, therefore:
+	    //        nMacroElements * 2^gr >= nProcs * 10
+	    //          =>  gr = log_2(nProcs * 10 / nMacroElements)  
+
+	    int allMacros = mesh->getNumberOfMacros();
+	    int allMacrosFromProcFiles = 0;
+	    
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+	    int sendValue = static_cast<int>(mesh->getNumberOfMacros());
+            MPI::COMM_WORLD.Allreduce(&sendValue, &allMacros, 1, MPI_INT, MPI_SUM);
+	    
+	    if(MPI::COMM_WORLD.Get_rank() == 0) { 
+#endif	      
+	      for(int i = 0; i < nProcs; i++) {
+		allMacrosFromProcFiles += readNumOfMacrosFromSgArh(filename, i);
+	      }
+	      
+	      TEST_EXIT(allMacros == allMacrosFromProcFiles)
+	      ("Number of macro elements in parallel ARH files doesn't match to the current runtime.\n"); 
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+	    }
+#endif    
 	    for (int i = 0; i < nProcs; i++) {
 	      string procFilename = name + "-p" + lexical_cast<string>(i) + "-.arh";
 	      read(procFilename, mesh, vecs, byName);
@@ -594,7 +631,37 @@ namespace AMDiS { namespace io {
 	  }
 	}
       } // end readMetaFromSgArh
-      
+    
+  
+      int readNumOfMacrosFromSgArh(std::string filename, int nProc)
+      {
+	FUNCNAME("Arh2Reader::readHeaderSize");
+	using boost::lexical_cast;
+	
+	int sPos = filename.find(".arh");
+	TEST_EXIT(sPos >= 0)("Failed to find file postfix!\n");
+	
+	if (nProc >= 0) {
+	  string name = filename.substr(0, sPos);
+	  filename = name + "-p" + lexical_cast<string>(nProc) + "-.arh";
+	}
+	
+	ifstream file;
+	  
+	file.open(filename.c_str(), ios::in | ios::binary);
+	TEST_EXIT(file.is_open())
+	  ("Cannot open file %s\n", filename.c_str());
+	  
+	int nMacroElements = 0;
+	detail::firstRead(file);  
+	file.seekg(20, ios_base::cur); 
+	file.read(reinterpret_cast<char*>(&nMacroElements), 4);
+	file.close();
+	
+	return nMacroElements;
+      }
+
+
     } // end namespace detail
   } // end namespace Arh2Reader
 } } // end namespace io, AMDiS

AMDiS/src/io/detail/Arh2Reader.h

@@ -70,7 +70,10 @@ namespace AMDiS { namespace io {
       /// read meta data from a single ARH-file 
       void readMetaFromSgArh(std::string filename, int nProc, 
 			   std::vector< std::set<std::pair<int, int> > >& data);
+
       
+      int readNumOfMacrosFromSgArh(std::string filename, int nProc = -1);
+
     }//end namespace detail
   } // end namespace Arh2Reader
 } } // end namespace io, AMDiS