From b115e944344ff6b55be5b31b9383bddbd00e37f1 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Wed, 8 Feb 2012 13:43:37 +0000
Subject: [PATCH] Small changes and code refactoring in parallel AMDiS.

---
 AMDiS/src/MeshStructure.cc                    | 12 ++++++++----
 AMDiS/src/MeshStructure.h                     |  6 ++----
 AMDiS/src/io/ArhReader.cc                     |  9 ++++-----
 AMDiS/src/parallel/DofComm.cc                 |  8 +++++++-
 AMDiS/src/parallel/MeshDistributor.cc         |  6 +++++-
 AMDiS/src/parallel/ParallelDebug.cc           |  2 +-
 AMDiS/src/parallel/PetscSolverGlobalMatrix.cc |  1 +
 7 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/AMDiS/src/MeshStructure.cc b/AMDiS/src/MeshStructure.cc
index 7c65aa6e..d3650818 100644
--- a/AMDiS/src/MeshStructure.cc
+++ b/AMDiS/src/MeshStructure.cc
@@ -137,14 +137,18 @@ namespace AMDiS {
 
       if (!reverseOrder) {
 	if (s1 != -1) 
-	  addAlongSide(el->getFirstChild(), subObj, s1, el->getChildType(elType), reverseOrder);
+	  addAlongSide(el->getFirstChild(), subObj, s1, 
+		       el->getChildType(elType), reverseOrder);
 	if (s2 != -1)
-	  addAlongSide(el->getSecondChild(), subObj, s2, el->getChildType(elType), reverseOrder);
+	  addAlongSide(el->getSecondChild(), subObj, s2, 
+		       el->getChildType(elType), reverseOrder);
       } else {
 	if (s2 != -1)
-	  addAlongSide(el->getSecondChild(), subObj, s2, el->getChildType(elType), reverseOrder);
+	  addAlongSide(el->getSecondChild(), subObj, s2, 
+		       el->getChildType(elType), reverseOrder);
 	if (s1 != -1) 
-	  addAlongSide(el->getFirstChild(), subObj, s1, el->getChildType(elType), reverseOrder);
+	  addAlongSide(el->getFirstChild(), subObj, s1, 
+		       el->getChildType(elType), reverseOrder);
       }
     }
   }
diff --git a/AMDiS/src/MeshStructure.h b/AMDiS/src/MeshStructure.h
index d2b18632..e9f2732d 100644
--- a/AMDiS/src/MeshStructure.h
+++ b/AMDiS/src/MeshStructure.h
@@ -66,10 +66,8 @@ namespace AMDiS {
       reset();
     }
 
-    /** \brief
-     * Sets all position counters, that are used to traverse the code, to the starting
-     * position. The code itself is not changed.
-     */
+    /// Sets all position counters, that are used to traverse the code, to the 
+    /// starting position. The code itself is not changed.
     void reset();
 
     /// Returns whether the code is empty or not.
diff --git a/AMDiS/src/io/ArhReader.cc b/AMDiS/src/io/ArhReader.cc
index 78bce83e..bad8fa0b 100644
--- a/AMDiS/src/io/ArhReader.cc
+++ b/AMDiS/src/io/ArhReader.cc
@@ -51,7 +51,6 @@ namespace AMDiS {
   {
     FUNCNAME("ArhReader::read()");
 
-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
     if (writeParallel) {
       using boost::lexical_cast;
       int sPos = filename.find(".arh");
@@ -59,8 +58,12 @@ namespace AMDiS {
       string name = filename.substr(0, sPos);      
 
       if (nProcs == -1) {     
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
 	string procFilename = name + "-p" + lexical_cast<string>(MPI::COMM_WORLD.Get_rank()) + "-.arh";
 	readFile(procFilename, mesh, vecs);
+#else
+	ERROR_EXIT("Reading parallel ARH files in sequential computations requires to specify the number of nodes on which the ARH file was created!\n");
+#endif
       } else {
 	for (int i = 0; i < nProcs; i++) {
 	  string procFilename = name + "-p" + lexical_cast<string>(i) + "-.arh";
@@ -70,10 +73,6 @@ namespace AMDiS {
     } else {
       readFile(filename, mesh, vecs);
     }
-#else
-    readFile(filename, mesh, vecs);
-#endif
-
     MSG("ARH file read from: %s\n", filename.c_str());
   }
 
diff --git a/AMDiS/src/parallel/DofComm.cc b/AMDiS/src/parallel/DofComm.cc
index 28a51091..dd812a66 100644
--- a/AMDiS/src/parallel/DofComm.cc
+++ b/AMDiS/src/parallel/DofComm.cc
@@ -18,7 +18,8 @@ namespace AMDiS {
 
   void DofComm::removeEmpty()
   {
-    for (DataIter dit = data.begin(); dit != data.end(); ++dit) {
+    DataIter dit = data.begin();
+    while (dit != data.end()) {
       FeMapIter it = dit->second.begin();
       while (it != dit->second.end()) {
 	if (it->second.size() == 0) {
@@ -28,6 +29,11 @@ namespace AMDiS {
 	} else
 	  ++it;
       }
+      
+      if (dit->second.size() == 0)
+	data.erase(dit++);
+      else
+	++dit;
     }
   }
 
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 9e8c07db..d36930c1 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -777,6 +777,9 @@ namespace AMDiS {
 
     double first = MPI::Wtime();
 
+    int skip = 0;
+    Parameters::get("parallel->debug->skip check mesh change", skip);
+
     // === If mesh has not been changed on all ranks, return. ===
 
     int recvAllValues = 0;
@@ -788,7 +791,8 @@ namespace AMDiS {
 
     // === At least one rank mesh has been changed, so the boundaries must be ===
     // === adapted to the new mesh structure.                                 ===
-   
+
+    if (skip == 0)
     do {
       bool meshChanged = false;
 
diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc
index 54bf712e..c0530c07 100644
--- a/AMDiS/src/parallel/ParallelDebug.cc
+++ b/AMDiS/src/parallel/ParallelDebug.cc
@@ -785,7 +785,7 @@ namespace AMDiS {
   {
     FUNCNAME("ParallelDebug::writeCoordsFile()");
 
-    const FiniteElemSpace *feSpace = pdb.feSpaces[0];
+    const FiniteElemSpace *feSpace = pdb.feSpaces[pdb.feSpaces.size() - 1];
 
     stringstream filename;
     filename << prefix << "-" << pdb.mpiRank << "." << postfix;
diff --git a/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc b/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
index afcd156a..8470ad97 100644
--- a/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
+++ b/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
@@ -431,6 +431,7 @@ namespace AMDiS {
 	    ("Should not happen!\n");
 
 	  int rowIndex = rowIt->first;
+
 	  MatSetValues(petscMatrix, 1, &rowIndex, rowIt->second.size(),
 		       &(rowIt->second[0]), &(valsMap[rowIt->first][0]), ADD_VALUES);
 	}
-- 
GitLab