adapted reinit (HL_SignedDistTraverse) algorithm for parallel (domain decomposition) AMDiS

AMDiS/src/reinit/HL_SignedDistTraverse.cc

@@ -109,6 +109,12 @@ void HL_SignedDistTraverse::initializeBoundary()
     
     elInfo = stack.traverseNext(elInfo);
   }  // end of: mesh traverse 
+  
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+  // In parallel AMDiS synchronize the bound_DOF DOFVector with the max-assigner and the sD_DOF DOFVector with the min-assigner
+  AMDiS::Parallel::MeshDistributor::globalMeshDistributor->synchVector(*bound_DOF, max_assigner());
+  AMDiS::Parallel::MeshDistributor::globalMeshDistributor->synchVector(*sD_DOF, min_to_zero_assigner());
+#endif
 }
 
 
@@ -123,6 +129,11 @@ void HL_SignedDistTraverse::HL_updateIteration()
   sDOld_DOF = new DOFVector<double>(feSpace, "sDOld_DOF");
   sDOld_DOF->copy(const_cast<DOFVector<double> &>(*sD_DOF));
   
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+  // Update sDOld_DOF on interior domain boundaries
+  AMDiS::Parallel::MeshDistributor::globalMeshDistributor->synchVector(*sDOld_DOF, min_to_zero_assigner());
+#endif  
+  
   // ===== Gauss-Seidel or Jacobi iteration ? =====
   if (GaussSeidelFlag)
     update_DOF = sD_DOF;  
@@ -147,8 +158,23 @@ void HL_SignedDistTraverse::HL_updateIteration()
       elInfo = stack.traverseNext(elInfo);
     }
     
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+    // Update sD_DOF on interior domain boundaries
+    AMDiS::Parallel::MeshDistributor::globalMeshDistributor->synchVector(*sD_DOF, min_to_zero_assigner());
+#endif    
+    
     // ===== Is tolerance reached ? =====
     tol_reached = checkTol();
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+    // Convert bool to int that the datatype MPI_INT can be used in the MPI_Allreduce() command below.
+    int int_tol_reached = (tol_reached ? 1 : 0);
+    int int_tol_reached_overall;
+    // Here MPI_MIN operation in the MPI_Allreduce() command is used that each processor runs the cycle again until each
+    // processor has reached the tolerance. Otherwise the synchVector() method from above would fail.
+    MPI_Allreduce(&int_tol_reached, &int_tol_reached_overall, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+    int_tol_reached = int_tol_reached_overall;
+    tol_reached = (int_tol_reached == 0 ? false : true);
+#endif
     
     sDOld_DOF->copy(const_cast<DOFVector<double> &>(*sD_DOF));
   }

AMDiS/src/reinit/HL_SignedDistTraverse.h

@@ -39,6 +39,32 @@ namespace reinit {
 
 using namespace AMDiS;
 
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+  /**
+   * Functor which assigns a the maximum of a and b.
+   * (Used for synchronizing DOFVectors on interior boundaries in parallel AMDiS)
+   */
+  struct max_assigner: FunctorBase
+  {
+    int getDegree(int d0) const { return d0; }
+    static void eval(double& v0, const double& v1) { v0 = std::max(v0, v1); }
+    void operator()(double &v0, const double& v1) { eval(v0, v1); }
+  };
+#endif
+
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+  /**
+   * Functor which assigns a the minimum of a and b with respect to 0.
+   * (Used for synchronizing DOFVectors on interior boundaries in parallel AMDiS)
+   */
+  struct min_to_zero_assigner: FunctorBase
+  {
+    int getDegree(int d0) const { return d0; }
+    static void eval(double& v0, const double& v1) { v0 = (std::abs(v0) < std::abs(v1) ? (v0) : (v1)); }
+    void operator()(double &v0, const double& v1) { eval(v0, v1); }
+  };
+#endif
+
 class HL_SignedDistTraverse : public HL_SignedDist
 {
 public: