SubDomainSolver interface for sequentiel sub solvers in FETI-DP method is...

SubDomainSolver interface for sequentiel sub solvers in FETI-DP method is fixed and should work now.

AMDiS/src/parallel/PetscProblemStat.cc

@@ -95,6 +95,8 @@ namespace AMDiS {
 
     petscSolver->solvePetscMatrix(*solution, adaptInfo);   
 
+    petscSolver->destroyVectorData();
+
     if (!storeMatrixData)
       petscSolver->destroyMatrixData();
 

AMDiS/src/parallel/PetscSolver.h

@@ -79,6 +79,9 @@ namespace AMDiS {
     /// Destroys all matrix data structures.
     virtual void destroyMatrixData() = 0;
 
+    /// Detroys all vector data structures.
+    virtual void destroyVectorData() = 0;
+
     virtual Flag getBoundaryDofRequirement()
     {
       return 0;

AMDiS/src/parallel/PetscSolverFeti.h

@@ -56,6 +56,9 @@ namespace AMDiS {
     /// Destroys all matrix data structures.
     void destroyMatrixData();
 
+    /// Detroys all vector data structures.
+    void destroyVectorData();
+
     /// Returns flags to denote which information of the boundary DOFs are 
     /// required by the FETI-DP solver.
     Flag getBoundaryDofRequirement()
@@ -66,8 +69,6 @@ namespace AMDiS {
 	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
     }
 
-    void solveLocalProblem(Vec &rhs, Vec &sol);
-
   protected:
     /// After mesh changes, or if the solver is called the first time, this
     /// function creates all matrix and vector objects with the approriated
@@ -179,32 +180,9 @@ namespace AMDiS {
     /// ranks in which the DOF is contained in.
     map<const FiniteElemSpace*, DofIndexToPartitions> boundaryDofRanks;
 
-#if 0
-    /// Global PETSc matrix of non primal variables.
-    Mat mat_b_b;
-
-    /// Global PETSc matrix of primal variables.
-    Mat mat_primal_primal;
-
-    /// Global PETSc matrices that connect the primal with the non 
-    /// primal variables.
-    Mat mat_b_primal, mat_primal_b;
-#endif
-
     /// Global PETSc matrix of Lagrange variables.
     Mat mat_lagrange;
 
-#if 0
-    /// Right hand side PETSc vectors for primal and non primal variables.
-    Vec f_b, f_primal;
-#endif
-
-#if 0
-    /// PETSc solver object that inverts the matrix of non primal 
-    /// variables, \ref mat_b_b
-    KSP ksp_b;
-#endif
-
     /// 0: Solve the Schur complement on primal variables with iterative solver.
     /// 1: Create the Schur complement matrix explicitly and solve it with a
     ///    direct solver.

AMDiS/src/parallel/PetscSolverFetiStructs.h

@@ -24,6 +24,7 @@
 #define AMDIS_PETSC_SOLVER_FETI_STRUCTS_H
 
 #include <map>
+#include "parallel/SubDomainSolver.h"
 
 namespace AMDiS {
 
@@ -36,19 +37,13 @@ namespace AMDiS {
    * primal schur complement. \ref petscMultMatSchurPrimal
    */
   struct SchurPrimalData {
-    /// Pointers to the matrix containing the primal variables.
-    Mat *mat_primal_primal;
-   
-    /// Coupling matrices between the primal and the B variables.
-    Mat *mat_primal_b, *mat_b_primal;
-
     /// Temporal vector on the B variables.
     Vec tmp_vec_b;
 
     /// Temporal vecor in the primal variables.
     Vec tmp_vec_primal;
 
-    PetscSolverFeti *fetiSolver;
+    SubDomainSolver* subSolver;
   };
 
 
@@ -58,9 +53,6 @@ namespace AMDiS {
    * \ref petscMultMatFeti
    */
   struct FetiData {
-    /// Coupling matrices between the primal and the B variables.
-    Mat *mat_primal_b, *mat_b_primal;
-
     /// Matrix of Lagrange variables.
     Mat *mat_lagrange;
 
@@ -73,7 +65,7 @@ namespace AMDiS {
     /// Temporal vector on the lagrange variables.
     Vec tmp_vec_lagrange;
 
-    PetscSolverFeti *fetiSolver;
+    SubDomainSolver* subSolver;
 
     /// Pointer to the solver of the schur complement on the primal variables.
     KSP *ksp_schur_primal;

AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc

@@ -164,14 +164,6 @@ namespace AMDiS {
     // === Synchronize DOFs at common DOFs, i.e., DOFs that correspond to ===
     // === more than one partition.                                       ===
     meshDistributor->synchVector(vec);
-
-
-    // === Destroy PETSc's variables. ===
-    VecDestroy(&petscRhsVec);
-    for (int i = 0; i < nComponents; i++)
-      VecDestroy(&(nestVec[i]));
-
-    VecDestroy(&petscSolVec);
   }
 
 
@@ -188,6 +180,18 @@ namespace AMDiS {
   }
 
 
+  void PetscSolverGlobalBlockMatrix::destroyVectorData()
+  {
+    FUNCNAME("PetscSolverGlobalBlockMatrix::destroyVectorData()");
+
+    VecDestroy(&petscRhsVec);
+    for (int i = 0; i < nComponents; i++)
+      VecDestroy(&(nestVec[i]));
+
+    VecDestroy(&petscSolVec);
+  }
+
+
   void PetscSolverGlobalBlockMatrix::setDofMatrix(Mat& petscMat, 
 						  DOFMatrix* mat,
 						  int dispRowBlock, 

AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.h

@@ -44,10 +44,12 @@ namespace AMDiS {
 
     void fillPetscRhs(SystemVector *vec);
 
-    virtual void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);    
+    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);    
 
     void destroyMatrixData();
 
+    void destroyVectorData();
+
   protected:
     /// Takes a DOF matrix and sends the values to the global PETSc matrix.
     void setDofMatrix(Mat& petscMat, DOFMatrix* mat,

AMDiS/src/parallel/PetscSolverGlobalMatrix.cc

@@ -211,10 +211,6 @@ namespace AMDiS {
 
     // Print iteration counter and residual norm of the solution.
     printSolutionInfo(adaptInfo);
-
-    // === Destroy PETSc's variables. ===
-
-    VecDestroy(&petscRhsVec);
   }
 
 
@@ -229,6 +225,14 @@ namespace AMDiS {
   }
 
 
+  void PetscSolverGlobalMatrix::destroyVectorData()
+  {
+    FUNCNAME("PetscSolverGlobalMatrix::destroyVectorData()");
+
+    VecDestroy(&petscRhsVec);
+  }
+
+
   void PetscSolverGlobalMatrix::setDofMatrix(DOFMatrix* mat,
 					     int nRowMat, int nColMat)
   {

AMDiS/src/parallel/PetscSolverGlobalMatrix.h

@@ -55,6 +55,8 @@ namespace AMDiS {
 
     void destroyMatrixData();
 
+    void destroyVectorData();
+
   protected:
     /// Creates a new non zero pattern structure for the PETSc matrix. 
     void createPetscNnzStructure(Matrix<DOFMatrix*> *mat);

AMDiS/src/parallel/PetscSolverSchur.h

@@ -43,7 +43,10 @@ namespace AMDiS {
 
     void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
 
-    void destroyMatrixData()
+    void destroyMatrixData() 
+    {}
+
+    void destroyVectorData() 
     {}
 
     Flag getBoundaryDofRequirement()

AMDiS/src/parallel/SubDomainSolver.cc

@@ -218,7 +218,13 @@ namespace AMDiS {
   void SubDomainSolver::solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo)
   {
     FUNCNAME("SubDomainSolver::solvePetscMatrix()");
-    
+  }
+
+
+  void SubDomainSolver::destroyVectorData()
+  {
+    FUNCNAME("SubDomainSolver::destroyVectorData()");
+
     VecDestroy(&rhsCoarseSpace);
     VecDestroy(&rhsInterior);
   }
@@ -243,6 +249,38 @@ namespace AMDiS {
   }
 
 
+  void SubDomainSolver::solveGlobal(Vec &rhs, Vec &sol)
+  {
+    FUNCNAME("SubDomainSolver::solveGlobal()");
+
+    Vec tmp;
+    VecCreateSeq(PETSC_COMM_SELF, interiorMap->getRankDofs(), &tmp);
+
+    PetscScalar *tmpValues, *rhsValues;
+    VecGetArray(tmp, &tmpValues);
+    VecGetArray(rhs, &rhsValues);
+
+    for (int i = 0; i < interiorMap->getRankDofs(); i++)
+      tmpValues[i] = rhsValues[i];
+
+    VecRestoreArray(rhs, &rhsValues);
+    VecRestoreArray(tmp, &tmpValues);
+
+    KSPSolve(kspInterior, tmp, tmp);
+
+    VecGetArray(tmp, &tmpValues);
+    VecGetArray(sol, &rhsValues);
+
+    for (int i = 0; i < interiorMap->getRankDofs(); i++) 
+      rhsValues[i] = tmpValues[i];
+
+    VecRestoreArray(sol, &rhsValues);
+    VecRestoreArray(tmp, &tmpValues);
+
+    VecDestroy(&tmp);
+  }
+
+
   vector<const FiniteElemSpace*> SubDomainSolver::getFeSpaces(Matrix<DOFMatrix*> *mat)
   {
     FUNCNAME("SubDomainSolver::getFeSpaces()");

AMDiS/src/parallel/SubDomainSolver.h

@@ -59,10 +59,14 @@ namespace AMDiS {
 
     void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
 
+    void destroyVectorData();
+
     void destroyMatrixData();
 
     void solve(Vec &rhs, Vec &sol);
 
+    void solveGlobal(Vec &rhs, Vec &sol);
+
     inline bool isCoarseSpace(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)
     {
       return (*coarseSpaceMap)[feSpace].isSet(dof);