Added FETI-DP timings.

AMDiS/src/parallel/PetscSolverFeti.h

@@ -258,13 +258,15 @@ namespace AMDiS {
 
     bool multiLevelTest;
 
-    PetscSolver *subDomainSolver;
+    PetscSolver *subdomain;
 
     int meshLevel;
 
     int rStartInterior;
 
     int nGlobalOverallInterior;
+
+    bool printTimings;
   };
 
 }

AMDiS/src/parallel/PetscSolverGlobalMatrix.cc

@@ -332,14 +332,16 @@ namespace AMDiS {
     TEST_EXIT_DBG(interiorMap)("No parallel DOF map defined!\n");
     
 
+    // === Transfer values from DOF vector to the PETSc vector. === 
     if (coarseSpaceMap) {
-      fillPetscRhsWithCoarseSpace(vec);
+      for (int i = 0; i < vec->getSize(); i++)
+	setDofVector(rhsInterior, rhsCoarseSpace, vec->getDOFVector(i), i);
     } else {
-      // === Transfer values from DOF vector to the PETSc vector. === 
       for (int i = 0; i < vec->getSize(); i++)
 	setDofVector(rhsInterior, vec->getDOFVector(i), i);
     }
 
+
     VecAssemblyBegin(rhsInterior);
     VecAssemblyEnd(rhsInterior);
 
@@ -360,27 +362,6 @@ namespace AMDiS {
   }
 
 
-  void PetscSolverGlobalMatrix::fillPetscRhsWithCoarseSpace(SystemVector *vec)
-  {
-    FUNCNAME("SubDomainSolver::fillPetscRhs()");
-
-    for (int i = 0; i < vec->getSize(); i++) {
-      const FiniteElemSpace *feSpace = vec->getDOFVector(i)->getFeSpace();
-      DOFVector<double>::Iterator dofIt(vec->getDOFVector(i), USED_DOFS);
-      for (dofIt.reset(); !dofIt.end(); ++dofIt) {
-	int index = dofIt.getDOFIndex();
-	if (isCoarseSpace(feSpace, index)) {	  
-	  index = coarseSpaceMap->getMatIndex(i, index);
-	  VecSetValue(rhsCoarseSpace, index, *dofIt, ADD_VALUES);
-	} else {
-	  index = interiorMap->getMatIndex(i, index) + rStartInterior;
-	  VecSetValue(rhsInterior, index, *dofIt, ADD_VALUES);
-	}      
-      }
-    }
-  }
-
-
   void PetscSolverGlobalMatrix::solvePetscMatrix(SystemVector &vec, 
 						 AdaptInfo *adaptInfo)
   {
@@ -704,7 +685,8 @@ namespace AMDiS {
   }
 
 
-  void PetscSolverGlobalMatrix::setDofVector(Vec& petscVec, 
+  void PetscSolverGlobalMatrix::setDofVector(Vec vecInterior, 
+					     Vec vecCoarse,
 					     DOFVector<double>* vec, 
 					     int nRowVec, 
 					     bool rankOnly)
@@ -726,24 +708,38 @@ namespace AMDiS {
       // Get PETSc's mat index of the row DOF.
       int index = 0;
       if (interiorMap->isMatIndexFromGlobal())
-	index = interiorMap->getMatIndex(nRowVec, globalRowDof);
+	index = 
+	  interiorMap->getMatIndex(nRowVec, globalRowDof) + rStartInterior;
       else
-	index = interiorMap->getMatIndex(nRowVec, dofIt.getDOFIndex());
+	index =
+	  interiorMap->getMatIndex(nRowVec, dofIt.getDOFIndex()) + rStartInterior;
 
       if (perMap.isPeriodic(feSpace, globalRowDof)) {
-	std::set<int>& perAsc = perMap.getAssociations(feSpace, globalRowDof);
-	double value = *dofIt / (perAsc.size() + 1.0);
-	VecSetValue(petscVec, index, value, ADD_VALUES);
-
-	for (std::set<int>::iterator perIt = perAsc.begin(); 
-	     perIt != perAsc.end(); ++perIt) {
-	  int mappedDof = perMap.map(feSpace, *perIt, globalRowDof);
-	  int mappedIndex = interiorMap->getMatIndex(nRowVec, mappedDof);
-	  VecSetValue(petscVec, mappedIndex, value, ADD_VALUES);
+	if (coarseSpaceMap && isCoarseSpace(feSpace, dofIt.getDOFIndex())) {
+	  ERROR_EXIT("Periodic coarse space not yet supported!\n");
+	} else {
+	  std::set<int>& perAsc = perMap.getAssociations(feSpace, globalRowDof);
+	  double value = *dofIt / (perAsc.size() + 1.0);
+	  VecSetValue(vecInterior, index, value, ADD_VALUES);
+	  
+	  for (std::set<int>::iterator perIt = perAsc.begin(); 
+	       perIt != perAsc.end(); ++perIt) {
+	    int mappedDof = perMap.map(feSpace, *perIt, globalRowDof);
+	    int mappedIndex = interiorMap->getMatIndex(nRowVec, mappedDof);
+	    VecSetValue(vecInterior, mappedIndex, value, ADD_VALUES);
+	  }
 	}
-      } else {
+      } else {	  
 	// The DOF index is not periodic.
-	VecSetValue(petscVec, index, *dofIt, ADD_VALUES);
+
+	if (coarseSpaceMap && isCoarseSpace(feSpace, dofIt.getDOFIndex())) {
+	  TEST_EXIT_DBG(vecCoarse != PETSC_NULL)("Should not happen!\n");
+
+	  index = coarseSpaceMap->getMatIndex(nRowVec, dofIt.getDOFIndex());
+	  VecSetValue(vecCoarse, index, *dofIt, ADD_VALUES);
+	} else {
+	  VecSetValue(vecInterior, index, *dofIt, ADD_VALUES);
+	}
       }
     }
   }

AMDiS/src/parallel/PetscSolverGlobalMatrix.h

@@ -54,8 +54,6 @@ namespace AMDiS {
 
     void fillPetscRhs(SystemVector *vec);
 
-    void fillPetscRhsWithCoarseSpace(SystemVector *vec);
-
     void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);    
 
     void solveGlobal(Vec &rhs, Vec &sol);
@@ -72,9 +70,19 @@ namespace AMDiS {
     void setDofMatrix(DOFMatrix* mat, int nRowMat = 0, int nColMat = 0);
 
     /// Takes a DOF vector and sends its values to a given PETSc vector.
-    void setDofVector(Vec& petscVec, DOFVector<double>* vec, 
+    void setDofVector(Vec vecInterior,
+		      Vec vecCoarse,
+		      DOFVector<double>* vec, 
 		      int nRowVec, bool rankOnly = false);
 
+    inline void setDofVector(Vec vecInterior,
+			     DOFVector<double>* vec, 
+			     int nRowVec, bool rankOnly = false)
+    {
+      setDofVector(vecInterior, PETSC_NULL, vec, nRowVec, rankOnly);
+    }
+
+
   protected:
     /// Arrays definig the non zero pattern of Petsc's matrix.
     int *d_nnz, *o_nnz;