Fixed one billon bugs, and added even more features.

AMDiS/src/parallel/PetscSolverFeti.cc

@@ -1428,7 +1428,7 @@ namespace AMDiS {
     FUNCNAME("PetscSolverFeti::solveReducedFetiMatrix()");
 
     // RHS vector.
-    Vec vec_rhs;
+    Vec vec_rhs, vec_sol;
 
     // Some temporary vectors.
     Vec tmp_b0, tmp_b1, tmp_lagrange0, tmp_primal0, tmp_primal1;
@@ -1450,6 +1450,7 @@ namespace AMDiS {
 
     MatGetVecs(mat_lagrange, PETSC_NULL, &tmp_lagrange0);
     MatGetVecs(mat_lagrange, PETSC_NULL, &vec_rhs);
+    MatGetVecs(mat_lagrange, PETSC_NULL, &vec_sol);
 
     // === Create new rhs ===
 
@@ -1486,7 +1487,7 @@ namespace AMDiS {
 
 
     // === Solve with FETI-DP operator. ===
-    KSPSolve(ksp_feti, vec_rhs, vec_rhs);
+    KSPSolve(ksp_feti, vec_rhs, vec_sol);
 
 
     if (printTimings) {
@@ -1506,7 +1507,7 @@ namespace AMDiS {
     subdomain->solveGlobal(subdomain->getRhsInterior(), tmp_b0);
     MatMult(subdomain->getMatCoarseInt(), tmp_b0, tmp_primal1);
     VecAXPBY(tmp_primal0, -1.0, 1.0, tmp_primal1);
-    MatMultTranspose(mat_lagrange, vec_rhs, tmp_b0);
+    MatMultTranspose(mat_lagrange, vec_sol, tmp_b0);
     subdomain->solveGlobal(tmp_b0, tmp_b0);
     MatMult(subdomain->getMatCoarseInt(), tmp_b0, tmp_primal1);
 
@@ -1517,7 +1518,7 @@ namespace AMDiS {
     // === Solve for u_b. ===
 
     VecCopy(subdomain->getRhsInterior(), tmp_b0);
-    MatMultTranspose(mat_lagrange, vec_rhs, tmp_b1);
+    MatMultTranspose(mat_lagrange, vec_sol, tmp_b1);
     VecAXPBY(tmp_b0, -1.0, 1.0, tmp_b1);
 
     MatMult(subdomain->getMatIntCoarse(), tmp_primal0, tmp_b1);
@@ -1534,6 +1535,7 @@ namespace AMDiS {
     }
 
     VecDestroy(&vec_rhs);
+    VecDestroy(&vec_sol);
     VecDestroy(&tmp_b0);
     VecDestroy(&tmp_b1);
     VecDestroy(&tmp_lagrange0);

AMDiS/src/parallel/PetscSolverGlobalMatrix.cc

@@ -105,7 +105,7 @@ namespace AMDiS {
 
     // === Remove Dirichlet BC DOFs. ===
 
-    removeDirichletBcDofs(mat);
+    //    removeDirichletBcDofs(mat);
     
 
     // === Init PETSc solver. ===
@@ -305,7 +305,7 @@ namespace AMDiS {
 
     // === Remove Dirichlet BC DOFs. ===
 
-    removeDirichletBcDofs(mat);
+    //    removeDirichletBcDofs(mat);
 
     // === Create solver for the non primal (thus local) variables. ===
 
@@ -352,11 +352,6 @@ namespace AMDiS {
 	setDofVector(rhsInterior, vec->getDOFVector(i), i);
     }
 
-
-    // === Remove Dirichlet BC DOFs. ===
-    removeDirichletBcDofs(vec);
-
-
     VecAssemblyBegin(rhsInterior);
     VecAssemblyEnd(rhsInterior);
 
@@ -365,7 +360,11 @@ namespace AMDiS {
       VecAssemblyEnd(rhsCoarseSpace);
     }
 
-   
+
+    // === Remove Dirichlet BC DOFs. ===
+    //    removeDirichletBcDofs(vec);
+
+  
     // === Remove null space, if requested. ===
 
     if (removeRhsNullSpace) {
@@ -398,16 +397,47 @@ namespace AMDiS {
       VecAssemblyEnd(petscSolVec);
     }
 
+    MatNullSpace matNullSpace;
+    Vec nullspaceBasis;
+
+    if (nullspace.size() > 0) {
+      TEST_EXIT_DBG(nullspace.size() == 1)("Not yet implemented!\n");
+
+      VecDuplicate(petscSolVec, &nullspaceBasis);
+
+      for (int i = 0; i < nComponents; i++)
+	setDofVector(nullspaceBasis, nullspace[0]->getDOFVector(i), i, true);
+
+      VecAssemblyBegin(nullspaceBasis);
+      VecAssemblyEnd(nullspaceBasis);
+
+      MatNullSpaceCreate(mpiCommGlobal, PETSC_FALSE, 1, &nullspaceBasis, &matNullSpace);
+      KSPSetNullSpace(kspInterior, matNullSpace);
+
+      MatMult(matIntInt, nullspaceBasis, petscSolVec);
+      PetscReal n;
+      VecNorm(petscSolVec, NORM_2, &n);
+      MSG("NORM IS: %e\n", n);
+    }
+
     // PETSc.
     solve(rhsInterior, petscSolVec);
 
+
+    if (nullspace.size() > 0) {
+      MatNullSpaceDestroy(&matNullSpace);
+      VecDestroy(&nullspaceBasis);
+    }
+
+
     // === Transfere values from PETSc's solution vectors to the DOF vectors. ===
     PetscScalar *vecPointer;
-    VecGetArray(petscSolVec, &vecPointer);
+    VecGetArray(petscSolVec, &vecPointer);    
 
     int c = 0;
     for (int i = 0; i < nComponents; i++) {
       DOFVector<double> &dv = *(vec.getDOFVector(i));
+
       DofMap& d = (*interiorMap)[dv.getFeSpace()].getMap();
       for (DofMap::iterator it = d.begin(); it != d.end(); ++it)
 	if (it->second.local != -1)
@@ -469,7 +499,7 @@ namespace AMDiS {
     VecDestroy(&tmp);
     t0 += MPI::Wtime() - wtime;
 
-    MSG("TIMEING: %.5f %.5f\n", t0, t1);
+    //    MSG("TIMEING: %.5f %.5f\n", t0, t1);
   }
 
 
@@ -765,7 +795,7 @@ namespace AMDiS {
 	  std::set<int>& perAsc = perMap.getAssociations(feSpace, globalRowDof);
 	  double value = *dofIt / (perAsc.size() + 1.0);
 	  VecSetValue(vecInterior, index, value, ADD_VALUES);
-	  
+
 	  for (std::set<int>::iterator perIt = perAsc.begin(); 
 	       perIt != perAsc.end(); ++perIt) {
 	    int mappedDof = perMap.map(feSpace, *perIt, globalRowDof);
@@ -786,6 +816,8 @@ namespace AMDiS {
   {
     FUNCNAME("PetscSolverGlobalMatrix::removeDirichletBcDofs()");
 
+    ERROR_EXIT("DO NOT CALL!\n");
+
     vector<int> dofsInterior, dofsCoarse;
 
     int nComponents = mat->getNumRows();
@@ -800,11 +832,15 @@ namespace AMDiS {
 	for (std::set<DegreeOfFreedom>::iterator it = dirichletDofs.begin();
 	     it != dirichletDofs.end(); ++it) {
 	  if (isCoarseSpace(feSpace, *it)) {
-	    if ((*coarseSpaceMap)[feSpace].isRankDof(*it))
-	      dofsCoarse.push_back(coarseSpaceMap->getMatIndex(i, *it));
+	    if ((*coarseSpaceMap)[feSpace].isRankDof(*it)) {
+	      int globalDof = (*coarseSpaceMap)[feSpace][*it].global;
+	      dofsCoarse.push_back(coarseSpaceMap->getMatIndex(i, globalDof));
+	    }
 	  } else {
-	    if ((*interiorMap)[feSpace].isRankDof(*it))
-	      dofsInterior.push_back(interiorMap->getMatIndex(i, *it));
+	    if ((*interiorMap)[feSpace].isRankDof(*it)) {
+	      int globalDof = (*interiorMap)[feSpace][*it].global;
+	      dofsInterior.push_back(interiorMap->getMatIndex(i, globalDof));
+	    }
 	  }
 	}
       } else {
@@ -825,6 +861,8 @@ namespace AMDiS {
   {
     FUNCNAME("PetscSolverGlobalMatrix::removeDirichletBcDofs()");
 
+    ERROR_EXIT("DO NOT CALL!\n");
+
     int nComponents = vec->getSize();
     for (int i = 0; i < nComponents; i++) {
       const FiniteElemSpace *feSpace = vec->getDOFVector(i)->getFeSpace();
@@ -832,32 +870,31 @@ namespace AMDiS {
       map<DegreeOfFreedom, double> &dirichletValues = 
 	vec->getDOFVector(i)->getDirichletValues();
 
-      MSG("DIRICHLET DOFS: %d -> %d\n", i, dirichletValues.size());
-
-      MSG("MAT IS GLOBAL: %d\n", interiorMap->isMatIndexFromGlobal());
-// 	if (interiorMap->isMatIndexFromGlobal())
-// 	  index = 
-// 	    interiorMap->getMatIndex(nRowVec, globalRowDof) + rStartInterior;
-// 	else
-// 	  index =
-// 	    interiorMap->getMatIndex(nRowVec, dofIt.getDOFIndex()) + rStartInterior;
-
-      
       for (map<DegreeOfFreedom, double>::iterator it = dirichletValues.begin();
 	   it != dirichletValues.end(); ++it) {
 	if (isCoarseSpace(feSpace, it->first)) {
-	  if ((*coarseSpaceMap)[feSpace].isRankDof(it->first))
-	    VecSetValue(rhsCoarseSpace, coarseSpaceMap->getMatIndex(i, it->first), 
+	  if ((*coarseSpaceMap)[feSpace].isRankDof(it->first)) {
+	    int globalDof = (*coarseSpaceMap)[feSpace][it->first].global;
+	    VecSetValue(rhsCoarseSpace, coarseSpaceMap->getMatIndex(i, globalDof), 
 			it->second, INSERT_VALUES);
+	  }
 	} else {
 	  if ((*interiorMap)[feSpace].isRankDof(it->first)) {
-	    MSG("REMOVE: %d %d %d\n", i, it->first, interiorMap->getMatIndex(i, it->first));
-	    VecSetValue(rhsInterior, interiorMap->getMatIndex(i, it->first),
+	    int globalDof = (*interiorMap)[feSpace][it->first].global;
+	    VecSetValue(rhsInterior, interiorMap->getMatIndex(i, globalDof),
 			it->second, INSERT_VALUES);	    
 	  }
 	}
       }
     }
+
+    VecAssemblyBegin(rhsInterior);
+    VecAssemblyEnd(rhsInterior);
+
+    if (coarseSpaceMap) {
+      VecAssemblyBegin(rhsCoarseSpace);
+      VecAssemblyEnd(rhsCoarseSpace);
+    }
   }