Commit 2aff8c3b authored by Praetorius, Simon's avatar Praetorius, Simon
Browse files

some corrections in the solvers

parent 0b5440f3
...@@ -255,6 +255,9 @@ int i ; ...@@ -255,6 +255,9 @@ int i ;
void *Symbolic, *Numeric ; void *Symbolic, *Numeric ;
double stats [2];
umfpack_tic (stats);
(void) umfpack_di_symbolic (n, n, Ap, Ai, Ax, &Symbolic, null, null) ; (void) umfpack_di_symbolic (n, n, Ap, Ai, Ax, &Symbolic, null, null) ;
(void) umfpack_di_numeric (Ap, Ai, Ax, Symbolic, &Numeric, null, null) ; (void) umfpack_di_numeric (Ap, Ai, Ax, Symbolic, &Numeric, null, null) ;
...@@ -265,6 +268,7 @@ umfpack_di_free_symbolic (&Symbolic) ; ...@@ -265,6 +268,7 @@ umfpack_di_free_symbolic (&Symbolic) ;
umfpack_di_free_numeric (&Numeric) ; umfpack_di_free_numeric (&Numeric) ;
umfpack_toc (stats);
return (0) ; return (0) ;
...@@ -276,18 +280,30 @@ return (0) ; ...@@ -276,18 +280,30 @@ return (0) ;
OUTPUT_VARIABLE CHOLMOD_OUT) OUTPUT_VARIABLE CHOLMOD_OUT)
if(NOT CHOLMOD_TEST) if(NOT CHOLMOD_TEST)
find_library(CHOLMOD_LIB cholmod) find_library(CHOLMOD_LIB cholmod)
find_library(COLAMD_LIB colamd)
if(CHOLMOD_LIB) if(CHOLMOD_LIB)
list(APPEND AMDIS_LIBRARIES ${CHOLMOD_LIB}) list(APPEND AMDIS_LIBRARIES ${CHOLMOD_LIB})
else() else()
message(FATAL_ERROR "your umfpack seems to need cholmod, but cmake could not find it") message(FATAL_ERROR "your umfpack seems to need cholmod, but cmake could not find it")
endif() endif()
find_library(COLAMD_LIB colamd)
if(COLAMD_LIB) if(COLAMD_LIB)
list(APPEND AMDIS_LIBRARIES ${COLAMD_LIB}) list(APPEND AMDIS_LIBRARIES ${COLAMD_LIB})
else() else()
message(FATAL_ERROR "your umfpack seems to need colamd, but cmake could not find it") message(FATAL_ERROR "your umfpack seems to need colamd, but cmake could not find it")
endif() endif()
try_compile(CHOLMOD_TEST2 ${_CHOLMOD_TEST_DIR}/build ${_CHOLMOD_TEST_DIR} cholmodTest
OUTPUT_VARIABLE CHOLMOD_OUT)
if(NOT CHOLMOD_TEST2)
find_library(SUITESPARSECONFIG_LIB libsuitesparseconfig)
if(SUITESPARSECONFIG_LIB)
list(APPEND AMDIS_LIBRARIES ${SUITESPARSECONFIG_LIB})
else()
message(FATAL_ERROR "your umfpack seems to need suitesparseconfig, but cmake could not find it")
endif()
endif()
endif() endif()
endif(AMDIS_NEED_UMFPACK) endif(AMDIS_NEED_UMFPACK)
......
...@@ -53,8 +53,11 @@ elseif(AMDIS_NEED_SEQ_PETSC) ...@@ -53,8 +53,11 @@ elseif(AMDIS_NEED_SEQ_PETSC)
endif() endif()
if(AMDIS_NEED_HYPRE) if(AMDIS_NEED_HYPRE)
find_package(HYPRE REQUIRED) find_package(HYPRE REQUIRED HINTS ${AMDIS_DIR})
if(HAVE_HYPRE) if(HAVE_HYPRE)
list(APPEND AMDIS_INCLUDE_DIRS ${HYPRE_INCLUDE_DIRECTORIES})
list(APPEND AMDIS_COMPILEFLAGS "-DMTL_HAS_HYPRE")
list(APPEND AMDIS_LIBRARIES ${HYPRE_LIBRARIES} )
if(NOT MPI_FOUND) if(NOT MPI_FOUND)
find_package(MPI REQUIRED) find_package(MPI REQUIRED)
if(MPI_FOUND) if(MPI_FOUND)
...@@ -63,9 +66,6 @@ if(AMDIS_NEED_HYPRE) ...@@ -63,9 +66,6 @@ if(AMDIS_NEED_HYPRE)
list(APPEND AMDIS_INCLUDE_DIRS ${MPI_INCLUDE_PATH}) list(APPEND AMDIS_INCLUDE_DIRS ${MPI_INCLUDE_PATH})
endif() endif()
endif() endif()
list(APPEND AMDIS_INCLUDE_DIRS ${HYPRE_INCLUDE_DIRECTORIES})
list(APPEND AMDIS_COMPILEFLAGS "-DMTL_HAS_HYPRE")
list(APPEND AMDIS_LIBRARIES ${HYPRE_LIBRARIES} )
endif() endif()
endif(AMDIS_NEED_HYPRE) endif(AMDIS_NEED_HYPRE)
......
...@@ -340,10 +340,10 @@ endif(ENABLE_UMFPACK) ...@@ -340,10 +340,10 @@ endif(ENABLE_UMFPACK)
if(ENABLE_HYPRE) if(ENABLE_HYPRE)
include(FindHYPRE.cmake) include(FindHYPRE.cmake)
message("have hypre: ${HAVE_HYPRE}") message("have hypre: ${HAVE_HYPRE}")
if(HAVE_HYPRE) if (HAVE_HYPRE)
if(NOT MPI_FOUND) if (NOT MPI_FOUND)
find_package(MPI REQUIRED) find_package(MPI REQUIRED)
if(MPI_FOUND) if (MPI_FOUND)
list(APPEND COMPILEFLAGS "${MPI_COMPILE_FLAGS}") list(APPEND COMPILEFLAGS "${MPI_COMPILE_FLAGS}")
include_directories(${MPI_INCLUDE_PATH}) include_directories(${MPI_INCLUDE_PATH})
endif() endif()
......
...@@ -48,7 +48,7 @@ namespace AMDiS { ...@@ -48,7 +48,7 @@ namespace AMDiS {
return MPI::Wtime() - first_mpi; return MPI::Wtime() - first_mpi;
#else #else
time_duration td = microsec_clock::local_time()-first_seq; time_duration td = microsec_clock::local_time()-first_seq;
return static_cast<double>(td.total_milliseconds())*1.e-3; return static_cast<double>(td.total_microseconds())*1.e-6;
#endif #endif
} }
......
...@@ -29,6 +29,8 @@ ...@@ -29,6 +29,8 @@
#include "parallel/PetscSolver.h" #include "parallel/PetscSolver.h"
#include "parallel/StdMpi.h" #include "parallel/StdMpi.h"
#include "tools.h"
namespace AMDiS { namespace Parallel { namespace AMDiS { namespace Parallel {
PetscSolver::PetscSolver(std::string name) PetscSolver::PetscSolver(std::string name)
...@@ -86,10 +88,10 @@ namespace AMDiS { namespace Parallel { ...@@ -86,10 +88,10 @@ namespace AMDiS { namespace Parallel {
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
Timer t; Timer t;
if (createMatrixData) if (createMatrixData)
fillPetscMatrix(const_cast< Matrix< DOFMatrix* >* >(A.getOriginalMat())); fillPetscMatrix(const_cast< Matrix< DOFMatrix* >* >(A.getOriginalMat()));
fillPetscRhs(&b); fillPetscRhs(&b);
INFO(info, 8)("creation of parallel data structures needed %.5f seconds\n", INFO(info, 8)("creation of parallel data structures needed %.5f seconds\n",
......
...@@ -66,6 +66,8 @@ namespace AMDiS { namespace Parallel { ...@@ -66,6 +66,8 @@ namespace AMDiS { namespace Parallel {
Parameters::get(name + "->pc_type", precon); Parameters::get(name + "->pc_type", precon);
if (!precon.size()) if (!precon.size())
Parameters::get(name + "->left precon", precon); Parameters::get(name + "->left precon", precon);
if (!precon.size())
Parameters::get(name + "->right precon", precon);
if (!matSolverPackage && params.emptyParam.find(precon) == params.emptyParam.end()) { if (!matSolverPackage && params.emptyParam.find(precon) == params.emptyParam.end()) {
precon = (params.preconMap.find(precon) != params.preconMap.end() ? params.preconMap[precon] : precon); precon = (params.preconMap.find(precon) != params.preconMap.end() ? params.preconMap[precon] : precon);
PetscOptionsInsertString(("-" + kspPrefix + "pc_type " + precon).c_str()); PetscOptionsInsertString(("-" + kspPrefix + "pc_type " + precon).c_str());
......
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
#include "parallel/PetscHelper.h" #include "parallel/PetscHelper.h"
#include "TransformDOF.h" #include "TransformDOF.h"
#include "tools.h"
namespace AMDiS { namespace Parallel { namespace AMDiS { namespace Parallel {
using namespace std; using namespace std;
...@@ -114,6 +116,7 @@ namespace AMDiS { namespace Parallel { ...@@ -114,6 +116,7 @@ namespace AMDiS { namespace Parallel {
{ {
FUNCNAME("PetscSolverNavierStokes::initSolver()"); FUNCNAME("PetscSolverNavierStokes::initSolver()");
printMem("begin(initSolver)");
// Create FGMRES based outer solver // Create FGMRES based outer solver
MSG("CREATE POS 1: %p\n", &ksp); MSG("CREATE POS 1: %p\n", &ksp);
...@@ -128,6 +131,8 @@ namespace AMDiS { namespace Parallel { ...@@ -128,6 +131,8 @@ namespace AMDiS { namespace Parallel {
// Create null space information. // Create null space information.
if (pressureNullSpace) if (pressureNullSpace)
setConstantNullSpace(ksp, pressureComponent, true); setConstantNullSpace(ksp, pressureComponent, true);
printMem("end(initSolver)");
} }
...@@ -140,6 +145,9 @@ namespace AMDiS { namespace Parallel { ...@@ -140,6 +145,9 @@ namespace AMDiS { namespace Parallel {
TEST_EXIT(nu)("nu pointer not set!\n"); TEST_EXIT(nu)("nu pointer not set!\n");
TEST_EXIT(invTau)("invtau pointer not set!\n"); TEST_EXIT(invTau)("invtau pointer not set!\n");
TEST_EXIT(solution)("solution pointer not set!\n"); TEST_EXIT(solution)("solution pointer not set!\n");
printMem("begin(initPreconditioner)");
int dim = componentSpaces[pressureComponent]->getMesh()->getDim(); int dim = componentSpaces[pressureComponent]->getMesh()->getDim();
...@@ -356,6 +364,9 @@ namespace AMDiS { namespace Parallel { ...@@ -356,6 +364,9 @@ namespace AMDiS { namespace Parallel {
} }
setConstantNullSpace(matShellContext.kspLaplace); setConstantNullSpace(matShellContext.kspLaplace);
printMem("end(initPreconditioner)");
MSG("Setup of Navier-Stokes preconditioner needed %.5f seconds\n", MSG("Setup of Navier-Stokes preconditioner needed %.5f seconds\n",
t.elapsed()); t.elapsed());
......
...@@ -89,11 +89,17 @@ namespace AMDiS { ...@@ -89,11 +89,17 @@ namespace AMDiS {
TEST_EXIT(preconPair.l != nullptr)("there is no left preconditioner\n"); TEST_EXIT(preconPair.l != nullptr)("there is no left preconditioner\n");
TEST_EXIT(preconPair.r != nullptr)("there is no right preconditioner\n"); TEST_EXIT(preconPair.r != nullptr)("there is no right preconditioner\n");
typedef typename mtl::Collection<MatrixType>::value_type value_type;
VectorType r(A * x - b); VectorType r(num_rows(b)); r = b;
if (two_norm(x) != 0) {
r = A * x ;
r -= b;
}
int error = 0; int error = 0;
if (oem.getInfo() == 0) { if (oem.getInfo() == 0) {
itl::basic_iteration<typename MatrixType::value_type> itl::basic_iteration<value_type>
iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance()); iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance());
error = solver(A, x, b, *(preconPair.l), *(preconPair.r), iter); error = solver(A, x, b, *(preconPair.l), *(preconPair.r), iter);
...@@ -101,7 +107,7 @@ namespace AMDiS { ...@@ -101,7 +107,7 @@ namespace AMDiS {
oem.setIterations(iter.iterations()); oem.setIterations(iter.iterations());
oem.setResidual(iter.resid()); oem.setResidual(iter.resid());
} else { } else {
itl::cyclic_iteration<typename MatrixType::value_type> itl::cyclic_iteration<value_type>
iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance(), iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance(),
oem.getPrint_cycle()); oem.getPrint_cycle());
...@@ -122,29 +128,31 @@ namespace AMDiS { ...@@ -122,29 +128,31 @@ namespace AMDiS {
TEST_EXIT(preconPair.l != nullptr)("there is no left preconditioner\n"); TEST_EXIT(preconPair.l != nullptr)("there is no left preconditioner\n");
TEST_EXIT(preconPair.r != nullptr)("there is no right preconditioner\n"); TEST_EXIT(preconPair.r != nullptr)("there is no right preconditioner\n");
mtl::matrix::transposed_view<const MatrixType> B(A); // typedef typename mtl::Collection<MatrixType>::value_type value_type;
VectorType r(B * x - b); // mtl::matrix::transposed_view<const MatrixType> B(A);
int error = 0; // VectorType r(B * x - b);
if (oem.getInfo() == 0) { // int error = 0;
itl::basic_iteration<typename MatrixType::value_type> // if (oem.getInfo() == 0) {
iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance()); // itl::basic_iteration<value_type>
// iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance());
error = solver(B, x, b, *(preconPair.l), *(preconPair.r), iter); //
oem.setErrorCode(error); // error = solver(B, x, b, *(preconPair.l), *(preconPair.r), iter);
oem.setIterations(iter.iterations()); // oem.setErrorCode(error);
oem.setResidual(iter.resid()); // oem.setIterations(iter.iterations());
} else { // oem.setResidual(iter.resid());
itl::cyclic_iteration<typename MatrixType::value_type> // } else {
iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance(), // itl::cyclic_iteration<value_type>
oem.getPrint_cycle()); // iter(r, oem.getMaxIterations(), oem.getRelative(), oem.getTolerance(),
// oem.getPrint_cycle());
error = solver(B, x, b, *(preconPair.l), *(preconPair.r), iter); //
oem.setErrorCode(error); // error = solver(B, x, b, *(preconPair.l), *(preconPair.r), iter);
oem.setIterations(iter.iterations()); // oem.setErrorCode(error);
oem.setResidual(iter.resid()); // oem.setIterations(iter.iterations());
} // oem.setResidual(iter.resid());
// }
return error; //
// return error;
return 1;
} }
......
...@@ -43,8 +43,8 @@ namespace AMDiS { ...@@ -43,8 +43,8 @@ namespace AMDiS {
/// Constructor /// Constructor
MTL4SolverBase(LinearSolver* oem_) MTL4SolverBase(LinearSolver* oem_)
: runner(oem_), : runner(oem_),
oem(*oem_), oem(*oem_)/*,
matrix(0,0) matrix(0,0)*/
{} {}
protected: protected:
......
...@@ -56,8 +56,9 @@ namespace AMDiS { ...@@ -56,8 +56,9 @@ namespace AMDiS {
mat(mat), mapper(m) {} mat(mat), mapper(m) {}
}; };
template< typename MTLMatrix, typename Mapper > // requires MatrixType to have an inserter
void operator<<(MTLMatrix& matrix, MatMap<const SolverMatrix<Matrix<DOFMatrix* > >, Mapper >& Asolver) template< typename MatrixType, typename Mapper >
void operator<<(MatrixType& matrix, MatMap<const SolverMatrix<Matrix<DOFMatrix* > >, Mapper >& Asolver)
{ {
const Matrix< DOFMatrix* >& A = *(Asolver.mat.getOriginalMat()); const Matrix< DOFMatrix* >& A = *(Asolver.mat.getOriginalMat());
int ns = A.getSize(); int ns = A.getSize();
...@@ -72,7 +73,7 @@ namespace AMDiS { ...@@ -72,7 +73,7 @@ namespace AMDiS {
nnz += A[rb][cb]->getBaseMatrix().nnz(); nnz += A[rb][cb]->getBaseMatrix().nnz();
{ {
typedef mtl::matrix::mapped_inserter< typename Collection< MTLMatrix >::Inserter, Mapper > Inserter; typedef mtl::matrix::mapped_inserter< typename Collection< MatrixType >::Inserter, Mapper > Inserter;
Inserter ins(matrix, mapper, int(1.2 * nnz / matrix.dim1())); Inserter ins(matrix, mapper, int(1.2 * nnz / matrix.dim1()));
for (int rb = 0; rb < ns; ++rb) { for (int rb = 0; rb < ns; ++rb) {
mapper.setRow(rb); mapper.setRow(rb);
...@@ -84,6 +85,37 @@ namespace AMDiS { ...@@ -84,6 +85,37 @@ namespace AMDiS {
} }
} }
} }
// requires MatrixType to have an inserter
template< typename MatrixType, typename Mapper >
void operator<<(MatrixType& matrix, MatMap<Matrix<MatrixType* >, Mapper >& Asolver)
{
Matrix< MatrixType* >& A = *(Asolver.mat);
int ns = A.getSize();
Mapper& mapper(Asolver.mapper);
set_to_zero(matrix);
int nnz = 0;
for (int rb = 0; rb < ns; ++rb)
for (int cb = 0; cb < ns; ++cb)
if (A[rb][cb])
nnz += A[rb][cb]->nnz();
{
typedef mtl::matrix::mapped_inserter< typename Collection< MatrixType >::Inserter, Mapper > Inserter;
Inserter ins(matrix, mapper, int(1.2 * nnz / matrix.dim1()));
for (int rb = 0; rb < ns; ++rb) {
mapper.setRow(rb);
for (int cb = 0; cb < ns; ++cb) {
mapper.setCol(cb);
if (A[rb][cb])
ins << (*A[rb][cb]);
}
}
}
}
template< typename Vector, typename CurMap > template< typename Vector, typename CurMap >
void operator<<(Vector& dest, VecMap<DOFVector<double >, CurMap >& rhs) void operator<<(Vector& dest, VecMap<DOFVector<double >, CurMap >& rhs)
...@@ -185,31 +217,31 @@ namespace AMDiS { ...@@ -185,31 +217,31 @@ namespace AMDiS {
/// create and fill \ref PetscMatrix /// create and fill \ref PetscMatrix
inline void operator<<(PetscMatrix& mat, const SolverMatrix<Matrix<DOFMatrix*> >& Asolver) inline void operator<<(PetscMatrix& mat, const SolverMatrix<Matrix<DOFMatrix*> >& Asolver)
{ {
const Matrix< DOFMatrix* >& A = *(Asolver.getOriginalMat()); const Matrix< DOFMatrix* >& A = *(Asolver.getOriginalMat());
if(mat.isNested) { if (mat.isNested) {
std::vector<PetscInt> nnz;
mat.nestMat.resize(A.getNumRows() * A.getNumCols());
for (size_t i = 0; i < static_cast<size_t>(A.getNumRows()); i++) {
for (size_t j = 0; j < static_cast<size_t>(A.getNumCols()); j++) {
size_t idx = i * A.getNumCols() + j;
if (A[i][j] == nullptr
|| num_rows(A[i][j]->getBaseMatrix()) == 0
|| num_cols(A[i][j]->getBaseMatrix()) == 0
|| A[i][j]->getBaseMatrix().nnz() == 0) {
mat.nestMat[idx] = PETSC_NULL;
continue;
}
mat.nestMat[idx] << *(A[i][j]); std::vector<PetscInt> nnz;
} mat.nestMat.resize(A.getNumRows() * A.getNumCols());
}
for (size_t i = 0; i < static_cast<size_t>(A.getNumRows()); i++) {
for (size_t j = 0; j < static_cast<size_t>(A.getNumCols()); j++) {
size_t idx = i * A.getNumCols() + j;
if (A[i][j] == nullptr
|| num_rows(A[i][j]->getBaseMatrix()) == 0
|| num_cols(A[i][j]->getBaseMatrix()) == 0
|| A[i][j]->getBaseMatrix().nnz() == 0) {
mat.nestMat[idx] = PETSC_NULL;
continue;
}
// create nested matrix from a vector of block matrices mat.nestMat[idx] << *(A[i][j]);
MatCreateNest(PETSC_COMM_SELF, A.getNumRows(), PETSC_NULL, A.getNumCols(), PETSC_NULL, &(mat.nestMat[0]), &mat.matrix); }
}else { }
// create nested matrix from a vector of block matrices
MatCreateNest(PETSC_COMM_SELF, A.getNumRows(), PETSC_NULL, A.getNumCols(), PETSC_NULL, &(mat.nestMat[0]), &mat.matrix);
} else {
bool initMatrix = false; bool initMatrix = false;
unsigned nRows(0); unsigned nRows(0);
unsigned nEntries(0); unsigned nEntries(0);
......
...@@ -63,6 +63,8 @@ namespace AMDiS { ...@@ -63,6 +63,8 @@ namespace AMDiS {
Parameters::get(oem.getName() + "->pc_type", precon); Parameters::get(oem.getName() + "->pc_type", precon);
if (!precon.size()) if (!precon.size())
Parameters::get(oem.getName() + "->left precon", precon); Parameters::get(oem.getName() + "->left precon", precon);
if (!precon.size())
Parameters::get(oem.getName() + "->right precon", precon);
if (!matSolverPackage && !params.emptyParam[precon]) { if (!matSolverPackage && !params.emptyParam[precon]) {
precon = (params.preconMap.find(precon) != params.preconMap.end() ? params.preconMap[precon] : precon); precon = (params.preconMap.find(precon) != params.preconMap.end() ? params.preconMap[precon] : precon);
PetscOptionsInsertString(("-" + kspPrefix + "pc_type " + precon).c_str()); PetscOptionsInsertString(("-" + kspPrefix + "pc_type " + precon).c_str());
...@@ -138,6 +140,7 @@ namespace AMDiS { ...@@ -138,6 +140,7 @@ namespace AMDiS {
PC pc_; PC pc_;
KSPGetPC(ksp_, &pc_); KSPGetPC(ksp_, &pc_);
PCSetType(pc_, pcType);
PCSetFromOptions(pc_); PCSetFromOptions(pc_);
} }
} }
......
...@@ -141,21 +141,23 @@ namespace AMDiS { ...@@ -141,21 +141,23 @@ namespace AMDiS {
return &runner; return &runner;
} }
void setNestedVectors(bool createNestedVector = true) void setNestedVectors(bool nested = true)
{ {
vecSol.isNested= createNestedVector; vecSol.isNested = nested;
vecRhs.isNested= createNestedVector; vecRhs.isNested = nested;
} }
void setNestedMatrix(bool nested=true) void setNestedMatrix(bool nested = true)
{ {
petscMat.isNested=nested; petscMat.isNested=nested;
} }
void setNested(bool n) void setNested(bool n)
{ {
setNestedVectors(n); setNestedMatrix(n); setNestedVectors(n);
setNestedMatrix(n);
} }
protected: protected:
Runner runner; Runner runner;
...@@ -170,10 +172,7 @@ namespace AMDiS { ...@@ -170,10 +172,7 @@ namespace AMDiS {
// transfer matrix and rhs-vector to PETSc data-structures // transfer matrix and rhs-vector to PETSc data-structures
if (createMatrixData) { if (createMatrixData) {
petscMat << A; petscMat << A;
if (x.getSize() == 1) runner.init(A, petscMat.matrix);
runner.init(A, petscMat.nestMat[0]);
else
runner.init(A, petscMat.matrix);
} }
vecSol << x; vecSol << x;
...@@ -182,10 +181,7 @@ namespace AMDiS { ...@@ -182,10 +181,7 @@ namespace AMDiS {
// solve the linear system using PETSc solvers // solve the linear system using PETSc solvers
t.reset(); t.reset();
if (vecSol.nestVec.size() == 1) error = runner.solve(petscMat.matrix, vecSol.vector, vecRhs.vector);
error = runner.solve(petscMat.nestMat[0], vecSol.nestVec[0], vecRhs.nestVec[0]);
else
error = runner.solve(petscMat.matrix, vecSol.vector, vecRhs.vector);
// transfer solution from PETSc vector to SystemVector // transfer solution from PETSc vector to SystemVector