Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 0e885e23 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Rosenbrock now should work also in parallel, tested only for FETI-DP solver.

parent 02638872
......@@ -64,7 +64,7 @@ namespace AMDiS {
virtual void solve(AdaptInfo* adaptInfo) {}
virtual void solve(AdaptInfo *adaptInfo, bool fixedMatrix)
virtual void solve(AdaptInfo *adaptInfo, bool, bool)
{
solve(adaptInfo);
}
......
......@@ -540,7 +540,7 @@ namespace AMDiS {
}
void ProblemStatSeq::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
void ProblemStatSeq::solve(AdaptInfo *adaptInfo, bool, bool)
{
FUNCNAME("Problem::solve()");
......
......@@ -67,7 +67,7 @@ namespace AMDiS {
public:
/// Constructor
ProblemStatSeq(string nameStr,
ProblemIterationInterface *problemIteration = NULL)
ProblemIterationInterface *problemIteration = NULL)
: StandardProblemIteration(this),
name(nameStr),
nComponents(-1),
......@@ -147,7 +147,9 @@ namespace AMDiS {
* Implementation of ProblemStatBase::solve(). Deligates the solving
* of problems system to \ref solver.
*/
virtual void solve(AdaptInfo *adaptInfo, bool fixedMatrix = false);
void solve(AdaptInfo *adaptInfo,
bool createMatrixData = true,
bool storeMatrixData = false);
/** \brief
* Implementation of ProblemStatBase::estimate(). Deligates the estimation
......
......@@ -104,8 +104,27 @@ namespace AMDiS {
/// Coarsening of the mesh.
virtual Flag coarsenMesh(AdaptInfo *adaptInfo) = 0;
/// Solves the assembled system. The result is an approximative solution.
virtual void solve(AdaptInfo *adaptInfo, bool fixedMatrix) = 0;
/** \brief
* Solves the assembled system. The result is an approximative solution.
* The last two boolean arguments can be used to controll successive
* solutions of systems with the same matrix.
*
* \param adaptInfo Pointer to an \ref AdaptInfo object.
* \param createMatrixData If false, the solver assumes that all of its
* internal data structures for the system
* matrix are already created. This is the case,
* if we solve different systems but with the
* same matrix. After the first call to this
* function (with this parameter set to true),
* all other calls may set it to false.
* \param storeMatrixData If true, all internal data structures for the
* system matrix are not deleted such that they
* can be used for next solutions with the same
* system matrix.
*/
virtual void solve(AdaptInfo *adaptInfo,
bool createMatrixData = true,
bool storeMatrixData = false) = 0;
/** \brief
* A posteriori error estimation of the calculated solution. Should store
......
......@@ -37,10 +37,10 @@ namespace AMDiS {
Flag flag = buildAndAdapt(adaptInfo, toDo);
if (toDo.isSet(SOLVE))
problem->solve(adaptInfo, false);
problem->solve(adaptInfo, true, false);
if (toDo.isSet(SOLVE_RHS))
problem->solve(adaptInfo, true);
problem->solve(adaptInfo, true, false);
if (toDo.isSet(ESTIMATE))
problem->estimate(adaptInfo);
......
......@@ -61,7 +61,9 @@ namespace AMDiS {
}
void PetscProblemStat::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
void PetscProblemStat::solve(AdaptInfo *adaptInfo,
bool createMatrixData,
bool storeMatrixData)
{
FUNCNAME("PetscProblemStat::solve()");
......@@ -79,8 +81,13 @@ namespace AMDiS {
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
#endif
petscSolver->setMeshDistributor(meshDistributor);
petscSolver->fillPetscMatrix(systemMatrix, rhs);
if (createMatrixData) {
petscSolver->setMeshDistributor(meshDistributor);
petscSolver->fillPetscMatrix(systemMatrix);
}
petscSolver->fillPetscRhs(rhs);
#if 0
processMemUsage(vm, rss);
......@@ -93,6 +100,9 @@ namespace AMDiS {
petscSolver->solvePetscMatrix(*solution, adaptInfo);
if (!storeMatrixData)
petscSolver->destroyMatrixData();
#if 0
processMemUsage(vm, rss);
MSG("STAGE 3\n");
......@@ -102,7 +112,6 @@ namespace AMDiS {
MSG("Overall memory usage is VM = %.1f MB RSS = %.1f MB\n", vm, rss);
#endif
INFO(info, 8)("solution of discrete system needed %.5f seconds\n",
MPI::Wtime() - wtime);
}
......
......@@ -48,7 +48,9 @@ namespace AMDiS {
ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING);
void solve(AdaptInfo *adaptInfo, bool fixedMatrix = false);
void solve(AdaptInfo *adaptInfo,
bool createMatrixData = true,
bool storeMatrixData = false);
protected:
PetscSolver *petscSolver;
......
......@@ -60,18 +60,26 @@ namespace AMDiS {
}
/** \brief
* Create a PETSc matrix and PETSc vectors. The given DOF matrices are used to
* create the nnz structure of the PETSc matrix and the values are transfered to it.
* The given DOF vectors are used to the the values of the PETSc rhs vector.
* Create a PETSc matrix. The given DOF matrices are used to create the nnz
* structure of the PETSc matrix and the values are transfered to it.
*
* \param[in] mat
*/
virtual void fillPetscMatrix(Matrix<DOFMatrix*> *mat) = 0;
/** \brief
* Create a PETSc vector and fills it with the rhs values of the system.
*
* \param[in] vec
*/
virtual void fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec) = 0;
virtual void fillPetscRhs(SystemVector *vec) = 0;
/// Use PETSc to solve the linear system of equations
virtual void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo) = 0;
/// Destroys all matrix data structures.
virtual void destroyMatrixData() = 0;
virtual Flag getBoundaryDofRequirement()
{
return 0;
......
......@@ -595,8 +595,13 @@ namespace AMDiS {
schurPrimalData.mat_b_primal = &mat_b_primal;
schurPrimalData.ksp_b = &ksp_b;
VecDuplicate(f_b, &(schurPrimalData.tmp_vec_b));
VecDuplicate(f_primal, &(schurPrimalData.tmp_vec_primal));
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents,
&(schurPrimalData.tmp_vec_b));
VecCreateMPI(PETSC_COMM_WORLD,
nRankPrimals * nComponents, nOverallPrimals * nComponents,
&(schurPrimalData.tmp_vec_primal));
MatCreateShell(PETSC_COMM_WORLD,
nRankPrimals * nComponents, nRankPrimals * nComponents,
......@@ -643,9 +648,15 @@ namespace AMDiS {
fetiData.ksp_b = &ksp_b;
fetiData.ksp_schur_primal = &ksp_schur_primal;
VecDuplicate(f_b, &(fetiData.tmp_vec_b0));
VecDuplicate(f_b, &(fetiData.tmp_vec_b1));
VecDuplicate(f_primal, &(fetiData.tmp_vec_primal));
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents,
&(fetiData.tmp_vec_b0));
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents,
&(fetiData.tmp_vec_b1));
VecCreateMPI(PETSC_COMM_WORLD,
nRankPrimals * nComponents, nOverallPrimals * nComponents,
&(fetiData.tmp_vec_primal));
MatCreateShell(PETSC_COMM_WORLD,
nRankLagrange * nComponents, nRankLagrange * nComponents,
......@@ -681,7 +692,9 @@ namespace AMDiS {
fetiDirichletPreconData.mat_duals_interior = &mat_duals_interior;
fetiDirichletPreconData.ksp_interior = &ksp_interior;
VecDuplicate(f_b, &(fetiDirichletPreconData.tmp_vec_b));
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents,
&(fetiDirichletPreconData.tmp_vec_b));
MatGetVecs(mat_duals_duals, PETSC_NULL, &(fetiDirichletPreconData.tmp_vec_duals0));
MatGetVecs(mat_duals_duals, PETSC_NULL, &(fetiDirichletPreconData.tmp_vec_duals1));
MatGetVecs(mat_interior_interior, PETSC_NULL, &(fetiDirichletPreconData.tmp_vec_interior));
......@@ -697,7 +710,9 @@ namespace AMDiS {
fetiLumpedPreconData.mat_lagrange_scaled = &mat_lagrange_scaled;
fetiLumpedPreconData.mat_duals_duals = &mat_duals_duals;
VecDuplicate(f_b, &(fetiLumpedPreconData.tmp_vec_b));
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents,
&(fetiLumpedPreconData.tmp_vec_b));
MatGetVecs(mat_duals_duals, PETSC_NULL, &(fetiLumpedPreconData.tmp_vec_duals0));
MatGetVecs(mat_duals_duals, PETSC_NULL, &(fetiLumpedPreconData.tmp_vec_duals1));
......@@ -851,12 +866,11 @@ namespace AMDiS {
}
void PetscSolverFeti::fillPetscMatrix(Matrix<DOFMatrix*> *mat,
SystemVector *vec)
void PetscSolverFeti::fillPetscMatrix(Matrix<DOFMatrix*> *mat)
{
FUNCNAME("PetscSolverFeti::fillPetscMatrix()");
nComponents = vec->getSize();
nComponents = mat->getSize();
// === Create all sets and indices. ===
......@@ -1156,16 +1170,40 @@ namespace AMDiS {
}
// === Create and fill PETSc's right hand side vectors. ===
// === Create and fill PETSc matrix for Lagrange constraints. ===
createMatLagrange();
// === Create PETSc solver for the Schur complement on primal variables. ===
createSchurPrimalKsp();
// === Create PETSc solver for the FETI-DP operator. ===
createFetiKsp();
// === Create solver for the non primal (thus local) variables. ===
KSPCreate(PETSC_COMM_WORLD, &ksp_b);
KSPSetOperators(ksp_b, mat_b_b, mat_b_b, SAME_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_b, "solver_b_");
KSPSetFromOptions(ksp_b);
}
VecCreate(PETSC_COMM_WORLD, &f_b);
VecSetSizes(f_b, nRankB * nComponents, nOverallB * nComponents);
VecSetType(f_b, VECMPI);
void PetscSolverFeti::fillPetscRhs(SystemVector *vec)
{
FUNCNAME("PetscSolverFeti::fillPetscRhs()");
int nComponents = vec->getSize();
VecCreate(PETSC_COMM_WORLD, &f_primal);
VecSetSizes(f_primal, nRankPrimals * nComponents,
nOverallPrimals * nComponents);
VecSetType(f_primal, VECMPI);
VecCreateMPI(PETSC_COMM_WORLD,
nRankB * nComponents, nOverallB * nComponents, &f_b);
VecCreateMPI(PETSC_COMM_WORLD,
nRankPrimals * nComponents,
nOverallPrimals * nComponents, &f_primal);
for (int i = 0; i < nComponents; i++) {
DOFVector<double>::Iterator dofIt(vec->getDOFVector(i), USED_DOFS);
......@@ -1194,21 +1232,6 @@ namespace AMDiS {
VecAssemblyBegin(f_primal);
VecAssemblyEnd(f_primal);
// === Create and fill PETSc matrix for Lagrange constraints. ===
createMatLagrange();
// === Create PETSc solver for the Schur complement on primal variables. ===
createSchurPrimalKsp();
// === Create PETSc solver for the FETI-DP operator. ===
createFetiKsp();
}
......@@ -1216,11 +1239,9 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverFeti::solveFetiMatrix()");
// Create transpose of Lagrange matrix.
Mat mat_lagrange_transpose;
Mat mat_lagrange_transpose;
MatTranspose(mat_lagrange, MAT_INITIAL_MATRIX, &mat_lagrange_transpose);
// === Create nested matrix which will contain the overall FETI system. ===
Mat A;
......@@ -1261,12 +1282,8 @@ namespace AMDiS {
// === Create rhs and solution vectors for the overall FETI system. ===
Vec f;
VecCreate(PETSC_COMM_WORLD, &f);
VecSetSizes(f, nRankNest, nOverallNest);
VecSetType(f, VECMPI);
Vec b;
Vec f, b;
VecCreateMPI(PETSC_COMM_WORLD, nRankNest, nOverallNest, &f);
VecDuplicate(f, &b);
......@@ -1352,14 +1369,7 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverFeti::solveReducedFetiMatrix()");
// === Create solver for the non primal (thus local) variables. ===
KSPCreate(PETSC_COMM_WORLD, &ksp_b);
KSPSetOperators(ksp_b, mat_b_b, mat_b_b, SAME_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_b, "solver_b_");
KSPSetFromOptions(ksp_b);
// RHS and solution vector.
// RHS vector.
Vec vec_rhs;
// Some temporary vectors.
......@@ -1444,8 +1454,14 @@ namespace AMDiS {
VecDestroy(&tmp_primal0);
VecDestroy(&tmp_primal1);
VecDestroy(&f_b);
VecDestroy(&f_primal);
}
KSPDestroy(&ksp_b);
void PetscSolverFeti::destroyMatrixData()
{
FUNCNAME("PetscSolverFeti::destroyMatrixData()");
MatDestroy(&mat_b_b);
MatDestroy(&mat_primal_primal);
......@@ -1453,14 +1469,6 @@ namespace AMDiS {
MatDestroy(&mat_primal_b);
MatDestroy(&mat_lagrange);
VecDestroy(&f_b);
VecDestroy(&f_primal);
destroySchurPrimalKsp();
destroyFetiKsp();
// === Destroy preconditioner data structures. ===
if (fetiPreconditioner != FETI_NONE)
......@@ -1471,8 +1479,13 @@ namespace AMDiS {
MatDestroy(&mat_interior_duals);
MatDestroy(&mat_duals_interior);
}
}
destroySchurPrimalKsp();
destroyFetiKsp();
KSPDestroy(&ksp_b);
}
void PetscSolverFeti::solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo)
{
......@@ -1490,6 +1503,7 @@ namespace AMDiS {
}
}
#endif
}
......@@ -133,13 +133,19 @@ namespace AMDiS {
public:
PetscSolverFeti();
/// Assemble the sequentially created matrices and vectors to the
/// global matrices and vectors required by the FETI-DP method.
void fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec);
/// Assemble the sequentially created matrices to the global matrices
/// required by the FETI-DP method.
void fillPetscMatrix(Matrix<DOFMatrix*> *mat);
/// Assembles the global rhs vectors from the sequentially created ones.
void fillPetscRhs(SystemVector *vec);
/// Solve the system using FETI-DP method.
void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
/// Destroys all matrix data structures.
void destroyMatrixData();
/// Returns flags to denote which information of the boundary DOFs are
/// required by the FETI-DP solver.
Flag getBoundaryDofRequirement()
......
......@@ -25,36 +25,28 @@ namespace AMDiS {
}
void PetscSolverGlobalMatrix::fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec)
void PetscSolverGlobalMatrix::fillPetscMatrix(Matrix<DOFMatrix*> *mat)
{
FUNCNAME("PetscSolverGlobalMatrix::fillPetscMatrix()");
TEST_EXIT_DBG(meshDistributor)("No mesh distributor object defined!\n");
TEST_EXIT_DBG(mat)("No DOF matrix defined!\n");
TEST_EXIT_DBG(vec)("NO DOF vector defined!\n");
double wtime = MPI::Wtime();
int nComponents = mat->getNumRows();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
// === Create PETSc vector (rhs, solution and a temporary vector). ===
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
// === Create PETSc vector (solution and a temporary vector). ===
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscSolVec);
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscTmpVec);
int recvAllValues = 0;
int sendValue = static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
meshDistributor->getMpiComm().Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
int sendValue =
static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
meshDistributor->getMpiComm().Allreduce(&sendValue, &recvAllValues,
1, MPI_INT, MPI_SUM);
if (!d_nnz || recvAllValues != 0) {
if (d_nnz) {
......@@ -102,10 +94,21 @@ namespace AMDiS {
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
#if (DEBUG != 0)
MSG("Fill petsc matrix 3 needed %.5f seconds\n",
TIME_USED(MPI::Wtime(), wtime));
#endif
MSG("Fill petsc matrix needed %.5f seconds\n", MPI::Wtime() - wtime);
}
void PetscSolverGlobalMatrix::fillPetscRhs(SystemVector *vec)
{
FUNCNAME("PetscSolverGlobalMatrix::fillPetscRhs()");
TEST_EXIT_DBG(vec)("NO DOF vector defined!\n");
int nComponents = vec->getSize();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscRhsVec);
// === Transfer values from DOF vector to the PETSc vector. ===
......@@ -114,12 +117,11 @@ namespace AMDiS {
VecAssemblyBegin(petscRhsVec);
VecAssemblyEnd(petscRhsVec);
MSG("Fill petsc matrix needed %.5f seconds\n", MPI::Wtime() - wtime);
}
void PetscSolverGlobalMatrix::solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo)
void PetscSolverGlobalMatrix::solvePetscMatrix(SystemVector &vec,
AdaptInfo *adaptInfo)
{
FUNCNAME("PetscSolverGlobalMatrix::solvePetscMatrix()");
......
......@@ -44,10 +44,15 @@ namespace AMDiS {
Parameters::get("parallel->use zero start vector", zeroStartVector);
}
void fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec);
void fillPetscMatrix(Matrix<DOFMatrix*> *mat);
void fillPetscRhs(SystemVector *vec);
void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
void destroyMatrixData()
{}
protected:
/// Creates a new non zero pattern structure for the PETSc matrix.
void createPetscNnzStructure(Matrix<DOFMatrix*> *mat);
......
......@@ -178,7 +178,7 @@ namespace AMDiS {
}
void PetscSolverSchur::fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec)
void PetscSolverSchur::fillPetscMatrix(Matrix<DOFMatrix*> *mat)
{
FUNCNAME("PetscSolverSchur::fillPetscMatrix()");
......@@ -248,18 +248,20 @@ namespace AMDiS {
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscSolVec);
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscTmpVec);
}
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
void PetscSolverSchur::fillPetscRhs(SystemVector *vec)
{
FUNCNAME("PetscSolverSchur::fillPetscRhs()");
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
int nComponents = vec->getSize();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscRhsVec);
for (int i = 0; i < nComponents; i++)
setDofVector(petscRhsVec, vec->getDOFVector(i), nComponents, i);
......@@ -269,7 +271,8 @@ namespace AMDiS {
}
void PetscSolverSchur::solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo)
void PetscSolverSchur::solvePetscMatrix(SystemVector &vec,
AdaptInfo *adaptInfo)
{
FUNCNAME("PetscSolverSchur::solvePetscMatrix()");
......
......@@ -38,10 +38,15 @@ namespace AMDiS {
: PetscSolver()
{}
void fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec);
void fillPetscMatrix(Matrix<DOFMatrix*> *mat);
void fillPetscRhs(SystemVector *vec);
void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
void destroyMatrixData()
{}
Flag getBoundaryDofRequirement()
{
return
......
......@@ -53,15 +53,15 @@ namespace AMDiS {
FUNCNAME("RosenbrockAdaptInstationary::RosenbrockAdaptInstationary()");
initConstructor(&problemStat);
}
void RosenbrockAdaptInstationary::initConstructor(
RosenbrockStationary *problemStat)