Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

PetscSolverFeti.h 10.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

#include "parallel/PetscSolver.h"
24
#include "parallel/ParallelTypes.h"
25 26 27 28 29 30 31 32 33 34

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;


#ifdef HAVE_PETSC_DEV
35

Thomas Witkowski's avatar
Thomas Witkowski committed
36 37
  class PetscSolverFeti;

38 39 40 41
  /** \brief
   * This structure is used when defining the MatShell operation for solving 
   * primal schur complement. \ref petscMultMatSchurPrimal
   */
42
  struct SchurPrimalData {
43 44 45 46 47
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
48

49
    /// Temporal vector on the B variables.
50 51
    Vec tmp_vec_b;

52
    /// Temporal vecor in the primal variables.
53 54
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
55
    PetscSolverFeti *fetiSolver;
56 57 58
  };


59 60 61 62 63
  /** \brief
   * This structure is used when defining the FETI-DP operator for solving 
   * the system matrix reduced to the Lagrange multipliers.
   * \ref petscMultMatFeti
   */
64
  struct FetiData {
65 66
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
67

68
    /// Matrix of Lagrange variables.
69 70
    Mat *mat_lagrange;

71 72
    /// Temporal vectors on the B variables.
    Vec tmp_vec_b0, tmp_vec_b1;
73

74
    /// Temporal vector on the primal variables.
75 76
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
77
    PetscSolverFeti *fetiSolver;
78

79
    /// Pointer to the solver of the schur complement on the primal variables.
80 81 82 83
    KSP *ksp_schur_primal;
  };


84
  struct FetiDirichletPreconData {
85 86 87
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

88
    Mat *mat_interior_interior, *mat_duals_duals, *mat_interior_duals, *mat_duals_interior;
89 90 91 92 93 94 95

    /// Pointer to the solver for \ref PetscSolverFeti::mat_bb.
    KSP *ksp_interior;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

96 97
    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
98
    
99 100 101
    /// Temporal vector on the interior variables.
    Vec tmp_vec_interior;    
  };
102

103

104 105 106 107 108 109 110 111 112 113 114
  struct FetiLumpedPreconData {
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

    Mat *mat_duals_duals;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
115 116
  };

117

118 119 120 121 122 123 124
  typedef enum {
    FETI_NONE = 0,
    FETI_DIRICHLET = 1,
    FETI_LUMPED = 2
  } FetiPreconditioner;


Thomas Witkowski's avatar
Thomas Witkowski committed
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
  struct FetiStatisticsData
  {
    /// Number of application of the FETI-DP operator.
    int nFetiApply;

    /// Time for solving the reduced FETI system.
    double timeFetiApply;

    /// Number of application of the Schur primal operator.
    int nSchurPrimalApply;

    /// Time for appling the Schur primal operator.
    double timeSchurPrimalApply;

    /// Number of solution of the Schur primal system.
    int nSchurPrimalSolve;

    /// Time for solving the Schur primal system.
    double timeSchurPrimalSolve;

    /// Number of solution of the local subdomain problems.
    int nLocalSolve;

    /// Time for solving the local subdomain problems.
    double timeLocalSolve;
  };


153 154 155
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
156 157 158
  class PetscSolverFeti : public PetscSolver
  {
  public:
159
    PetscSolverFeti();
160

161 162 163 164 165 166
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
167

168
    /// Solve the system using FETI-DP method.
169
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
170

171 172 173
    /// Destroys all matrix data structures.
    void destroyMatrixData();

174 175
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
176 177 178 179 180 181 182
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
183

Thomas Witkowski's avatar
Thomas Witkowski committed
184 185
    void solveLocalProblem(Vec &rhs, Vec &sol);

186
  protected:
187 188 189
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all matrix and vector objects with the approriated
    /// sizes.
190
    void updateDofData();
191

192 193
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
194 195
    void createPrimals();

196 197
    /// Defines the set of dual variables and creates the global index of
    // dual variables.
198 199
    void createDuals();

200 201
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
202 203
    void createLagrange();

204
    /// Creates a global index of the B variables.
205 206
    void createIndexB();

207 208 209
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
    void createMatLagrange();
210

211 212 213
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
    void createSchurPrimalKsp();
214

215
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
216 217
    void destroySchurPrimalKsp();

218
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
219 220
    void createFetiKsp();

221
    /// Destroys FETI-DP operator, \ref ksp_feti
222 223
    void destroyFetiKsp();

224 225 226 227 228 229 230 231 232 233 234 235 236 237
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
238 239 240 241
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

242 243 244 245 246 247 248 249
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
250 251 252 253
    void resetStatistics();

    void printStatistics();

254
  protected:
255 256 257 258
    /// Number of components in the PDE to be solved.
    int nComponents;

    /// Set of DOF indices that are considered to be primal variables.
259
    DofIndexSet primals;
260

261
    /// Mapping from primal DOF indices to a global index of primals.
262
    DofMapping globalPrimalIndex;
263

264 265
    /// Number of rank owned primals and global primals
    int nRankPrimals, nOverallPrimals, rStartPrimals;
266

267
    /// Set of DOF indices that are considered to be dual variables.
268 269
    DofIndexSet duals;
    
270
    /// Mapping from dual DOF indices to a global index of duals.
271 272
    DofMapping globalDualIndex;

273 274
    /// Stores to each dual boundary DOF the set of ranks in which the DOF
    /// is contained in.
275 276
    DofIndexToPartitions boundaryDofRanks;

277 278
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
279 280
    DofMapping dofFirstLagrange;

281 282 283
    /// Number of rank owned Lagrange variables, number of global
    /// Lagrange variables.
    int nRankLagrange, nOverallLagrange, rStartLagrange;
284
    
285 286
    /// Index for each non primal variables to the global index of
    /// B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
287
    DofMapping globalIndexB, localIndexB;
288

289 290
    /// Number of non primal, thus B, variables on rank and globally.
    int nRankB, nOverallB, rStartB;    
291

292
    /// Global PETSc matrix of non primal variables.
293 294
    Mat mat_b_b;

295
    /// Global PETSc matrix of primal variables.
296 297
    Mat mat_primal_primal;

298 299
    /// Global PETSc matrices that connect the primal with the non 
    /// primal variables.
300 301
    Mat mat_b_primal, mat_primal_b;

302
    /// Global PETSc matrix of Lagrange variables.
303 304
    Mat mat_lagrange;

305
    /// Right hand side PETSc vectors for primal and non primal variables.
306 307
    Vec f_b, f_primal;

308 309
    /// PETSc solver object that inverts the matrix of non primal 
    /// variables, \ref mat_b_b
310 311
    KSP ksp_b;

Thomas Witkowski's avatar
Thomas Witkowski committed
312 313 314 315 316
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

317 318
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
319 320
    KSP ksp_schur_primal;

321 322
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
323 324
    Mat mat_schur_primal;

325
    /// Data for MatMult operation in matrix \ref mat_schur_primal
326
    SchurPrimalData schurPrimalData;
327

328
    /// PETSc solver object to solve a system with FETI-DP.    
329 330
    KSP ksp_feti;

331 332
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
333
    Mat mat_feti;
334

335
    /// Data for MatMult operation in matrix \ref mat_feti
336
    FetiData fetiData;
337

338 339 340 341 342 343
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
344 345 346

    Mat mat_lagrange_scaled;
   
347
    FetiDirichletPreconData fetiDirichletPreconData;
348

349
    FetiLumpedPreconData fetiLumpedPreconData;
350

351
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
352 353 354 355 356

    KSP ksp_interior;

    int nLocalInterior;
    
357 358
    // Number of local nodes that are duals.
    int nLocalDuals;
Thomas Witkowski's avatar
Thomas Witkowski committed
359 360 361

  public:
    static FetiStatisticsData fetiStatistics;
362 363 364 365 366 367
  };
#endif

}

#endif