PetscSolverFeti.h 10.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

#include "parallel/PetscSolver.h"
24
#include "parallel/ParallelTypes.h"
25
26
27
28
29
30
31
32
33
34

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;


#ifdef HAVE_PETSC_DEV
35

Thomas Witkowski's avatar
Thomas Witkowski committed
36
37
  class PetscSolverFeti;

38
39
40
41
  /** \brief
   * This structure is used when defining the MatShell operation for solving 
   * primal schur complement. \ref petscMultMatSchurPrimal
   */
42
  struct SchurPrimalData {
43
44
45
46
47
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
48

49
    /// Temporal vector on the B variables.
50
51
    Vec tmp_vec_b;

52
    /// Temporal vecor in the primal variables.
53
54
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
55
    PetscSolverFeti *fetiSolver;
56
57
58
  };


59
60
61
62
63
  /** \brief
   * This structure is used when defining the FETI-DP operator for solving 
   * the system matrix reduced to the Lagrange multipliers.
   * \ref petscMultMatFeti
   */
64
  struct FetiData {
65
66
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
67

68
    /// Matrix of Lagrange variables.
69
70
    Mat *mat_lagrange;

71
    /// Temporal vectors on the B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
72
    Vec tmp_vec_b;
73

74
    /// Temporal vector on the primal variables.
75
76
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
77
78
79
    /// Temporal vector on the lagrange variables.
    Vec tmp_vec_lagrange;

Thomas Witkowski's avatar
Thomas Witkowski committed
80
    PetscSolverFeti *fetiSolver;
81

82
    /// Pointer to the solver of the schur complement on the primal variables.
83
84
85
86
    KSP *ksp_schur_primal;
  };


87
  struct FetiDirichletPreconData {
88
89
90
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

91
    Mat *mat_interior_interior, *mat_duals_duals, *mat_interior_duals, *mat_duals_interior;
92
93
94
95
96
97
98

    /// Pointer to the solver for \ref PetscSolverFeti::mat_bb.
    KSP *ksp_interior;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

99
100
    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
101
    
102
103
104
    /// Temporal vector on the interior variables.
    Vec tmp_vec_interior;    
  };
105

106

107
108
109
110
111
112
113
114
115
116
117
  struct FetiLumpedPreconData {
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

    Mat *mat_duals_duals;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
118
119
  };

120

121
122
123
124
125
126
127
  typedef enum {
    FETI_NONE = 0,
    FETI_DIRICHLET = 1,
    FETI_LUMPED = 2
  } FetiPreconditioner;


128
/** \brief
129
130
   * FETI-DP implementation based on PETSc.
   */
131
132
133
  class PetscSolverFeti : public PetscSolver
  {
  public:
134
    PetscSolverFeti();
135

136
137
138
139
140
141
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
142

143
    /// Solve the system using FETI-DP method.
144
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
145

146
147
148
    /// Destroys all matrix data structures.
    void destroyMatrixData();

149
150
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
151
152
153
154
155
156
157
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
158

Thomas Witkowski's avatar
Thomas Witkowski committed
159
160
    void solveLocalProblem(Vec &rhs, Vec &sol);

161
  protected:
162
163
164
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all matrix and vector objects with the approriated
    /// sizes.
165
    void updateDofData();
166

167
168
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
169
170
    void createPrimals();

171
172
    /// Defines the set of dual variables and creates the global index of
    // dual variables.
173
174
    void createDuals();

175
176
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
177
178
    void createLagrange();

179
    /// Creates a global index of the B variables.
180
181
    void createIndexB();

182
183
184
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
    void createMatLagrange();
185

186
187
188
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
    void createSchurPrimalKsp();
189

190
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
191
192
    void destroySchurPrimalKsp();

193
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
194
195
    void createFetiKsp();

196
    /// Destroys FETI-DP operator, \ref ksp_feti
197
198
    void destroyFetiKsp();

199
200
201
202
203
204
205
206
207
208
209
210
211
212
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
213
214
215
216
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
217
218
219
220
221
222
223
224
225
    /** \brief
     * Solves the FETI-DP system globally, thus without reducing it to the 
     * Lagrange multipliers. This should be used for debugging only to test
     * if the FETI-DP system is setup correctly.
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveFetiMatrix(SystemVector &vec);

226
227
228
229
230
231
232
233
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
234
235
236
237
    void resetStatistics();

    void printStatistics();

238
  protected:
239
240
241
242
    /// Number of components in the PDE to be solved.
    int nComponents;

    /// Mapping from primal DOF indices to a global index of primals.
243
    DofMapping globalPrimalIndex;
244

245
246
    /// Number of rank owned primals and global primals
    int nRankPrimals, nOverallPrimals, rStartPrimals;
247

248
    /// Set of DOF indices that are considered to be dual variables.
249
250
    DofIndexSet duals;
    
251
    /// Mapping from dual DOF indices to a global index of duals.
252
253
    DofMapping globalDualIndex;

254
255
    /// Stores to each dual boundary DOF the set of ranks in which the DOF
    /// is contained in.
256
257
    DofIndexToPartitions boundaryDofRanks;

258
259
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
260
261
    DofMapping dofFirstLagrange;

262
263
264
    /// Number of rank owned Lagrange variables, number of global
    /// Lagrange variables.
    int nRankLagrange, nOverallLagrange, rStartLagrange;
265
    
266
267
    /// Index for each non primal variables to the global index of
    /// B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
268
    DofMapping globalIndexB, localIndexB;
269

270
271
    /// Number of non primal, thus B, variables on rank and globally.
    int nRankB, nOverallB, rStartB;    
272

273
    /// Global PETSc matrix of non primal variables.
274
275
    Mat mat_b_b;

276
    /// Global PETSc matrix of primal variables.
277
278
    Mat mat_primal_primal;

279
280
    /// Global PETSc matrices that connect the primal with the non 
    /// primal variables.
281
282
    Mat mat_b_primal, mat_primal_b;

283
    /// Global PETSc matrix of Lagrange variables.
284
285
    Mat mat_lagrange;

286
    /// Right hand side PETSc vectors for primal and non primal variables.
287
288
    Vec f_b, f_primal;

289
290
    /// PETSc solver object that inverts the matrix of non primal 
    /// variables, \ref mat_b_b
291
292
    KSP ksp_b;

Thomas Witkowski's avatar
Thomas Witkowski committed
293
294
295
296
297
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

298
299
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
300
301
    KSP ksp_schur_primal;

302
303
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
304
305
    Mat mat_schur_primal;

306
    /// Data for MatMult operation in matrix \ref mat_schur_primal
307
    SchurPrimalData schurPrimalData;
308

309
    /// PETSc solver object to solve a system with FETI-DP.    
310
311
    KSP ksp_feti;

312
313
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
314
    Mat mat_feti;
315

316
    /// Data for MatMult operation in matrix \ref mat_feti
317
    FetiData fetiData;
318

319
320
321
322
323
324
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
325
326
327

    Mat mat_lagrange_scaled;
   
328
    FetiDirichletPreconData fetiDirichletPreconData;
329

330
    FetiLumpedPreconData fetiLumpedPreconData;
331

332
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
333
334
335
336
337

    KSP ksp_interior;

    int nLocalInterior;
    
338
339
    // Number of local nodes that are duals.
    int nLocalDuals;
340
341
342
343
344
345
  };
#endif

}

#endif