PetscSolverFeti.h 10.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

#include "parallel/PetscSolver.h"
24
#include "parallel/ParallelTypes.h"
25
26
27
28
29
30
31
32
33
34

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;


#ifdef HAVE_PETSC_DEV
35

Thomas Witkowski's avatar
Thomas Witkowski committed
36
37
  class PetscSolverFeti;

38
39
40
41
  /** \brief
   * This structure is used when defining the MatShell operation for solving 
   * primal schur complement. \ref petscMultMatSchurPrimal
   */
42
  struct SchurPrimalData {
43
44
45
46
47
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
48

49
    /// Temporal vector on the B variables.
50
51
    Vec tmp_vec_b;

52
    /// Temporal vecor in the primal variables.
53
54
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
55
    PetscSolverFeti *fetiSolver;
56
57
58
  };


59
60
61
62
63
  /** \brief
   * This structure is used when defining the FETI-DP operator for solving 
   * the system matrix reduced to the Lagrange multipliers.
   * \ref petscMultMatFeti
   */
64
  struct FetiData {
65
66
67
68
69
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
70

71
    /// Matrix of Lagrange variables.
72
73
    Mat *mat_lagrange;

74
75
    /// Temporal vectors on the B variables.
    Vec tmp_vec_b0, tmp_vec_b1;
76

77
    /// Temporal vector on the primal variables.
78
79
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
80
    PetscSolverFeti *fetiSolver;
81

82
    /// Pointer to the solver of the schur complement on the primal variables.
83
84
85
86
    KSP *ksp_schur_primal;
  };


87
  struct FetiDirichletPreconData {
88
89
90
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

91
    Mat *mat_interior_interior, *mat_duals_duals, *mat_interior_duals, *mat_duals_interior;
92
93
94
95
96
97
98

    /// Pointer to the solver for \ref PetscSolverFeti::mat_bb.
    KSP *ksp_interior;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

99
100
    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
101
    
102
103
104
    /// Temporal vector on the interior variables.
    Vec tmp_vec_interior;    
  };
105

106

107
108
109
110
111
112
113
114
115
116
117
  struct FetiLumpedPreconData {
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

    Mat *mat_duals_duals;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
118
119
  };

120

121
122
123
124
125
126
127
  typedef enum {
    FETI_NONE = 0,
    FETI_DIRICHLET = 1,
    FETI_LUMPED = 2
  } FetiPreconditioner;


Thomas Witkowski's avatar
Thomas Witkowski committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
  struct FetiStatisticsData
  {
    /// Number of application of the FETI-DP operator.
    int nFetiApply;

    /// Time for solving the reduced FETI system.
    double timeFetiApply;

    /// Number of application of the Schur primal operator.
    int nSchurPrimalApply;

    /// Time for appling the Schur primal operator.
    double timeSchurPrimalApply;

    /// Number of solution of the Schur primal system.
    int nSchurPrimalSolve;

    /// Time for solving the Schur primal system.
    double timeSchurPrimalSolve;

    /// Number of solution of the local subdomain problems.
    int nLocalSolve;

    /// Time for solving the local subdomain problems.
    double timeLocalSolve;
  };


156
157
158
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
159
160
161
  class PetscSolverFeti : public PetscSolver
  {
  public:
162
    PetscSolverFeti();
163

164
165
166
167
168
169
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
170

171
    /// Solve the system using FETI-DP method.
172
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
173

174
175
176
    /// Destroys all matrix data structures.
    void destroyMatrixData();

177
178
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
179
180
181
182
183
184
185
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
186

Thomas Witkowski's avatar
Thomas Witkowski committed
187
188
    void solveLocalProblem(Vec &rhs, Vec &sol);

189
  protected:
190
191
192
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all matrix and vector objects with the approriated
    /// sizes.
193
    void updateDofData();
194

195
196
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
197
198
    void createPrimals();

199
200
    /// Defines the set of dual variables and creates the global index of
    // dual variables.
201
202
    void createDuals();

203
204
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
205
206
    void createLagrange();

207
    /// Creates a global index of the B variables.
208
209
    void createIndexB();

210
211
212
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
    void createMatLagrange();
213

214
215
216
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
    void createSchurPrimalKsp();
217

218
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
219
220
    void destroySchurPrimalKsp();

221
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
222
223
    void createFetiKsp();

224
    /// Destroys FETI-DP operator, \ref ksp_feti
225
226
    void destroyFetiKsp();

227
228
229
230
231
232
233
234
235
236
237
238
239
240
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
241
242
243
244
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

245
246
247
248
249
250
251
252
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
253
254
255
256
    void resetStatistics();

    void printStatistics();

257
  protected:
258
259
260
261
    /// Number of components in the PDE to be solved.
    int nComponents;

    /// Set of DOF indices that are considered to be primal variables.
262
    DofIndexSet primals;
263

264
    /// Mapping from primal DOF indices to a global index of primals.
265
    DofMapping globalPrimalIndex;
266

267
268
    /// Number of rank owned primals and global primals
    int nRankPrimals, nOverallPrimals, rStartPrimals;
269

270
    /// Set of DOF indices that are considered to be dual variables.
271
272
    DofIndexSet duals;
    
273
    /// Mapping from dual DOF indices to a global index of duals.
274
275
    DofMapping globalDualIndex;

276
277
    /// Stores to each dual boundary DOF the set of ranks in which the DOF
    /// is contained in.
278
279
    DofIndexToPartitions boundaryDofRanks;

280
281
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
282
283
    DofMapping dofFirstLagrange;

284
285
286
    /// Number of rank owned Lagrange variables, number of global
    /// Lagrange variables.
    int nRankLagrange, nOverallLagrange, rStartLagrange;
287
    
288
289
    /// Index for each non primal variables to the global index of
    /// B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
290
    DofMapping globalIndexB, localIndexB;
291

292
293
    /// Number of non primal, thus B, variables on rank and globally.
    int nRankB, nOverallB, rStartB;    
294

295
    /// Global PETSc matrix of non primal variables.
296
297
    Mat mat_b_b;

298
    /// Global PETSc matrix of primal variables.
299
300
    Mat mat_primal_primal;

301
302
    /// Global PETSc matrices that connect the primal with the non 
    /// primal variables.
303
304
    Mat mat_b_primal, mat_primal_b;

305
    /// Global PETSc matrix of Lagrange variables.
306
307
    Mat mat_lagrange;

308
    /// Right hand side PETSc vectors for primal and non primal variables.
309
310
    Vec f_b, f_primal;

311
312
    /// PETSc solver object that inverts the matrix of non primal 
    /// variables, \ref mat_b_b
313
314
    KSP ksp_b;

Thomas Witkowski's avatar
Thomas Witkowski committed
315
316
317
318
319
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

320
321
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
322
323
    KSP ksp_schur_primal;

324
325
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
326
327
    Mat mat_schur_primal;

328
    /// Data for MatMult operation in matrix \ref mat_schur_primal
329
    SchurPrimalData schurPrimalData;
330

331
    /// PETSc solver object to solve a system with FETI-DP.    
332
333
    KSP ksp_feti;

334
335
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
336
    Mat mat_feti;
337

338
    /// Data for MatMult operation in matrix \ref mat_feti
339
    FetiData fetiData;
340

341
342
343
344
345
346
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
347
348
349

    Mat mat_lagrange_scaled;
   
350
    FetiDirichletPreconData fetiDirichletPreconData;
351

352
    FetiLumpedPreconData fetiLumpedPreconData;
353

354
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
355
356
357
358
359

    KSP ksp_interior;

    int nLocalInterior;
    
360
361
    // Number of local nodes that are duals.
    int nLocalDuals;
Thomas Witkowski's avatar
Thomas Witkowski committed
362
363
364

  public:
    static FetiStatisticsData fetiStatistics;
365
366
367
368
369
370
  };
#endif

}

#endif