PetscSolverFeti.h 10.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

#include "parallel/PetscSolver.h"
24
#include "parallel/ParallelTypes.h"
25
26
27
28
29
30
31
32
33
34

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;


#ifdef HAVE_PETSC_DEV
35

Thomas Witkowski's avatar
Thomas Witkowski committed
36
37
  class PetscSolverFeti;

38
39
40
41
  /** \brief
   * This structure is used when defining the MatShell operation for solving 
   * primal schur complement. \ref petscMultMatSchurPrimal
   */
42
  struct SchurPrimalData {
43
44
45
46
47
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
48

49
    /// Temporal vector on the B variables.
50
51
    Vec tmp_vec_b;

52
    /// Temporal vecor in the primal variables.
53
54
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
55
    PetscSolverFeti *fetiSolver;
56
57
58
  };


59
60
61
62
63
  /** \brief
   * This structure is used when defining the FETI-DP operator for solving 
   * the system matrix reduced to the Lagrange multipliers.
   * \ref petscMultMatFeti
   */
64
  struct FetiData {
65
66
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
67

68
    /// Matrix of Lagrange variables.
69
70
    Mat *mat_lagrange;

71
    /// Temporal vectors on the B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
72
    Vec tmp_vec_b;
73

74
    /// Temporal vector on the primal variables.
75
76
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
77
78
79
    /// Temporal vector on the lagrange variables.
    Vec tmp_vec_lagrange;

Thomas Witkowski's avatar
Thomas Witkowski committed
80
    PetscSolverFeti *fetiSolver;
81

82
    /// Pointer to the solver of the schur complement on the primal variables.
83
84
85
86
    KSP *ksp_schur_primal;
  };


87
  struct FetiDirichletPreconData {
88
89
90
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

91
    Mat *mat_interior_interior, *mat_duals_duals, *mat_interior_duals, *mat_duals_interior;
92
93
94
95
96
97
98

    /// Pointer to the solver for \ref PetscSolverFeti::mat_bb.
    KSP *ksp_interior;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

99
100
    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
101
    
102
103
104
    /// Temporal vector on the interior variables.
    Vec tmp_vec_interior;    
  };
105

106

107
108
109
110
111
112
113
114
115
116
117
  struct FetiLumpedPreconData {
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

    Mat *mat_duals_duals;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
118
119
  };

120

121
122
123
124
125
126
127
  typedef enum {
    FETI_NONE = 0,
    FETI_DIRICHLET = 1,
    FETI_LUMPED = 2
  } FetiPreconditioner;


Thomas Witkowski's avatar
Thomas Witkowski committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
  struct FetiStatisticsData
  {
    /// Number of application of the FETI-DP operator.
    int nFetiApply;

    /// Time for solving the reduced FETI system.
    double timeFetiApply;

    /// Number of application of the Schur primal operator.
    int nSchurPrimalApply;

    /// Time for appling the Schur primal operator.
    double timeSchurPrimalApply;

    /// Number of solution of the Schur primal system.
    int nSchurPrimalSolve;

    /// Time for solving the Schur primal system.
    double timeSchurPrimalSolve;

    /// Number of solution of the local subdomain problems.
    int nLocalSolve;

    /// Time for solving the local subdomain problems.
    double timeLocalSolve;
  };


156
157
158
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
159
160
161
  class PetscSolverFeti : public PetscSolver
  {
  public:
162
    PetscSolverFeti();
163

164
165
166
167
168
169
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
170

171
    /// Solve the system using FETI-DP method.
172
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
173

174
175
176
    /// Destroys all matrix data structures.
    void destroyMatrixData();

177
178
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
179
180
181
182
183
184
185
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
186

Thomas Witkowski's avatar
Thomas Witkowski committed
187
188
    void solveLocalProblem(Vec &rhs, Vec &sol);

189
  protected:
190
191
192
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all matrix and vector objects with the approriated
    /// sizes.
193
    void updateDofData();
194

195
196
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
197
198
    void createPrimals();

199
200
    /// Defines the set of dual variables and creates the global index of
    // dual variables.
201
202
    void createDuals();

203
204
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
205
206
    void createLagrange();

207
    /// Creates a global index of the B variables.
208
209
    void createIndexB();

210
211
212
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
    void createMatLagrange();
213

214
215
216
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
    void createSchurPrimalKsp();
217

218
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
219
220
    void destroySchurPrimalKsp();

221
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
222
223
    void createFetiKsp();

224
    /// Destroys FETI-DP operator, \ref ksp_feti
225
226
    void destroyFetiKsp();

227
228
229
230
231
232
233
234
235
236
237
238
239
240
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
241
242
243
244
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
245
246
247
248
249
250
251
252
253
    /** \brief
     * Solves the FETI-DP system globally, thus without reducing it to the 
     * Lagrange multipliers. This should be used for debugging only to test
     * if the FETI-DP system is setup correctly.
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveFetiMatrix(SystemVector &vec);

254
255
256
257
258
259
260
261
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
262
263
264
265
    void resetStatistics();

    void printStatistics();

266
  protected:
267
268
269
270
    /// Number of components in the PDE to be solved.
    int nComponents;

    /// Mapping from primal DOF indices to a global index of primals.
271
    DofMapping globalPrimalIndex;
272

273
274
    /// Number of rank owned primals and global primals
    int nRankPrimals, nOverallPrimals, rStartPrimals;
275

276
    /// Set of DOF indices that are considered to be dual variables.
277
278
    DofIndexSet duals;
    
279
    /// Mapping from dual DOF indices to a global index of duals.
280
281
    DofMapping globalDualIndex;

282
283
    /// Stores to each dual boundary DOF the set of ranks in which the DOF
    /// is contained in.
284
285
    DofIndexToPartitions boundaryDofRanks;

286
287
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
288
289
    DofMapping dofFirstLagrange;

290
291
292
    /// Number of rank owned Lagrange variables, number of global
    /// Lagrange variables.
    int nRankLagrange, nOverallLagrange, rStartLagrange;
293
    
294
295
    /// Index for each non primal variables to the global index of
    /// B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
296
    DofMapping globalIndexB, localIndexB;
297

298
299
    /// Number of non primal, thus B, variables on rank and globally.
    int nRankB, nOverallB, rStartB;    
300

301
    /// Global PETSc matrix of non primal variables.
302
303
    Mat mat_b_b;

304
    /// Global PETSc matrix of primal variables.
305
306
    Mat mat_primal_primal;

307
308
    /// Global PETSc matrices that connect the primal with the non 
    /// primal variables.
309
310
    Mat mat_b_primal, mat_primal_b;

311
    /// Global PETSc matrix of Lagrange variables.
312
313
    Mat mat_lagrange;

314
    /// Right hand side PETSc vectors for primal and non primal variables.
315
316
    Vec f_b, f_primal;

317
318
    /// PETSc solver object that inverts the matrix of non primal 
    /// variables, \ref mat_b_b
319
320
    KSP ksp_b;

Thomas Witkowski's avatar
Thomas Witkowski committed
321
322
323
324
325
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

326
327
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
328
329
    KSP ksp_schur_primal;

330
331
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
332
333
    Mat mat_schur_primal;

334
    /// Data for MatMult operation in matrix \ref mat_schur_primal
335
    SchurPrimalData schurPrimalData;
336

337
    /// PETSc solver object to solve a system with FETI-DP.    
338
339
    KSP ksp_feti;

340
341
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
342
    Mat mat_feti;
343

344
    /// Data for MatMult operation in matrix \ref mat_feti
345
    FetiData fetiData;
346

347
348
349
350
351
352
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
353
354
355

    Mat mat_lagrange_scaled;
   
356
    FetiDirichletPreconData fetiDirichletPreconData;
357

358
    FetiLumpedPreconData fetiLumpedPreconData;
359

360
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
361
362
363
364
365

    KSP ksp_interior;

    int nLocalInterior;
    
366
367
    // Number of local nodes that are duals.
    int nLocalDuals;
Thomas Witkowski's avatar
Thomas Witkowski committed
368
369
370

  public:
    static FetiStatisticsData fetiStatistics;
371
372
373
374
375
376
  };
#endif

}

#endif