PetscSolverFeti.h 10.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

#include "parallel/PetscSolver.h"
24
#include "parallel/ParallelTypes.h"
25
26
27
28
29
30
31
32
33
34

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;


#ifdef HAVE_PETSC_DEV
35

Thomas Witkowski's avatar
Thomas Witkowski committed
36
37
  class PetscSolverFeti;

38
39
40
41
  /** \brief
   * This structure is used when defining the MatShell operation for solving 
   * primal schur complement. \ref petscMultMatSchurPrimal
   */
42
  struct SchurPrimalData {
43
44
45
46
47
    /// Pointers to the matrix containing the primal variables.
    Mat *mat_primal_primal;
   
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
48

49
    /// Temporal vector on the B variables.
50
51
    Vec tmp_vec_b;

52
    /// Temporal vecor in the primal variables.
53
54
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
55
    PetscSolverFeti *fetiSolver;
56
57
58
  };


59
60
61
62
63
  /** \brief
   * This structure is used when defining the FETI-DP operator for solving 
   * the system matrix reduced to the Lagrange multipliers.
   * \ref petscMultMatFeti
   */
64
  struct FetiData {
65
66
    /// Coupling matrices between the primal and the B variables.
    Mat *mat_primal_b, *mat_b_primal;
67

68
    /// Matrix of Lagrange variables.
69
70
    Mat *mat_lagrange;

71
72
    /// Temporal vectors on the B variables.
    Vec tmp_vec_b0, tmp_vec_b1;
73

74
    /// Temporal vector on the primal variables.
75
76
    Vec tmp_vec_primal;

Thomas Witkowski's avatar
Thomas Witkowski committed
77
    PetscSolverFeti *fetiSolver;
78

79
    /// Pointer to the solver of the schur complement on the primal variables.
80
81
82
83
    KSP *ksp_schur_primal;
  };


84
  struct FetiDirichletPreconData {
85
86
87
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

88
    Mat *mat_interior_interior, *mat_duals_duals, *mat_interior_duals, *mat_duals_interior;
89
90
91
92
93
94
95

    /// Pointer to the solver for \ref PetscSolverFeti::mat_bb.
    KSP *ksp_interior;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

96
97
    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
98
    
99
100
101
    /// Temporal vector on the interior variables.
    Vec tmp_vec_interior;    
  };
102

103

104
105
106
107
108
109
110
111
112
113
114
  struct FetiLumpedPreconData {
    /// Matrix of scaled Lagrange variables.
    Mat *mat_lagrange_scaled;

    Mat *mat_duals_duals;

    /// Temporal vector on the B variables.
    Vec tmp_vec_b;

    /// Temporal vector on the dual variables.
    Vec tmp_vec_duals0, tmp_vec_duals1;
115
116
  };

117

118
119
120
121
122
123
124
  typedef enum {
    FETI_NONE = 0,
    FETI_DIRICHLET = 1,
    FETI_LUMPED = 2
  } FetiPreconditioner;


Thomas Witkowski's avatar
Thomas Witkowski committed
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
  struct FetiStatisticsData
  {
    /// Number of application of the FETI-DP operator.
    int nFetiApply;

    /// Time for solving the reduced FETI system.
    double timeFetiApply;

    /// Number of application of the Schur primal operator.
    int nSchurPrimalApply;

    /// Time for appling the Schur primal operator.
    double timeSchurPrimalApply;

    /// Number of solution of the Schur primal system.
    int nSchurPrimalSolve;

    /// Time for solving the Schur primal system.
    double timeSchurPrimalSolve;

    /// Number of solution of the local subdomain problems.
    int nLocalSolve;

    /// Time for solving the local subdomain problems.
    double timeLocalSolve;
  };


153
154
155
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
156
157
158
  class PetscSolverFeti : public PetscSolver
  {
  public:
159
    PetscSolverFeti();
160

161
162
163
164
165
166
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
167

168
    /// Solve the system using FETI-DP method.
169
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
170

171
172
173
    /// Destroys all matrix data structures.
    void destroyMatrixData();

174
175
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
176
177
178
179
180
181
182
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
183

Thomas Witkowski's avatar
Thomas Witkowski committed
184
185
    void solveLocalProblem(Vec &rhs, Vec &sol);

186
  protected:
187
188
189
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all matrix and vector objects with the approriated
    /// sizes.
190
    void updateDofData();
191

192
193
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
194
195
    void createPrimals();

196
197
    /// Defines the set of dual variables and creates the global index of
    // dual variables.
198
199
    void createDuals();

200
201
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
202
203
    void createLagrange();

204
    /// Creates a global index of the B variables.
205
206
    void createIndexB();

207
208
209
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
    void createMatLagrange();
210

211
212
213
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
    void createSchurPrimalKsp();
214

215
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
216
217
    void destroySchurPrimalKsp();

218
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
219
220
    void createFetiKsp();

221
    /// Destroys FETI-DP operator, \ref ksp_feti
222
223
    void destroyFetiKsp();

224
225
226
227
228
229
230
231
232
233
234
235
236
237
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
238
239
240
241
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

242
243
244
245
246
247
248
249
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
250
251
252
253
    void resetStatistics();

    void printStatistics();

254
  protected:
255
256
257
258
    /// Number of components in the PDE to be solved.
    int nComponents;

    /// Mapping from primal DOF indices to a global index of primals.
259
    DofMapping globalPrimalIndex;
260

261
262
    /// Number of rank owned primals and global primals
    int nRankPrimals, nOverallPrimals, rStartPrimals;
263

264
    /// Set of DOF indices that are considered to be dual variables.
265
266
    DofIndexSet duals;
    
267
    /// Mapping from dual DOF indices to a global index of duals.
268
269
    DofMapping globalDualIndex;

270
271
    /// Stores to each dual boundary DOF the set of ranks in which the DOF
    /// is contained in.
272
273
    DofIndexToPartitions boundaryDofRanks;

274
275
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
276
277
    DofMapping dofFirstLagrange;

278
279
280
    /// Number of rank owned Lagrange variables, number of global
    /// Lagrange variables.
    int nRankLagrange, nOverallLagrange, rStartLagrange;
281
    
282
283
    /// Index for each non primal variables to the global index of
    /// B variables.
Thomas Witkowski's avatar
Thomas Witkowski committed
284
    DofMapping globalIndexB, localIndexB;
285

286
287
    /// Number of non primal, thus B, variables on rank and globally.
    int nRankB, nOverallB, rStartB;    
288

289
    /// Global PETSc matrix of non primal variables.
290
291
    Mat mat_b_b;

292
    /// Global PETSc matrix of primal variables.
293
294
    Mat mat_primal_primal;

295
296
    /// Global PETSc matrices that connect the primal with the non 
    /// primal variables.
297
298
    Mat mat_b_primal, mat_primal_b;

299
    /// Global PETSc matrix of Lagrange variables.
300
301
    Mat mat_lagrange;

302
    /// Right hand side PETSc vectors for primal and non primal variables.
303
304
    Vec f_b, f_primal;

305
306
    /// PETSc solver object that inverts the matrix of non primal 
    /// variables, \ref mat_b_b
307
308
    KSP ksp_b;

Thomas Witkowski's avatar
Thomas Witkowski committed
309
310
311
312
313
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

314
315
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
316
317
    KSP ksp_schur_primal;

318
319
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
320
321
    Mat mat_schur_primal;

322
    /// Data for MatMult operation in matrix \ref mat_schur_primal
323
    SchurPrimalData schurPrimalData;
324

325
    /// PETSc solver object to solve a system with FETI-DP.    
326
327
    KSP ksp_feti;

328
329
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
330
    Mat mat_feti;
331

332
    /// Data for MatMult operation in matrix \ref mat_feti
333
    FetiData fetiData;
334

335
336
337
338
339
340
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
341
342
343

    Mat mat_lagrange_scaled;
   
344
    FetiDirichletPreconData fetiDirichletPreconData;
345

346
    FetiLumpedPreconData fetiLumpedPreconData;
347

348
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
349
350
351
352
353

    KSP ksp_interior;

    int nLocalInterior;
    
354
355
    // Number of local nodes that are duals.
    int nLocalDuals;
Thomas Witkowski's avatar
Thomas Witkowski committed
356
357
358

  public:
    static FetiStatisticsData fetiStatistics;
359
360
361
362
363
364
  };
#endif

}

#endif