PetscSolverFeti.h 10.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

23
24
#include <map>
#include "parallel/MpiHelper.h"
25
#include "parallel/PetscSolver.h"
26
#include "parallel/PetscSolverFetiStructs.h"
27
#include "parallel/ParallelDofMapping.h"
28
#include "parallel/ParallelTypes.h"
29
30
31
32
33
34
35
36

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;

37
38
39
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
40
41
42
  class PetscSolverFeti : public PetscSolver
  {
  public:
43
    PetscSolverFeti();
44

45
46
47
48
49
50
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
51

52
    /// Solve the system using FETI-DP method.
53
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
54

55
56
57
    /// Destroys all matrix data structures.
    void destroyMatrixData();

58
59
60
    /// Detroys all vector data structures.
    void destroyVectorData();

61
62
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
63
64
65
66
67
68
69
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
70

71
72
73
74
75
76
    /// Initialization of the data structures with a given list of the FE 
    /// spaces of all components.
    void initialize(vector<const FiniteElemSpace*> feSpaces);

    int getNumberOfPrimals()
    {
77
      return primalDofMap.getOverallDofs();
78
    }
79

80
81
    int getNumberOfRankPrimals()
    {
82
      return primalDofMap.getRankDofs();
83
84
    }

85
86
    int getNumberOfDuals()
    {
87
      return dualDofMap.getOverallDofs();
88
89
    }

90
91
    int getNumberOfRankDuals()
    {
92
      return dualDofMap.getRankDofs();
93
94
    }

Thomas Witkowski's avatar
Blub    
Thomas Witkowski committed
95
96
97
98
99
    int getNumberOfLagrange()
    {
      return lagrangeMap.getOverallDofs();
    }

100
  protected:
Thomas Witkowski's avatar
Blub    
Thomas Witkowski committed
101
102
103
104
105
106
107
108
    ///
    void createDirichletData(Matrix<DOFMatrix*> &mat);

    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all information about primal nodes, dual nodes and
    /// lagrange constraints.    
    void createFetiData();

109
110
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
111
    void createPrimals(const FiniteElemSpace *feSpace);
112

113
    /// Defines the set of dual variables and creates the global index of
114
    /// dual variables.
115
    void createDuals(const FiniteElemSpace *feSpace);
116

117
118
119
    /// 
    void createInterfaceNodes(const FiniteElemSpace *feSpace);

120
121
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
122
    void createLagrange(const FiniteElemSpace *feSpace);
123

124
    /// Creates a global index of the B variables.
125
    void createIndexB(const FiniteElemSpace *feSpace);
126

127
128
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
129
    void createMatLagrange(vector<const FiniteElemSpace*> &feSpaces);
130

131
132
133
    ///
    void createPreconditionerMatrix(Matrix<DOFMatrix*> *mat);

134
135
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
136
    void createSchurPrimalKsp(vector<const FiniteElemSpace*> &feSpaces);
137

138
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
139
140
    void destroySchurPrimalKsp();

141
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
142
    void createFetiKsp(vector<const FiniteElemSpace*> &feSpaces);
143

144
    /// Destroys FETI-DP operator, \ref ksp_feti
145
146
    void destroyFetiKsp();

Thomas Witkowski's avatar
Thomas Witkowski committed
147
148
149
150
    /// Create the null space of the FETI-DP operator (if there is one) and
    /// attachets it to the corresponding matrices and KSP objects.
    void createNullSpace();

151
152
153
154
    /// In debug modes, this function runs some debug tests on the FETI
    /// matrices. In optimized mode, nothing is done here.
    void dbgMatrix();

155
156
157
158
159
160
161
162
163
164
165
166
167
168
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
169
170
171
172
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

Thomas Witkowski's avatar
Blub    
Thomas Witkowski committed
173
    ///
174
175
176
    void recoverInterfaceSolution(Vec& vecInterface, 
				  SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
177
178
179
180
181
182
183
184
185
    /** \brief
     * Solves the FETI-DP system globally, thus without reducing it to the 
     * Lagrange multipliers. This should be used for debugging only to test
     * if the FETI-DP system is setup correctly.
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveFetiMatrix(SystemVector &vec);

186
187
188
189
190
191
192
193
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
194
195
196
197
    void resetStatistics();

    void printStatistics();

198
    /// Checks whether a given DOF in a given FE space is a primal DOF.
199
200
    inline bool isPrimal(const FiniteElemSpace *feSpace, 
			 DegreeOfFreedom dof)
201
    {
202
      return primalDofMap[feSpace].isSet(dof);
203
204
    }

205
    /// Checks whether a given DOF in a give FE space is a dual DOF.
206
207
    inline bool isDual(const FiniteElemSpace *feSpace, 
		       DegreeOfFreedom dof)
208
    {
209
      return dualDofMap[feSpace].isSet(dof);
210
211
    }

212
213
214
215
    /// Checks whether a given DOF in a give FE space is an interface DOF.
    inline bool isInterface(const FiniteElemSpace *feSpace, 
			    DegreeOfFreedom dof)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
216
      if (feSpace == pressureFeSpace)
Thomas Witkowski's avatar
Blub    
Thomas Witkowski committed
217
218
219
	return interfaceDofMap[feSpace].isSet(dof);
      
      return false;
220
221
    }

222
223
224
225
    /// Traverse the mesh and create a DOF vector where each DOF is in the
    /// order of: 1/h^2
    void createH2vec(DOFVector<double> &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
226
227
228
229
230
231
    /// For debugging only!
    void createInteriorMat(Mat &mat);

    /// For debugging only!
    void createNestedFetiMat(Mat &mat);

232
233
234
    /// For debugging only!
    void debugNullSpace(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
235
236
237
238
239
240
241
242
243
    /// For debugging only!
    void createExplicitFetiMat(Mat fetiMat, Mat &explicitmat, int &nnzCounter);

    /// For debugging only!
    void createExplicitVec(Vec nestedVec, Vec &explicitVec);

    /// For debugging only!
    void debugFeti(Vec dbgRhsVec);

244
  protected:
245
    /// Mapping from primal DOF indices to a global index of primals.
Thomas Witkowski's avatar
Thomas Witkowski committed
246
    ParallelDofMapping primalDofMap;
247

248
    /// Mapping from dual DOF indices to a global index of duals.
Thomas Witkowski's avatar
Thomas Witkowski committed
249
    ParallelDofMapping dualDofMap;
250

251
252
253
254
255
    /// Mapping from interface DOF indices to a global index of interface 
    /// nodes. This is mainly used for Stokes-like solvers, where the pressure
    /// interface nodes are neither primal nor dual.
    ParallelDofMapping interfaceDofMap;

Thomas Witkowski's avatar
Thomas Witkowski committed
256
257
258
259
    /// Index for each non primal DOF to the global index of B variables (thus,
    /// all pure local variables).
    ParallelDofMapping localDofMap;

260
261
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
262
    ParallelDofMapping lagrangeMap;
263
    
264
265
    /// Mapping of pure local DOF indices, thus no primal and no dual DOFs are
    /// in this map. Is used for the Dirichlet preconditioner only.
Thomas Witkowski's avatar
Thomas Witkowski committed
266
    ParallelDofMapping interiorDofMap;
267

Thomas Witkowski's avatar
Thomas Witkowski committed
268
269
    /// Stores to all dual boundary DOFs in each FE space the set of
    /// ranks which contain this global DOF.
270
    map<const FiniteElemSpace*, DofIndexToPartitions> boundaryDofRanks;
271

272
    /// Global PETSc matrix of Lagrange variables.
273
274
    Mat mat_lagrange;

Thomas Witkowski's avatar
Thomas Witkowski committed
275
276
277
278
279
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

280
281
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
282
283
    KSP ksp_schur_primal;

284
285
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
286
287
    Mat mat_schur_primal;

288
    /// Data for MatMult operation in matrix \ref mat_schur_primal
289
    SchurPrimalData schurPrimalData;
290

291
    /// PETSc solver object to solve a system with FETI-DP.    
292
293
    KSP ksp_feti;

294
295
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
296
    Mat mat_feti;
297

298
    /// Data for MatMult operation in matrix \ref mat_feti
299
    FetiData fetiData;
300

301
302
303
304
305
306
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
307
308
309

    Mat mat_lagrange_scaled;
   
310
    FetiDirichletPreconData fetiDirichletPreconData;
311

312
    FetiLumpedPreconData fetiLumpedPreconData;
313

Thomas Witkowski's avatar
Thomas Witkowski committed
314
315
    FetiInterfaceLumpedPreconData fetiInterfaceLumpedPreconData;

316
317
    FetiKspData fetiKspData;

Thomas Witkowski's avatar
Thomas Witkowski committed
318
    /// Matrices for Dirichlet preconditioner.
319
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
320
321

    KSP ksp_interior;
322
323
324

    bool multiLevelTest;

Thomas Witkowski's avatar
Thomas Witkowski committed
325
    PetscSolver *subdomain;
326

327
328
    PetscSolver *massMatrixSolver;

329
    int meshLevel;
330
331
332
333

    int rStartInterior;

    int nGlobalOverallInterior;
Thomas Witkowski's avatar
Thomas Witkowski committed
334
335

    bool printTimings;
Thomas Witkowski's avatar
Thomas Witkowski committed
336

Thomas Witkowski's avatar
Thomas Witkowski committed
337
338
339
340
341
    bool stokesMode;

    const FiniteElemSpace* pressureFeSpace;

    int pressureComponent;
Thomas Witkowski's avatar
Blub    
Thomas Witkowski committed
342
343

    map<const FiniteElemSpace*, std::set<DegreeOfFreedom> > dirichletRows;
344
  };
345

346
347
348
}

#endif