PetscSolverFeti.h 11.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file PetscSolverFeti.h */

23
#include <map>
24
#include "AMDiS_fwd.h"
25
#include "parallel/MpiHelper.h"
26
#include "parallel/PetscSolver.h"
27
#include "parallel/PetscSolverFetiStructs.h"
28
#include "parallel/ParallelDofMapping.h"
29
#include "parallel/ParallelTypes.h"
30
31
32
33
34
35
36
37

#ifndef AMDIS_PETSC_SOLVER_FETI_H
#define AMDIS_PETSC_SOLVER_FETI_H

namespace AMDiS {

  using namespace std;

38
39
40
  /** \brief
   * FETI-DP implementation based on PETSc.
   */
41
42
43
  class PetscSolverFeti : public PetscSolver
  {
  public:
44
45
46
47
48
49
50
51
52
53
54
55
56
    /// Creator class
    class Creator : public OEMSolverCreator
    {
    public:
      virtual ~Creator() {}

      /// Returns a new PetscSolver object.
      OEMSolver* create() 
      { 
	return new PetscSolverFeti(this->name); 
      }
    };
    
57
58
    /// Constructor of FETI-DP solver class.
    PetscSolverFeti(string name);
59

60
61
62
63
64
    /// After mesh changes, or if the solver is called the first time, this
    /// function creates all information about primal nodes, dual nodes and
    /// lagrange constraints.    
    void createFetiData();

65
66
67
68
69
70
    /// Assemble the sequentially created matrices to the global matrices
    /// required by the FETI-DP method.
    void fillPetscMatrix(Matrix<DOFMatrix*> *mat);

    /// Assembles the global rhs vectors from the sequentially created ones.
    void fillPetscRhs(SystemVector *vec);
71

72
    /// Solve the system using FETI-DP method.
73
    void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
74

75
76
77
    /// Destroys all matrix data structures.
    void destroyMatrixData();

78
79
80
    /// Detroys all vector data structures.
    void destroyVectorData();

81
82
    /// Returns flags to denote which information of the boundary DOFs are 
    /// required by the FETI-DP solver.
83
84
85
86
87
88
89
    Flag getBoundaryDofRequirement()
    {
      return 
	MeshDistributor::BOUNDARY_SUBOBJ_SORTED |
	MeshDistributor::BOUNDARY_FILL_INFO_SEND_DOFS |
	MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
    }
90

91
92
    /// Initialization of the FETI-DPdata structures.
    void initialize();
93
94
95

    int getNumberOfPrimals()
    {
96
      return primalDofMap.getOverallDofs();
97
    }
98

99
100
    int getNumberOfRankPrimals()
    {
101
      return primalDofMap.getRankDofs();
102
103
    }

104
105
    int getNumberOfDuals()
    {
106
      return dualDofMap.getOverallDofs();
107
108
    }

109
110
    int getNumberOfRankDuals()
    {
111
      return dualDofMap.getRankDofs();
112
113
    }

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
114
115
116
117
118
    int getNumberOfLagrange()
    {
      return lagrangeMap.getOverallDofs();
    }

119
  protected:
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
120
121
122
    ///
    void createDirichletData(Matrix<DOFMatrix*> &mat);

123
124
    /// Defines which boundary nodes are primal. Creates global index of
    /// the primal variables.
125
    void createPrimals(int component);
126

127
    /// Defines the set of dual variables and creates the global index of
128
    /// dual variables.
129
    void createDuals(int component);
130

131
    /// 
132
    void createInterfaceNodes(int component);
133

134
135
    /// Create Lagrange multiplier variables corresponding to the dual 
    /// variables.
136
    void createLagrange(int component);
137

138
    void createAugmentedLagrange(int component);
139

140
    /// Creates a global index of the B variables.
141
    void createIndexB(int component);
142

143
144
    /// Creates the Lagrange multiplier constraints and assembles them 
    /// to \ref mat_lagrange.
145
    void createMatLagrange();
146

147
148
149
    vector<vector<BoundaryObject> > getCoarseEdges();

    vector<vector<BoundaryObject> > getCoarseFaces();
150

151
    void createMatAugmentedLagrange();
152

Thomas Witkowski's avatar
Thomas Witkowski committed
153
154
    bool testWirebasketEdge(BoundaryObject &edge, const FiniteElemSpace *feSpace);

155
156
157
    ///
    void createPreconditionerMatrix(Matrix<DOFMatrix*> *mat);

158
159
    /// Creates PETSc KSP solver object for solving the Schur complement
    /// system on the primal variables, \ref ksp_schur_primal
160
    void createSchurPrimalKsp();
161

162
163
164
165
166
167
    ///
    void createMatExplicitSchurPrimal();
    
    ///
    void createMatExplicitAugmentedSchurPrimal();

168
    /// Destroys PETSc KSP solver object \ref ksp_schur_primal
169
170
    void destroySchurPrimalKsp();

171
    /// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
172
    void createFetiKsp();
173

174
    /// Destroys FETI-DP operator, \ref ksp_feti
175
176
    void destroyFetiKsp();

Thomas Witkowski's avatar
Thomas Witkowski committed
177
178
179
180
    /// Create the null space of the FETI-DP operator (if there is one) and
    /// attachets it to the corresponding matrices and KSP objects.
    void createNullSpace();

181
182
    /// In debug modes, this function runs some debug tests on the FETI
    /// matrices. In optimized mode, nothing is done here.
183
    void dbgMatrix(Matrix<DOFMatrix*> *mat);
184

185
186
187
188
189
190
191
192
193
194
195
196
197
198
    /** \brief
     * Recovers AMDiS solution vector from PETSc's solution vectors of the
     * FETI-DP system. First, the B variables can locally be copied to the
     * corresponding entries in the DOF vectors. The primal variable must
     * be communicated such that all ranks sharing a primal get a copy of
     * the corresponding value.
     *
     * \param[in]   vec_sol_b        Global PETSc vector of the solution of
     *                               the B variables.
     * \param[in]   vec_sol_primal   Global PETSc vector of the solution of
     *                               the primal variables.
     * \param[out]  vec              SystemVector containing all solution 
     *                               DOF vectors.
     */
199
200
201
202
    void recoverSolution(Vec &vec_sol_b,
			 Vec &vec_sol_primal,
			 SystemVector &vec);

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
203
    ///
204
205
206
    void recoverInterfaceSolution(Vec& vecInterface, 
				  SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
207
208
209
210
211
212
213
214
215
    /** \brief
     * Solves the FETI-DP system globally, thus without reducing it to the 
     * Lagrange multipliers. This should be used for debugging only to test
     * if the FETI-DP system is setup correctly.
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveFetiMatrix(SystemVector &vec);

216
217
218
219
220
221
222
223
    /** \brief
     * Solves the FETI-DP system with reducing it first to the Lagrange
     * multipliers. This is what one expects when using the FETI-DP methid :)
     *
     * \param[out]  vec    Solution DOF vectors.
     */
    void solveReducedFetiMatrix(SystemVector &vec);

Thomas Witkowski's avatar
Thomas Witkowski committed
224
225
226
227
    void resetStatistics();

    void printStatistics();

228
229
    /// Checks whether a given DOF is a primal DOF in a given component.
    inline bool isPrimal(int component, DegreeOfFreedom dof)
230
    {
231
      return primalDofMap[component].isSet(dof);
232
233
    }

234
235
    /// Checks whether a given DOF is a dual DOF in a given component.
    inline bool isDual(int component, DegreeOfFreedom dof)
236
    {
237
      return dualDofMap[component].isSet(dof);
238
239
    }

240
241
    /// Checks whether a given DOF is an interface DOF in a given component.
    inline bool isInterface(int component, DegreeOfFreedom dof)
242
    {
243
244
      if (component == pressureComponent)
	return interfaceDofMap[component].isSet(dof);
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
245
246
      
      return false;
247
248
    }

249
  protected:
250
    /// Mapping from primal DOF indices to a global index of primals.
Thomas Witkowski's avatar
Thomas Witkowski committed
251
    ParallelDofMapping primalDofMap;
252

253
    /// Mapping from dual DOF indices to a global index of duals.
Thomas Witkowski's avatar
Thomas Witkowski committed
254
    ParallelDofMapping dualDofMap;
255

256
257
258
259
260
    /// Mapping from interface DOF indices to a global index of interface 
    /// nodes. This is mainly used for Stokes-like solvers, where the pressure
    /// interface nodes are neither primal nor dual.
    ParallelDofMapping interfaceDofMap;

Thomas Witkowski's avatar
Thomas Witkowski committed
261
262
263
264
    /// Index for each non primal DOF to the global index of B variables (thus,
    /// all pure local variables).
    ParallelDofMapping localDofMap;

265
266
    /// Stores to each dual DOF index the index of the first Lagrange
    /// constraint that is assigned to this DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
267
    ParallelDofMapping lagrangeMap;
268
    
269
270
    /// Mapping of pure local DOF indices, thus no primal and no dual DOFs are
    /// in this map. Is used for the Dirichlet preconditioner only.
Thomas Witkowski's avatar
Thomas Witkowski committed
271
    ParallelDofMapping interiorDofMap;
272

Thomas Witkowski's avatar
Thomas Witkowski committed
273
274
    /// Stores to all dual boundary DOFs in each FE space the set of
    /// ranks which contain this global DOF.
275
    map<const FiniteElemSpace*, DofIndexToPartitions> boundaryDofRanks;
276

277
    /// Global PETSc matrix of Lagrange variables.
278
279
    Mat mat_lagrange;

280
281
282
    ///
    Mat mat_augmented_lagrange; 

Thomas Witkowski's avatar
Thomas Witkowski committed
283
284
285
286
287
    /// 0: Solve the Schur complement on primal variables with iterative solver.
    /// 1: Create the Schur complement matrix explicitly and solve it with a
    ///    direct solver.
    int schurPrimalSolver;

288
289
    /// PETSc solver object to solve the Schur complement on the 
    /// primal variables.
290
291
    KSP ksp_schur_primal;

292
293
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the Schur complement on the primal variables.
294
295
    Mat mat_schur_primal;

296
    /// Data for MatMult operation in matrix \ref mat_schur_primal
297
    SchurPrimalData schurPrimalData;
298

299
300
301
    ///
    SchurPrimalAugmentedData schurPrimalAugmentedData;

302
    /// PETSc solver object to solve a system with FETI-DP.    
303
304
    KSP ksp_feti;

305
306
    /// Matrix object that defines a matrix-free implementation for the action
    /// of the FETI-DP operator.
307
    Mat mat_feti;
308

309
    /// Data for MatMult operation in matrix \ref mat_feti
310
    FetiData fetiData;
311

312
313
314
315
316
317
    /// Defines which preconditioner should be used to solve the reduced
    /// FETI-DP system.
    FetiPreconditioner fetiPreconditioner;

    /// Preconditioner object for the reduced FETI-DP system.
    PC precon_feti;
318
319
320

    Mat mat_lagrange_scaled;
   
321
    FetiDirichletPreconData fetiDirichletPreconData;
322

323
    FetiLumpedPreconData fetiLumpedPreconData;
324

Thomas Witkowski's avatar
Thomas Witkowski committed
325
326
    FetiInterfaceLumpedPreconData fetiInterfaceLumpedPreconData;

327
328
    FetiKspData fetiKspData;

Thomas Witkowski's avatar
Thomas Witkowski committed
329
    /// Matrices for Dirichlet preconditioner.
330
    Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
331
332

    KSP ksp_interior;
333
334
335

    bool multiLevelTest;

Thomas Witkowski's avatar
Thomas Witkowski committed
336
    PetscSolver *subdomain;
337

338
339
    PetscSolver *massMatrixSolver;

340
    int meshLevel;
341
342
343
344

    int rStartInterior;

    int nGlobalOverallInterior;
Thomas Witkowski's avatar
Thomas Witkowski committed
345
346

    bool printTimings;
Thomas Witkowski's avatar
Thomas Witkowski committed
347

348
349
350
351
352
353
    bool augmentedLagrange;

    int nRankEdges;

    int nOverallEdges;

Thomas Witkowski's avatar
d  
Thomas Witkowski committed
354
355
356
357
358
359
    /// There are two different dirichlet modes:
    ///   0: dirichlet rows are zeroed and a diagonal element is set to one.
    ///   1: dirichlet rows are removed (this mode does not work correctly, but
    ///      many function are prepered to make use of it)
    int dirichletMode;

360
361
362
    /// If true, the FETI-DP solver is applied to a Stokes like problem. Thus, 
    /// there is a pressure variable which is not part of the coarse grid
    /// problem.
Thomas Witkowski's avatar
Thomas Witkowski committed
363
364
    bool stokesMode;

365
366
    /// Only used if \ref stokesMode is enabled. In this case, this variable
    /// defines the component number of the pressure variable.
Thomas Witkowski's avatar
Thomas Witkowski committed
367
    int pressureComponent;
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
368

369
370
371
    /// Maps from component number to set of DOFs which are Dirichlet DOfs in
    /// this component.
    map<int, std::set<DegreeOfFreedom> > dirichletRows;
372
373

    friend class PetscSolverFetiDebug;
374
  };
375

376
377
378
}

#endif