PetscSolverNSCH.cc 15.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/******************************************************************************
 *
 * AMDiS - Adaptive multidimensional simulations
 *
 * Copyright (C) 2013 Dresden University of Technology. All Rights Reserved.
 * Web: https://fusionforge.zih.tu-dresden.de/projects/amdis
 *
 * Authors: 
 * Simon Vey, Thomas Witkowski, Andreas Naumann, Simon Praetorius, et al.
 *
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *
 * This file is part of AMDiS
 *
 * See also license.opensource.txt in the distribution.
 * 
 ******************************************************************************/
Sebastian Aland's avatar
Sebastian Aland committed
20
21
22
23
24
25
26
27
28
29



#include "parallel/PetscSolverNSCH.h"
#include "parallel/PetscHelper.h"
#include "TransformDOF.h"




30
namespace AMDiS { namespace Parallel {
Sebastian Aland's avatar
Sebastian Aland committed
31
32
33
34
35
36
37
38
39
40
41
42
  
  using namespace std;
  
  
  PetscErrorCode pcShell(PC pc, Vec b, Vec x) // solve Px=b
  {FUNCNAME("PCApply()");
  
  void *ctx;
  PCShellGetContext(pc, &ctx);
  CahnHilliardData2* data = static_cast<CahnHilliardData2*>(ctx);
  
  /// extract vectors
43
44
45
46
47
  Vec b1, b2, b34, b5, x1, x2, x34, x5;
  data->globalMatrixSolver->extractVectorComponent(b, data->dim+1, &b1);
  data->globalMatrixSolver->extractVectorComponent(b, data->dim+2, &b2);
  data->globalMatrixSolver->extractVectorComponent(b, 0, &b34, data->dim);  
  data->globalMatrixSolver->extractVectorComponent(b, data->dim, &b5);
Sebastian Aland's avatar
Sebastian Aland committed
48
  
49
50
51
52
  data->globalMatrixSolver->extractVectorComponent(x, data->dim+1, &x1);
  data->globalMatrixSolver->extractVectorComponent(x, data->dim+2, &x2);
  data->globalMatrixSolver->extractVectorComponent(x, 0, &x34, data->dim);  
  data->globalMatrixSolver->extractVectorComponent(x, data->dim, &x5);
Sebastian Aland's avatar
Sebastian Aland committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    
  
  /// solve Cahn-Hilliard Preconditioner  
    Vec y1, y2;
     VecDuplicate(b1, &y1);
     VecDuplicate(b2, &y2);
    
     KSPSolve(data->kspMassCH, b1, y1); 			// M*y1 = b1    
     MatMultAdd(data->matMinusDeltaK, y1, b2, x1); 	// -> x1 := b2-delta*K*y1
     
     KSPSolve(data->kspLaplaceCH, x1, y2); 			// (M+eps*sqrt(delta))*y2 = x1
     MatMult(data->matMassCH, y2, x1); 			// x1 := M*y2
     
     KSPSolve(data->kspLaplaceCH, x1, x2);			// (M+eps*sqrt(delta))*x2 = x1
     double factor = (*data->eps)/sqrt(*data->delta);
     VecCopy(x2, x1); 					// x1 := x2
     VecAXPBYPCZ(x1, 1.0, factor, -factor, y1, y2);	// x1 = 1*y1 + factor*y2 - factor*x1
     
     VecDestroy(&y1);
     VecDestroy(&y2);     
    /**/ 
     
  /// solve Navier-Stokes Preconditioner  
  Vec tmp34, tmp5, tmp5_2, tmp34_2;
  VecDuplicate(b34, &tmp34);
  VecDuplicate(b34, &tmp34_2);
  VecDuplicate(b5, &tmp5);
  VecDuplicate(b5, &tmp5_2);
  
  KSPSolve(data->kspVelocity, b34, tmp34);
  VecScale(tmp34, -1.0);
  MatMultAdd(data->matDiv, tmp34, b5, tmp5);  
  
  /// approximierte Schur Komplement
   KSPSolve(data->kspLaplace, tmp5, x5);
   { //project out constant Null-space
      int vecSize;
      VecGetSize(x5, &vecSize);
      PetscScalar vecSum;
      VecSum(x5, &vecSum);
      vecSum = vecSum / static_cast<PetscScalar>(-1.0 * vecSize);
      VecShift(x5, vecSum); 
      //VecView(y, PETSC_VIEWER_STDOUT_WORLD);
    }
    MatMult(data->matConDif, x5, tmp5);
    KSPSolve(data->kspMass, tmp5, x5);
    VecScale(x5,-1.0);
    
  MatMultAdd(data->matGrad, x5, b34, tmp34); 
  KSPSolve(data->kspVelocity, tmp34, x34);
  VecScale(x5,-1.0);
/**/  

  VecDestroy(&tmp34);
  VecDestroy(&tmp5);
  VecDestroy(&tmp5_2);
  VecDestroy(&tmp34_2);
  VecDestroy(&b1);
  VecDestroy(&b2);
  VecDestroy(&b34);
  VecDestroy(&b5);
  VecDestroy(&x1);
  VecDestroy(&x2);
  VecDestroy(&x34);
  VecDestroy(&x5);
  
119
    PetscFunctionReturn(0);
Sebastian Aland's avatar
Sebastian Aland committed
120
121
122
123
124
125
  }
  
  
  
  PetscSolverNSCH::PetscSolverNSCH(string name)
  : PetscSolverGlobalMatrix(name),      
126
  pressureNullSpace(true),
Sebastian Aland's avatar
Sebastian Aland committed
127
  useOldInitialGuess(false),      
128
129
  velocitySolutionMode(0),
  massSolutionMode(0),
Sebastian Aland's avatar
Sebastian Aland committed
130
  laplaceSolutionMode(0),
131
  regularizeLaplace(0),
132
133
134
135
136
137
138
139
140
141
  massMatrixSolverCH(nullptr),
  laplaceMatrixSolverCH(nullptr), 
  deltaKMatrixSolver(nullptr),
  massMatrixSolver(nullptr),
  laplaceMatrixSolver(nullptr),
  conDifMatrixSolver(nullptr),
  nu(nullptr),
  invTau(nullptr),
  solution(nullptr),
  phase(nullptr)
Sebastian Aland's avatar
Sebastian Aland committed
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  {    
    Parameters::get(initFileStr + "->use old initial guess", 
		    useOldInitialGuess);
    
    Parameters::get(initFileStr + "->navierstokes->velocity solver", 
		    velocitySolutionMode);
    
    Parameters::get(initFileStr + "->navierstokes->mass solver", 
		    massSolutionMode);
    
    Parameters::get(initFileStr + "->navierstokes->laplace solver", 
		    laplaceSolutionMode);
    Parameters::get(initFileStr + "->navierstokes->regularize laplace", 
		    regularizeLaplace);
    
    
  }
  
  
  void PetscSolverNSCH::solvePetscMatrix(SystemVector &vec, 
						  AdaptInfo *adaptInfo)
  {
    FUNCNAME("PetscSolverNSCH::solvePetscMatrix()");
    
       if (useOldInitialGuess) {      
          VecSet(getVecSolInterior(), 0.0);
          
          for (int i = 0; i < solution->getSize(); i++)
    	setDofVector(getVecSolInterior(), solution->getDOFVector(i), i, true);
          
          vecSolAssembly();
          KSPSetInitialGuessNonzero(kspInterior, PETSC_TRUE);
  }
  
    PetscSolverGlobalMatrix::solvePetscMatrix(vec, adaptInfo);
  }
  
  
  void PetscSolverNSCH::initSolver(KSP &ksp)
  {
    FUNCNAME("PetscSolverNSCH::initSolver()");
    
    // Create FGMRES based outer solver    
    MSG("CREATE POS 1: %p\n", &ksp);
    KSPCreate(domainComm, &ksp);
    KSPSetOperators(ksp, getMatInterior(), getMatInterior(), SAME_NONZERO_PATTERN);
    KSPMonitorSet(ksp, KSPMonitorTrueResidualNorm, PETSC_NULL, PETSC_NULL);
189
    petsc_helper::setSolver(ksp, "ch_", KSPFGMRES, PCSHELL, getRelative(), getTolerance(), getMaxIterations());
Sebastian Aland's avatar
Sebastian Aland committed
190
191
192
193
194
195
196
197
198
199
200
201
    setConstantNullSpace(ksp, componentSpaces[0]->getMesh()->getDim() , true);
  }
  
  
  void PetscSolverNSCH::initPreconditioner(PC pc)
  {
    FUNCNAME("PetscSolverNSCH::initPreconditioner()");
       
    MPI::COMM_WORLD.Barrier();
    double wtime = MPI::Wtime();
    int dim = componentSpaces[0]->getMesh()->getDim();     
    pressureComponent=dim;
202
    const FiniteElemSpace *cahnHilliardFeSpace = componentSpaces[dim+1];
203
//     const FiniteElemSpace *velocityFeSpace= componentSpaces[0];
Sebastian Aland's avatar
Sebastian Aland committed
204
205
206
207
208
209
210
    const FiniteElemSpace *pressureFeSpace = componentSpaces[pressureComponent];
        
    PCSetType(pc, PCSHELL);
    PCShellSetApply(pc, pcShell);
    PCShellSetContext(pc, &matShellContext);
    matShellContext.globalMatrixSolver = (this);
    matShellContext.mpiCommGlobal= &(meshDistributor->getMpiComm(0));
211
    matShellContext.dim = dim;
Sebastian Aland's avatar
Sebastian Aland committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
    
    
    /// Init Cahn-Hilliard part
    
    DOFMatrix laplaceMatrixCH(cahnHilliardFeSpace, cahnHilliardFeSpace);
    Operator laplaceOpCH(cahnHilliardFeSpace, cahnHilliardFeSpace);
    
    DOFMatrix massMatrixCH(cahnHilliardFeSpace, cahnHilliardFeSpace);
    Operator massOpCH(cahnHilliardFeSpace, cahnHilliardFeSpace);
    
    DOFMatrix deltaKMatrix(cahnHilliardFeSpace, cahnHilliardFeSpace);
    Operator laplaceOp2(cahnHilliardFeSpace, cahnHilliardFeSpace);
    
    { 
      Simple_ZOT zot;
      massOpCH.addTerm(&zot);
      laplaceOpCH.addTerm(&zot); // M
      Simple_SOT sot2((*eps)*sqrt(*delta));
      laplaceOpCH.addTerm(&sot2); // eps*sqrt(delta)*K
      Simple_SOT sot(-(*delta));    
      laplaceOp2.addTerm(&sot); // -delta*K
      
      massMatrixCH.assembleOperator(massOpCH);        
235
      massMatrixSolverCH = createSubSolver(dim+1, "mass_");       
Sebastian Aland's avatar
Sebastian Aland committed
236
237
238
      massMatrixSolverCH->fillPetscMatrix(&massMatrixCH);     
      // === matrix (M + eps*sqrt(delta)*K) ===
      laplaceMatrixCH.assembleOperator(laplaceOpCH);
239
      laplaceMatrixSolverCH = createSubSolver(dim+1, "laplace_");
Sebastian Aland's avatar
Sebastian Aland committed
240
241
242
243
      laplaceMatrixSolverCH->fillPetscMatrix(&laplaceMatrixCH);     
      
      // === matrix (-delta*K) ===
      deltaKMatrix.assembleOperator(laplaceOp2);
244
      deltaKMatrixSolver = createSubSolver(dim+1, "laplace2_");
Sebastian Aland's avatar
Sebastian Aland committed
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
      deltaKMatrixSolver->fillPetscMatrix(&deltaKMatrix);      
    }
    
    // === Setup solver ===
    matShellContext.kspMassCH = massMatrixSolverCH->getSolver();
    matShellContext.kspLaplaceCH = laplaceMatrixSolverCH->getSolver();
    matShellContext.matMassCH = massMatrixSolverCH->getMatInterior();    
    matShellContext.matMinusDeltaK = deltaKMatrixSolver->getMatInterior();   
    matShellContext.eps = eps;
    matShellContext.delta = delta;
    
    petsc_helper::setSolver(matShellContext.kspMassCH, "", KSPCG, PCJACOBI, 0.0, 1e-14, 2);
    petsc_helper::setSolver(matShellContext.kspLaplaceCH, "laplace_", KSPRICHARDSON, PCHYPRE, 0.0, 1e-14, 1);
  
    
    /// Init Navier-Stokes part
    
    DOFMatrix massMatrix(pressureFeSpace, pressureFeSpace);
      Operator massOp(pressureFeSpace, pressureFeSpace);
      ZeroOrderTerm *massTerm = new Simple_ZOT;
      massOp.addTerm(massTerm);
      massMatrix.assembleOperator(massOp);
      delete massTerm;
    massMatrixSolver = createSubSolver(pressureComponent, "mass_");
    massMatrixSolver->fillPetscMatrix(&massMatrix);
    matShellContext.kspMass = massMatrixSolver->getSolver();   
    
    DOFMatrix laplaceMatrix(pressureFeSpace, pressureFeSpace);    
      Operator laplaceOp(pressureFeSpace, pressureFeSpace);
      SecondOrderTerm *laplaceTerm = new Simple_SOT;
      laplaceOp.addTerm(laplaceTerm);
      laplaceMatrix.assembleOperator(laplaceOp);
      delete laplaceTerm;    
    laplaceMatrixSolver = createSubSolver(pressureComponent, string("laplace_"));
    laplaceMatrixSolver->fillPetscMatrix(&laplaceMatrix);
        
    // === Create convection-diffusion operator ===
    
    DOFVector<double> vx(pressureFeSpace, "vx");
    DOFVector<double> vy(pressureFeSpace, "vy");
    DOFVector<double> vz(pressureFeSpace, "vz");
    DOFVector<double> vp(pressureFeSpace, "vp");
    vx.interpol(solution->getDOFVector(0));
    vy.interpol(solution->getDOFVector(1));
    if (dim == 3) vz.interpol(solution->getDOFVector(2));
    DOFMatrix conDifMatrix(pressureFeSpace, pressureFeSpace);
    {
      Operator conDifOp(pressureFeSpace, pressureFeSpace);
293
294
295
      ZeroOrderTerm *conDif0 = nullptr;
      SecondOrderTerm *conDif1 = nullptr;
      FirstOrderTerm *conDif2 = nullptr, *conDif3 = nullptr, *conDif4 = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
      vp.interpol(solution->getDOFVector(dim+2));
      
      densityFunctionTau = new LinearInterpolation(*rho1,*rho2,*invTau);
      viscosityFunction = new LinearInterpolation(*nu1,*nu2);
      densityFunction = new LinearInterpolation2(*rho1,*rho2);      
      
      conDif0 = new VecAtQP_ZOT(&vp, densityFunctionTau);
      conDifOp.addTerm(conDif0);	
      conDif1 = new VecAtQP_SOT(&vp, viscosityFunction );
      conDifOp.addTerm(conDif1);	
      conDif2 = new Vec2AtQP_FOT(&vx, &vp, densityFunction, 0);
      conDifOp.addTerm(conDif2, GRD_PHI);
      
      conDif3 = new Vec2AtQP_FOT(&vy, &vp, densityFunction, 1);
      conDifOp.addTerm(conDif3, GRD_PHI);
      
      if (dim == 3) {
	conDif4 = new Vec2AtQP_FOT(&vz, &vp, densityFunction, 2);
	conDifOp.addTerm(conDif4, GRD_PHI);
      }
         
      conDifMatrix.assembleOperator(conDifOp);
      
      delete conDif0;
      delete conDif1;
      delete conDif2;
      delete conDif3;
      if (dim == 3) delete conDif4;      
    }
    
    conDifMatrixSolver = createSubSolver(pressureComponent, "condif_");
    conDifMatrixSolver->fillPetscMatrix(&conDifMatrix);
    matShellContext.matConDif = conDifMatrixSolver->getMatInterior();    
    
330
331
332
    extractMatrixComponent(mat[0][0], dim, 1, 0, dim, &(matShellContext.matDiv));
    extractMatrixComponent(mat[0][0], 0, dim, dim, 1, &(matShellContext.matGrad));
    extractMatrixComponent(mat[0][0], 0, dim, 0, dim, &(matShellContext.velocityMat));
Sebastian Aland's avatar
Sebastian Aland committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
    
    ///erstelle kspVelocity
    KSPCreate((meshDistributor->getMpiComm(0)), &(matShellContext.kspVelocity));
    KSPSetOperators(matShellContext.kspVelocity, matShellContext.velocityMat, matShellContext.velocityMat, SAME_NONZERO_PATTERN);            
    
    ///regularisiere LaplaceMatrix
  if (regularizeLaplace)
  {
    PetscInt rows[1];
    rows[0]=0;
    MatZeroRows(laplaceMatrixSolver->getMatInterior(), 1, rows, 0, PETSC_NULL, PETSC_NULL);
    KSPCreate((meshDistributor->getMpiComm(0)), &(matShellContext.kspLaplace));
    KSPSetOperators(matShellContext.kspLaplace, laplaceMatrixSolver->getMatInterior(), laplaceMatrixSolver->getMatInterior(), SAME_NONZERO_PATTERN);    
  }
   else
   {  matShellContext.kspLaplace=laplaceMatrixSolver->getSolver();
     setConstantNullSpace(matShellContext.kspLaplace);
   }
    
  
    
    // === Setup solver ===
    
    switch (massSolutionMode) {
      case 0:
	petsc_helper::setSolver(matShellContext.kspMass, "mass_", KSPCG, PCJACOBI, 0.0, 1e-14, 2);
	break;
      case 1:
	petsc_helper::setSolverWithLu(matShellContext.kspMass, "mass_", KSPRICHARDSON,     PCLU, MATSOLVERMUMPS, 0.0, 1e-14, 1);
	break;
      default:
	ERROR_EXIT("No mass solution mode %d available!\n", massSolutionMode);
    }
    
    switch (laplaceSolutionMode) {
      case 0:
	petsc_helper::setSolver(matShellContext.kspLaplace, "laplace_", KSPRICHARDSON, PCHYPRE, 0.0, 1e-14, 1);
	break;
      case 1:
	petsc_helper::setSolverWithLu(matShellContext.kspLaplace, "mass_",   KSPRICHARDSON, PCLU, MATSOLVERMUMPS,   0.0, 1e-14, 1);
	break;
      default:
	ERROR_EXIT("No laplace solution mode %d available!\n", laplaceSolutionMode);
    }
    
    switch (velocitySolutionMode) {
      case 0:      
	petsc_helper::setSolver(matShellContext.kspVelocity, "", KSPRICHARDSON, PCHYPRE, 0.0, 1e-14, 1);
	break;
      case 1:
	petsc_helper::setSolverWithLu(matShellContext.kspVelocity, "", KSPPREONLY,   PCLU, MATSOLVERMUMPS , 0.0, 1e-14, 1);
	break;
      default:
	ERROR_EXIT("No velocity solution mode %d available!\n", velocitySolutionMode);
    }
    
 
    PCSetFromOptions(pc);
    
    MSG("Setup of Cahn-Hilliard preconditioner needed %.5f seconds\n", 
	MPI::Wtime() - wtime);
     }
     
     
     void PetscSolverNSCH::exitPreconditioner(PC pc)
     {
       FUNCNAME("PetscSolverNSCH::exitPreconditioner()");
       
       massMatrixSolverCH->destroyMatrixData();
       massMatrixSolverCH->destroyVectorData();
       laplaceMatrixSolverCH->destroyMatrixData();
       laplaceMatrixSolverCH->destroyVectorData();    
       deltaKMatrixSolver->destroyMatrixData();
       deltaKMatrixSolver->destroyVectorData();
       
       
       massMatrixSolverCH->destroyVectorData();
       laplaceMatrixSolverCH->destroyVectorData();
       deltaKMatrixSolver->destroyVectorData();
       
       delete massMatrixSolverCH;
414
       massMatrixSolverCH = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
415
416
       
       delete laplaceMatrixSolverCH;
417
       laplaceMatrixSolverCH = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
418
419
       
       delete deltaKMatrixSolver;
420
       deltaKMatrixSolver = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
       
       massMatrixSolver->destroyMatrixData();
       massMatrixSolver->destroyVectorData();
       laplaceMatrixSolver->destroyMatrixData();
       laplaceMatrixSolver->destroyVectorData();
       conDifMatrixSolver->destroyMatrixData();
       conDifMatrixSolver->destroyVectorData();
       
       massMatrixSolver->destroyVectorData();
       laplaceMatrixSolver->destroyVectorData();
       conDifMatrixSolver->destroyVectorData();
       
       
       delete massMatrixSolver;
435
       massMatrixSolver = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
436
437
       
       delete laplaceMatrixSolver;
438
       laplaceMatrixSolver = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
439
440
       
       delete conDifMatrixSolver;
441
       conDifMatrixSolver = nullptr;
Sebastian Aland's avatar
Sebastian Aland committed
442
443
444
445
446
447
448
449
450
451
452
453
454
       
       KSPDestroy(&(matShellContext.kspVelocity));
       if (regularizeLaplace)
        KSPDestroy(&(matShellContext.kspLaplace));
       
       MatDestroy(&(matShellContext.matGrad));
       MatDestroy(&(matShellContext.matDiv));
       MatDestroy(&(matShellContext.velocityMat));
       
       delete densityFunction;
       delete densityFunctionTau;
       delete viscosityFunction;
     }
455
  } }
Sebastian Aland's avatar
Sebastian Aland committed
456