PetscSolverSchur.cc 14.7 KB
Newer Older
Thomas Witkowski's avatar
Thomas Witkowski committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


#include "parallel/PetscSolverSchur.h"
#include "parallel/StdMpi.h"
#include "parallel/MpiHelper.h"

namespace AMDiS {

  using namespace std;

  void PetscSolverSchur::updateDofData(int nComponents)
  {
    FUNCNAME("PetscSolverSchur::updateDofData()");

    TEST_EXIT_DBG(meshDistributor)("No mesh distributor object defined!\n");

    MPI::Intracomm& mpiComm = meshDistributor->getMpiComm();
28
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
29
30
31
32
33
34

    typedef map<int, DofContainer> RankToDofContainer;
    typedef map<DegreeOfFreedom, bool> DofIndexToBool;

    boundaryDofs.clear();
    std::set<DegreeOfFreedom> boundaryLocalDofs;
35
36
37
38
    for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
	boundaryLocalDofs.insert(it.getDofIndex());	  
39
	boundaryDofs.insert(meshDistributor->mapDofToGlobal(feSpace, it.getDofIndex()));
Thomas Witkowski's avatar
Thomas Witkowski committed
40
      }
41

42
      
Thomas Witkowski's avatar
Thomas Witkowski committed
43
44
45
46
    nBoundaryDofs = boundaryDofs.size();
    mpi::getDofNumbering(mpiComm, nBoundaryDofs, 
			 rStartBoundaryDofs, nOverallBoundaryDofs);

47

48
49
50
51
    DofContainerSet& edgeDofs = 
      meshDistributor->getBoundaryDofInfo(feSpace).geoDofs[EDGE];
    DofContainerSet& vertexDofs = 
      meshDistributor->getBoundaryDofInfo(feSpace).geoDofs[VERTEX];
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    int nEdgeDofs = edgeDofs.size();
    int nVertexDofs = vertexDofs.size();

    TEST_EXIT_DBG(nEdgeDofs + nVertexDofs == nBoundaryDofs)
      ("Should not happen!\n");

    int rStartEdgeDofs, nOverallEdgeDofs;
    mpi::getDofNumbering(mpiComm, nEdgeDofs, 
			 rStartEdgeDofs, nOverallEdgeDofs);

    int rStartVertexDofs, nOverallVertexDofs;
    mpi::getDofNumbering(mpiComm, nVertexDofs, 
			 rStartVertexDofs, nOverallVertexDofs);

    TEST_EXIT_DBG(nOverallEdgeDofs + nOverallVertexDofs == nOverallBoundaryDofs)
      ("Should not happen!\n");
    

Thomas Witkowski's avatar
Thomas Witkowski committed
70
    mapGlobalBoundaryDof.clear();
71
72
73
74
75
#if 1
    {
      int counter = rStartEdgeDofs;
      for (DofContainerSet::iterator it = edgeDofs.begin();
	   it != edgeDofs.end(); ++it)
76
	mapGlobalBoundaryDof[meshDistributor->mapDofToGlobal(feSpace, **it)] = 
77
78
79
80
81
82
	  counter++;
    }
    {
      int counter = nOverallEdgeDofs + rStartVertexDofs;
      for (DofContainerSet::iterator it = vertexDofs.begin();
	   it != vertexDofs.end(); ++it)
83
	mapGlobalBoundaryDof[meshDistributor->mapDofToGlobal(feSpace, **it)] = 
84
85
86
87
88
	  counter++;
    }
#else
 {
    int counter = rStartBoundaryDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
89
90
91
    for (std::set<DegreeOfFreedom>::iterator it = boundaryDofs.begin();
	 it != boundaryDofs.end(); ++it)
      mapGlobalBoundaryDof[*it] = counter++;
92
93
 }
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
94
95
96


    std::set<DegreeOfFreedom> otherBoundaryLocalDofs;
97
98
99
100
    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	otherBoundaryLocalDofs.insert(it.getDofIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
101
102
      
    interiorDofs.clear();
103
    DofIndexToBool& isRankDof = meshDistributor->getIsRankDof(feSpace);
Thomas Witkowski's avatar
Thomas Witkowski committed
104
105
106
107
108
    for (DofIndexToBool::iterator dofIt = isRankDof.begin(); 
	 dofIt != isRankDof.end(); ++dofIt) {
      if (dofIt->second && 
	  boundaryLocalDofs.count(dofIt->first) == 0 && 
	  otherBoundaryLocalDofs.count(dofIt->first) == 0)
109
	interiorDofs.insert(meshDistributor->mapDofToGlobal(feSpace, dofIt->first));
Thomas Witkowski's avatar
Thomas Witkowski committed
110
111
112
113
114
115
    }
      
    nInteriorDofs = interiorDofs.size();
    mpi::getDofNumbering(mpiComm, nInteriorDofs, 
			 rStartInteriorDofs, nOverallInteriorDofs);

116
117
118
119
120
121
122
    {
      int counter = rStartInteriorDofs;
      mapGlobalInteriorDof.clear();
      for (std::set<DegreeOfFreedom>::iterator it = interiorDofs.begin();
	   it != interiorDofs.end(); ++it)
	mapGlobalInteriorDof[*it] = counter++;
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
123
124
125
126
127


    TEST_EXIT_DBG(nInteriorDofs > 0)("Should not happen!\n");


128
    StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
129
130
131
132
133
134
135
    for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
	 !it.end(); it.nextRank()) {
      stdMpi.getSendData(it.getRank()).resize(0);
      stdMpi.getSendData(it.getRank()).reserve(it.getDofs().size());

      for (; !it.endDofIter(); it.nextDof()) {
	int globalSendDof = 
136
	  meshDistributor->mapDofToGlobal(feSpace, it.getDofIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
137
138
139
140

	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalSendDof))
	  ("No mapping for boundary DOF %d!\n", globalSendDof);

141
	stdMpi.getSendData(it.getRank()).push_back(mapGlobalBoundaryDof[globalSendDof]);
Thomas Witkowski's avatar
Thomas Witkowski committed
142
      }
143
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
144
145

    stdMpi.updateSendDataSize();
146

147
148
149
    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
	 !it.end(); it.nextRank())
      stdMpi.recv(it.getRank());
Thomas Witkowski's avatar
Thomas Witkowski committed
150

151
    stdMpi.startCommunication();
152

153
154
155
156
    for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
	int globalRecvDof = 
157
	  meshDistributor->mapDofToGlobal(feSpace, it.getDofIndex());
158
159
	mapGlobalBoundaryDof[globalRecvDof] = 
	  stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
Thomas Witkowski's avatar
Thomas Witkowski committed
160
161
162
163
164
165
	boundaryDofs.insert(globalRecvDof);
      }


    // === Create PETSc IS structurs for interior and boundary DOFs. ===

166
    ISCreateStride(mpiComm, 
Thomas Witkowski's avatar
Thomas Witkowski committed
167
168
169
170
		   nInteriorDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs) * nComponents, 
		   1, &interiorIs);

171
    ISCreateStride(mpiComm,
Thomas Witkowski's avatar
Thomas Witkowski committed
172
173
174
175
176
177
		   nBoundaryDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs + nInteriorDofs) * nComponents, 
		   1, &boundaryIs);
  }


178
  void PetscSolverSchur::fillPetscMatrix(Matrix<DOFMatrix*> *mat)
Thomas Witkowski's avatar
Thomas Witkowski committed
179
180
  {
    FUNCNAME("PetscSolverSchur::fillPetscMatrix()");
181
182

    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
183
184
185
186
187
188
189
190
191
    int nComponents = mat->getNumRows();
    updateDofData(nComponents);

    int nInteriorRows = nInteriorDofs * nComponents;
    int nOverallInteriorRows = nOverallInteriorDofs * nComponents;
    int nBoundaryRows = nBoundaryDofs * nComponents;
    int nOverallBoundaryRows = nOverallBoundaryDofs * nComponents;


192
    MatCreateMPIAIJ(mpiComm, 
Thomas Witkowski's avatar
Thomas Witkowski committed
193
194
195
196
		    nInteriorRows, nInteriorRows, 
		    nOverallInteriorRows, nOverallInteriorRows,
		    100, PETSC_NULL, 100, PETSC_NULL, &matA11);

197
    MatCreateMPIAIJ(mpiComm, 
Thomas Witkowski's avatar
Thomas Witkowski committed
198
199
200
201
		    nBoundaryRows, nBoundaryRows, 
		    nOverallBoundaryRows, nOverallBoundaryRows,
		    100, PETSC_NULL, 100, PETSC_NULL, &matA22);

202
    MatCreateMPIAIJ(mpiComm, 
Thomas Witkowski's avatar
Thomas Witkowski committed
203
204
205
206
		    nInteriorRows, nBoundaryRows, 
		    nOverallInteriorRows, nOverallBoundaryRows,
		    100, PETSC_NULL, 100, PETSC_NULL, &matA12);    

207
    MatCreateMPIAIJ(mpiComm, 
Thomas Witkowski's avatar
Thomas Witkowski committed
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
		    nBoundaryRows, nInteriorRows, 
		    nOverallBoundaryRows, nOverallInteriorRows,
		    100, PETSC_NULL, 100, PETSC_NULL, &matA21);


    for (int i = 0; i < nComponents; i++)
      for (int j = 0; j < nComponents; j++)
	if ((*mat)[i][j])
	  setDofMatrix((*mat)[i][j], nComponents, i, j);

    MatAssemblyBegin(matA11, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA11, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA12, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA12, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA21, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA21, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA22, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA22, MAT_FINAL_ASSEMBLY);

    Mat tmpMat[2][2];
    tmpMat[0][0] = matA11;
    tmpMat[0][1] = matA12;
    tmpMat[1][0] = matA21;
    tmpMat[1][1] = matA22;

    IS tmpIS[2];
    tmpIS[0] = interiorIs;
    tmpIS[1] = boundaryIs;

240
    MatCreateNest(mpiComm, 2, &tmpIS[0], 2, &tmpIS[0], &tmpMat[0][0], &petscMatrix);
Thomas Witkowski's avatar
Thomas Witkowski committed
241
242
243
244
245
    MatNestSetVecType(petscMatrix, VECNEST);
    MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);


246
247
    int nRankRows = meshDistributor->getNumberRankDofs(feSpace) * nComponents;
    int nOverallRows = meshDistributor->getNumberOverallDofs(feSpace) * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
248

249
250
    VecCreateMPI(mpiComm, nRankRows, nOverallRows, &petscSolVec);
    VecCreateMPI(mpiComm, nRankRows, nOverallRows, &petscTmpVec);
251
252
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
253

254
255
256
  void PetscSolverSchur::fillPetscRhs(SystemVector *vec)
  {
    FUNCNAME("PetscSolverSchur::fillPetscRhs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
257

258
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
259
    int nComponents = vec->getSize();
260
261
    int nRankRows = meshDistributor->getNumberRankDofs(feSpace) * nComponents;
    int nOverallRows = meshDistributor->getNumberOverallDofs(feSpace) * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
262

263
    VecCreateMPI(mpiComm, nRankRows, nOverallRows, &petscRhsVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
264
265
266
267
268
269
270
271
272

    for (int i = 0; i < nComponents; i++)
      setDofVector(petscRhsVec, vec->getDOFVector(i), nComponents, i);

    VecAssemblyBegin(petscRhsVec);
    VecAssemblyEnd(petscRhsVec);
  }


273
274
  void PetscSolverSchur::solvePetscMatrix(SystemVector &vec, 
					  AdaptInfo *adaptInfo)
Thomas Witkowski's avatar
Thomas Witkowski committed
275
276
277
  {
    FUNCNAME("PetscSolverSchur::solvePetscMatrix()");

278
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
279
280
    int nComponents = vec.getSize();

281
    KSPCreate(mpiComm, &solver);
Thomas Witkowski's avatar
Thomas Witkowski committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304

    KSPSetOperators(solver, petscMatrix, petscMatrix, SAME_NONZERO_PATTERN); 
    KSPSetTolerances(solver, 0.0, 1e-8, PETSC_DEFAULT, PETSC_DEFAULT);
    KSPSetFromOptions(solver);

    KSPGetPC(solver, &pc);
    PCSetType(pc, PCFIELDSPLIT);
    PCFieldSplitSetIS(pc, "interior", interiorIs);
    PCFieldSplitSetIS(pc, "boundary", boundaryIs);
    PCSetFromOptions(pc);


    KSPSolve(solver, petscRhsVec, petscSolVec);

    // === Transfere values from PETSc's solution vectors to AMDiS vectors. ===

    PetscScalar *vecPointer;
    VecGetArray(petscSolVec, &vecPointer);

    for (int i = 0; i < nComponents; i++) {
      DOFVector<double>::Iterator dofIt(vec.getDOFVector(i), USED_DOFS);
      for (dofIt.reset(); !dofIt.end(); ++dofIt) {
	DegreeOfFreedom globalRowDof = 
305
	  meshDistributor->mapDofToGlobal(feSpace, dofIt.getDOFIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
	if (boundaryDofs.count(globalRowDof)) {
	  int index =
	    (mapGlobalBoundaryDof[globalRowDof] - rStartBoundaryDofs + nInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	} else {
	  int index = 
	    (mapGlobalInteriorDof[globalRowDof] - rStartInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	}
      }
    }

    VecRestoreArray(petscSolVec, &vecPointer);


    // === Synchronize DOFs at common DOFs, i.e., DOFs that correspond to ===
    // === more than one partition.                                       ===
    meshDistributor->synchVector(vec);



    // === Print iteration counter and residual norm of the solution. ===
    printSolutionInfo(adaptInfo);


    // === Destroy PETSC's variables. ===

333
334
335
    VecDestroy(&petscRhsVec);
    VecDestroy(&petscSolVec);
    VecDestroy(&petscTmpVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
336

337
338
339
340
341
    MatDestroy(&matA11);
    MatDestroy(&matA12);
    MatDestroy(&matA21);
    MatDestroy(&matA22);
    MatDestroy(&petscMatrix);
Thomas Witkowski's avatar
Thomas Witkowski committed
342

343
    KSPDestroy(&solver);
Thomas Witkowski's avatar
Thomas Witkowski committed
344
345
346
347
348
349
350
351
352
353
  }
  

  void PetscSolverSchur::setDofMatrix(DOFMatrix* mat, int dispMult, 
				      int dispAddRow, int dispAddCol)
  {
    FUNCNAME("PetscSolverSchur::setDofMatrix()");

    TEST_EXIT(mat)("No DOFMatrix!\n");

354
    const FiniteElemSpace* feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
    using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

    traits::col<Matrix>::type col(mat->getBaseMatrix());
    traits::const_value<Matrix>::type value(mat->getBaseMatrix());

    typedef traits::range_generator<row, Matrix>::type cursor_type;
    typedef traits::range_generator<nz, cursor_type>::type icursor_type;

    vector<int> colsBoundary, colsInterior;
    vector<double> valuesBoundary, valuesInterior;
    colsBoundary.reserve(300);
    colsInterior.reserve(300);
    valuesBoundary.reserve(300);
    valuesInterior.reserve(300);

    for (cursor_type cursor = begin<row>(mat->getBaseMatrix()), 
	   cend = end<row>(mat->getBaseMatrix()); cursor != cend; ++cursor) {

      // Global index of the current row DOF.
376
      int globalRowDof = meshDistributor->mapDofToGlobal(feSpace, *cursor);
Thomas Witkowski's avatar
Thomas Witkowski committed
377
378
379
380
381
382
383
384
     
      colsBoundary.clear();
      colsInterior.clear();
      valuesBoundary.clear();
      valuesInterior.clear();
      
      for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); 
	   icursor != icend; ++icursor) {
385
	int globalColDof = meshDistributor->mapDofToGlobal(feSpace, col(*icursor));
Thomas Witkowski's avatar
Thomas Witkowski committed
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439

	if (boundaryDofs.count(globalColDof)) {
	  TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalColDof))
	    ("Should not happen!\n");

	  int colIndex = 
	    mapGlobalBoundaryDof[globalColDof] * dispMult + dispAddCol;
	  
	  colsBoundary.push_back(colIndex);
	  valuesBoundary.push_back(value(*icursor));
	} else {
	  TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalColDof))
	    ("Cannot find global interior mapping for global column DOF %d!\n", 
	     globalColDof);

	  int colIndex = 
	    mapGlobalInteriorDof[globalColDof] * dispMult + dispAddCol;

	  colsInterior.push_back(colIndex);
	  valuesInterior.push_back(value(*icursor));
	}
      }

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int rowIndex = 
	  mapGlobalBoundaryDof[globalRowDof] * dispMult + dispAddRow;

 	MatSetValues(matA22, 1, &rowIndex, colsBoundary.size(), 
 		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
 	MatSetValues(matA21, 1, &rowIndex, colsInterior.size(), 
 		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Cannot find global interior mapping for global row DOF %d!\n", 
	   globalRowDof);

	int rowIndex = 
	  mapGlobalInteriorDof[globalRowDof] * dispMult + dispAddRow;

  	MatSetValues(matA11, 1, &rowIndex, colsInterior.size(), 
  		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
  	MatSetValues(matA12, 1, &rowIndex, colsBoundary.size(), 
  		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
      }
    }
  }


  void PetscSolverSchur::setDofVector(Vec& petscVec, DOFVector<double>* vec,
				      int dispMult, int dispAdd, bool rankOnly)
  {
440
441
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);

Thomas Witkowski's avatar
Thomas Witkowski committed
442
443
    DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
    for (dofIt.reset(); !dofIt.end(); ++dofIt) {
444
      if (rankOnly && !meshDistributor->getIsRankDof(feSpace, dofIt.getDOFIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
445
446
	continue;

447
      // Calculate global row index of the DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
448
      DegreeOfFreedom globalRowDof = 
449
	meshDistributor->mapDofToGlobal(feSpace, dofIt.getDOFIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474

      double value = *dofIt;

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int index = 
	  (rStartInteriorDofs + 
	   nInteriorDofs +
	   mapGlobalBoundaryDof[globalRowDof]) * dispMult + dispAdd;
	VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES);
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Should not happen!\n");
	
	int index = 
	  (rStartBoundaryDofs + 
	   mapGlobalInteriorDof[globalRowDof]) * dispMult + dispAdd;
	VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES);
      }
    }
  }

}