PetscSolverSchur.cc 14.8 KB
Newer Older
Thomas Witkowski's avatar
Thomas Witkowski committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


#include "parallel/PetscSolverSchur.h"
#include "parallel/StdMpi.h"
#include "parallel/MpiHelper.h"

namespace AMDiS {

  using namespace std;

  void PetscSolverSchur::updateDofData(int nComponents)
  {
    FUNCNAME("PetscSolverSchur::updateDofData()");

    TEST_EXIT_DBG(meshDistributor)("No mesh distributor object defined!\n");
26
    TEST_EXIT_DBG(interiorMap)("No parallel DOF map defined!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
27

28
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
29
30
31
32
33
    typedef map<int, DofContainer> RankToDofContainer;
    typedef map<DegreeOfFreedom, bool> DofIndexToBool;

    boundaryDofs.clear();
    std::set<DegreeOfFreedom> boundaryLocalDofs;
34
    for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), feSpace);
35
36
37
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
	boundaryLocalDofs.insert(it.getDofIndex());	  
38
	boundaryDofs.insert((*interiorMap)[feSpace][it.getDofIndex()].global);
Thomas Witkowski's avatar
Thomas Witkowski committed
39
      }
40

41
      
Thomas Witkowski's avatar
Thomas Witkowski committed
42
    nBoundaryDofs = boundaryDofs.size();
43
    mpi::getDofNumbering(mpiCommGlobal, nBoundaryDofs, 
Thomas Witkowski's avatar
Thomas Witkowski committed
44
45
			 rStartBoundaryDofs, nOverallBoundaryDofs);

46

47
    DofContainerSet& edgeDofs = 
48
      meshDistributor->getBoundaryDofInfo(feSpace, 0).geoDofs[EDGE];
49
    DofContainerSet& vertexDofs = 
50
      meshDistributor->getBoundaryDofInfo(feSpace, 0).geoDofs[VERTEX];
51
52
53
54
55
56
57
    int nEdgeDofs = edgeDofs.size();
    int nVertexDofs = vertexDofs.size();

    TEST_EXIT_DBG(nEdgeDofs + nVertexDofs == nBoundaryDofs)
      ("Should not happen!\n");

    int rStartEdgeDofs, nOverallEdgeDofs;
58
    mpi::getDofNumbering(mpiCommGlobal, nEdgeDofs, 
59
60
61
			 rStartEdgeDofs, nOverallEdgeDofs);

    int rStartVertexDofs, nOverallVertexDofs;
62
    mpi::getDofNumbering(mpiCommGlobal, nVertexDofs, 
63
64
65
66
67
68
			 rStartVertexDofs, nOverallVertexDofs);

    TEST_EXIT_DBG(nOverallEdgeDofs + nOverallVertexDofs == nOverallBoundaryDofs)
      ("Should not happen!\n");
    

Thomas Witkowski's avatar
Thomas Witkowski committed
69
    mapGlobalBoundaryDof.clear();
70
71
72
73
74
#if 1
    {
      int counter = rStartEdgeDofs;
      for (DofContainerSet::iterator it = edgeDofs.begin();
	   it != edgeDofs.end(); ++it)
75
	mapGlobalBoundaryDof[(*interiorMap)[feSpace][**it].global] = 
76
77
78
79
80
81
	  counter++;
    }
    {
      int counter = nOverallEdgeDofs + rStartVertexDofs;
      for (DofContainerSet::iterator it = vertexDofs.begin();
	   it != vertexDofs.end(); ++it)
82
	mapGlobalBoundaryDof[(*interiorMap)[feSpace][**it].global] = 
83
84
85
86
87
	  counter++;
    }
#else
 {
    int counter = rStartBoundaryDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
88
89
90
    for (std::set<DegreeOfFreedom>::iterator it = boundaryDofs.begin();
	 it != boundaryDofs.end(); ++it)
      mapGlobalBoundaryDof[*it] = counter++;
91
92
 }
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
93
94
95


    std::set<DegreeOfFreedom> otherBoundaryLocalDofs;
96
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
97
98
99
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	otherBoundaryLocalDofs.insert(it.getDofIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
100
101
      
    interiorDofs.clear();
Thomas Witkowski's avatar
Thomas Witkowski committed
102
103
104

    ERROR_EXIT("Rewrite the following code block!\n");
#if 0
105
    DofIndexToBool& isRankDof = meshDistributor->getIsRankDof(feSpace);
Thomas Witkowski's avatar
Thomas Witkowski committed
106
107
108
109
110
    for (DofIndexToBool::iterator dofIt = isRankDof.begin(); 
	 dofIt != isRankDof.end(); ++dofIt) {
      if (dofIt->second && 
	  boundaryLocalDofs.count(dofIt->first) == 0 && 
	  otherBoundaryLocalDofs.count(dofIt->first) == 0)
111
	interiorDofs.insert(meshDistributor->mapDofToGlobal(feSpace, dofIt->first));
Thomas Witkowski's avatar
Thomas Witkowski committed
112
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
113
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
114
115
      
    nInteriorDofs = interiorDofs.size();
116
    mpi::getDofNumbering(mpiCommGlobal, nInteriorDofs, 
Thomas Witkowski's avatar
Thomas Witkowski committed
117
118
			 rStartInteriorDofs, nOverallInteriorDofs);

119
120
121
122
123
124
125
    {
      int counter = rStartInteriorDofs;
      mapGlobalInteriorDof.clear();
      for (std::set<DegreeOfFreedom>::iterator it = interiorDofs.begin();
	   it != interiorDofs.end(); ++it)
	mapGlobalInteriorDof[*it] = counter++;
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
126
127
128
129
130


    TEST_EXIT_DBG(nInteriorDofs > 0)("Should not happen!\n");


131
    StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiCommGlobal);
132
    for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), feSpace);
133
134
135
136
137
	 !it.end(); it.nextRank()) {
      stdMpi.getSendData(it.getRank()).resize(0);
      stdMpi.getSendData(it.getRank()).reserve(it.getDofs().size());

      for (; !it.endDofIter(); it.nextDof()) {
138
	int globalSendDof = (*interiorMap)[feSpace][it.getDofIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
139
140
141
142

	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalSendDof))
	  ("No mapping for boundary DOF %d!\n", globalSendDof);

143
	stdMpi.getSendData(it.getRank()).push_back(mapGlobalBoundaryDof[globalSendDof]);
Thomas Witkowski's avatar
Thomas Witkowski committed
144
      }
145
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
146
147

    stdMpi.updateSendDataSize();
148

149
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
150
151
	 !it.end(); it.nextRank())
      stdMpi.recv(it.getRank());
Thomas Witkowski's avatar
Thomas Witkowski committed
152

153
    stdMpi.startCommunication();
154

155
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
156
157
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
158
	int globalRecvDof = (*interiorMap)[feSpace][it.getDofIndex()].global;
159
160
	mapGlobalBoundaryDof[globalRecvDof] = 
	  stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
Thomas Witkowski's avatar
Thomas Witkowski committed
161
162
163
164
165
166
	boundaryDofs.insert(globalRecvDof);
      }


    // === Create PETSc IS structurs for interior and boundary DOFs. ===

167
    ISCreateStride(mpiCommGlobal, 
Thomas Witkowski's avatar
Thomas Witkowski committed
168
169
170
171
		   nInteriorDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs) * nComponents, 
		   1, &interiorIs);

172
    ISCreateStride(mpiCommGlobal,
Thomas Witkowski's avatar
Thomas Witkowski committed
173
174
175
176
177
178
		   nBoundaryDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs + nInteriorDofs) * nComponents, 
		   1, &boundaryIs);
  }


Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
179
  void PetscSolverSchur::fillPetscMatrix(Matrix<DOFMatrix*> *seqMat)
Thomas Witkowski's avatar
Thomas Witkowski committed
180
181
  {
    FUNCNAME("PetscSolverSchur::fillPetscMatrix()");
182

183
184
    petscData.create(interiorMap, coarseSpaceMap, 
		     subdomainLevel, mpiCommLocal, mpiCommGlobal);
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
185

186
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
187
    int nComponents = seqMat->getNumRows();
Thomas Witkowski's avatar
Thomas Witkowski committed
188
189
190
191
192
193
194
195
    updateDofData(nComponents);

    int nInteriorRows = nInteriorDofs * nComponents;
    int nOverallInteriorRows = nOverallInteriorDofs * nComponents;
    int nBoundaryRows = nBoundaryDofs * nComponents;
    int nOverallBoundaryRows = nOverallBoundaryDofs * nComponents;


196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    MatCreateAIJ(mpiCommGlobal, 
		 nInteriorRows, nInteriorRows, 
		 nOverallInteriorRows, nOverallInteriorRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA11);

    MatCreateAIJ(mpiCommGlobal, 
		 nBoundaryRows, nBoundaryRows, 
		 nOverallBoundaryRows, nOverallBoundaryRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA22);

    MatCreateAIJ(mpiCommGlobal, 
		 nInteriorRows, nBoundaryRows, 
		 nOverallInteriorRows, nOverallBoundaryRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA12);    

    MatCreateAIJ(mpiCommGlobal, 
		 nBoundaryRows, nInteriorRows, 
		 nOverallBoundaryRows, nOverallInteriorRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA21);
Thomas Witkowski's avatar
Thomas Witkowski committed
215
216
217
218


    for (int i = 0; i < nComponents; i++)
      for (int j = 0; j < nComponents; j++)
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
219
220
	if ((*seqMat)[i][j])
	  setDofMatrix((*seqMat)[i][j], nComponents, i, j);
Thomas Witkowski's avatar
Thomas Witkowski committed
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243

    MatAssemblyBegin(matA11, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA11, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA12, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA12, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA21, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA21, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA22, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA22, MAT_FINAL_ASSEMBLY);

    Mat tmpMat[2][2];
    tmpMat[0][0] = matA11;
    tmpMat[0][1] = matA12;
    tmpMat[1][0] = matA21;
    tmpMat[1][1] = matA22;

    IS tmpIS[2];
    tmpIS[0] = interiorIs;
    tmpIS[1] = boundaryIs;

244
245
246
    MatCreateNest(mpiCommGlobal, 2, &tmpIS[0], 2, &tmpIS[0], &tmpMat[0][0], 
		  &petscData.getInteriorMat());
    MatNestSetVecType(petscData.getInteriorMat(), VECNEST);
Thomas Witkowski's avatar
Thomas Witkowski committed
247

248
    petscData.assembly();
Thomas Witkowski's avatar
Thomas Witkowski committed
249

250
251
    int nRankRows = (*interiorMap)[feSpace].nRankDofs * nComponents;
    int nOverallRows = (*interiorMap)[feSpace].nOverallDofs * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
252

253
    VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &petscSolVec);
254
255
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
256

257
258
259
  void PetscSolverSchur::fillPetscRhs(SystemVector *vec)
  {
    FUNCNAME("PetscSolverSchur::fillPetscRhs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
260

261
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
262
    int nComponents = vec->getSize();
263
264
    int nRankRows = (*interiorMap)[feSpace].nRankDofs * nComponents;
    int nOverallRows = (*interiorMap)[feSpace].nOverallDofs * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
265

266
    VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &rhsInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
267
268

    for (int i = 0; i < nComponents; i++)
269
      setDofVector(rhsInterior, vec->getDOFVector(i), nComponents, i);
Thomas Witkowski's avatar
Thomas Witkowski committed
270

271
272
    VecAssemblyBegin(rhsInterior);
    VecAssemblyEnd(rhsInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
273
274
275
  }


276
277
  void PetscSolverSchur::solvePetscMatrix(SystemVector &vec, 
					  AdaptInfo *adaptInfo)
Thomas Witkowski's avatar
Thomas Witkowski committed
278
279
280
  {
    FUNCNAME("PetscSolverSchur::solvePetscMatrix()");

281
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
282
283
    int nComponents = vec.getSize();

284
    KSPCreate(mpiCommGlobal, &kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
285

286
287
288
    KSPSetOperators(kspInterior, 
		    petscData.getInteriorMat(), 
		    petscData.getInteriorMat(), SAME_NONZERO_PATTERN); 
289
290
    KSPSetTolerances(kspInterior, 0.0, 1e-8, PETSC_DEFAULT, PETSC_DEFAULT);
    KSPSetFromOptions(kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
291

292
293
294
295
296
    KSPGetPC(kspInterior, &pcInterior);
    PCSetType(pcInterior, PCFIELDSPLIT);
    PCFieldSplitSetIS(pcInterior, "interior", interiorIs);
    PCFieldSplitSetIS(pcInterior, "boundary", boundaryIs);
    PCSetFromOptions(pcInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
297
298


299
    KSPSolve(kspInterior, rhsInterior, petscSolVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
300
301
302
303
304
305
306
307
308

    // === Transfere values from PETSc's solution vectors to AMDiS vectors. ===

    PetscScalar *vecPointer;
    VecGetArray(petscSolVec, &vecPointer);

    for (int i = 0; i < nComponents; i++) {
      DOFVector<double>::Iterator dofIt(vec.getDOFVector(i), USED_DOFS);
      for (dofIt.reset(); !dofIt.end(); ++dofIt) {
309
	DegreeOfFreedom globalRowDof = (*interiorMap)[feSpace][dofIt.getDOFIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
	if (boundaryDofs.count(globalRowDof)) {
	  int index =
	    (mapGlobalBoundaryDof[globalRowDof] - rStartBoundaryDofs + nInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	} else {
	  int index = 
	    (mapGlobalInteriorDof[globalRowDof] - rStartInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	}
      }
    }

    VecRestoreArray(petscSolVec, &vecPointer);


    // === Synchronize DOFs at common DOFs, i.e., DOFs that correspond to ===
    // === more than one partition.                                       ===
    meshDistributor->synchVector(vec);


    // === Destroy PETSC's variables. ===

332
    VecDestroy(&rhsInterior);
333
    VecDestroy(&petscSolVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
334

335
336
337
338
    MatDestroy(&matA11);
    MatDestroy(&matA12);
    MatDestroy(&matA21);
    MatDestroy(&matA22);
339
340

    petscData.destroy();
Thomas Witkowski's avatar
Thomas Witkowski committed
341

342
    KSPDestroy(&kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
343
344
345
  }
  

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
346
  void PetscSolverSchur::setDofMatrix(DOFMatrix* seqMat, int dispMult, 
Thomas Witkowski's avatar
Thomas Witkowski committed
347
348
349
350
				      int dispAddRow, int dispAddCol)
  {
    FUNCNAME("PetscSolverSchur::setDofMatrix()");

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
351
    TEST_EXIT(seqMat)("No DOFMatrix!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
352

353
    const FiniteElemSpace* feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
354
355
356
357
    using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
358
359
    traits::col<Matrix>::type col(seqMat->getBaseMatrix());
    traits::const_value<Matrix>::type value(seqMat->getBaseMatrix());
Thomas Witkowski's avatar
Thomas Witkowski committed
360
361
362
363
364
365
366
367
368
369
370

    typedef traits::range_generator<row, Matrix>::type cursor_type;
    typedef traits::range_generator<nz, cursor_type>::type icursor_type;

    vector<int> colsBoundary, colsInterior;
    vector<double> valuesBoundary, valuesInterior;
    colsBoundary.reserve(300);
    colsInterior.reserve(300);
    valuesBoundary.reserve(300);
    valuesInterior.reserve(300);

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
371
372
    for (cursor_type cursor = begin<row>(seqMat->getBaseMatrix()), 
	   cend = end<row>(seqMat->getBaseMatrix()); cursor != cend; ++cursor) {
Thomas Witkowski's avatar
Thomas Witkowski committed
373
374

      // Global index of the current row DOF.
375
      int globalRowDof = (*interiorMap)[feSpace][*cursor].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
376
377
378
379
380
381
382
383
     
      colsBoundary.clear();
      colsInterior.clear();
      valuesBoundary.clear();
      valuesInterior.clear();
      
      for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); 
	   icursor != icend; ++icursor) {
384
	int globalColDof = (*interiorMap)[feSpace][col(*icursor)].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438

	if (boundaryDofs.count(globalColDof)) {
	  TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalColDof))
	    ("Should not happen!\n");

	  int colIndex = 
	    mapGlobalBoundaryDof[globalColDof] * dispMult + dispAddCol;
	  
	  colsBoundary.push_back(colIndex);
	  valuesBoundary.push_back(value(*icursor));
	} else {
	  TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalColDof))
	    ("Cannot find global interior mapping for global column DOF %d!\n", 
	     globalColDof);

	  int colIndex = 
	    mapGlobalInteriorDof[globalColDof] * dispMult + dispAddCol;

	  colsInterior.push_back(colIndex);
	  valuesInterior.push_back(value(*icursor));
	}
      }

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int rowIndex = 
	  mapGlobalBoundaryDof[globalRowDof] * dispMult + dispAddRow;

 	MatSetValues(matA22, 1, &rowIndex, colsBoundary.size(), 
 		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
 	MatSetValues(matA21, 1, &rowIndex, colsInterior.size(), 
 		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Cannot find global interior mapping for global row DOF %d!\n", 
	   globalRowDof);

	int rowIndex = 
	  mapGlobalInteriorDof[globalRowDof] * dispMult + dispAddRow;

  	MatSetValues(matA11, 1, &rowIndex, colsInterior.size(), 
  		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
  	MatSetValues(matA12, 1, &rowIndex, colsBoundary.size(), 
  		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
      }
    }
  }


  void PetscSolverSchur::setDofVector(Vec& petscVec, DOFVector<double>* vec,
				      int dispMult, int dispAdd, bool rankOnly)
  {
439
440
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);

Thomas Witkowski's avatar
Thomas Witkowski committed
441
442
    DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
    for (dofIt.reset(); !dofIt.end(); ++dofIt) {
443
      if (rankOnly && !(*interiorMap)[feSpace].isRankDof(dofIt.getDOFIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
444
445
	continue;

446
      // Calculate global row index of the DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
447
      DegreeOfFreedom globalRowDof = 
448
	(*interiorMap)[feSpace][dofIt.getDOFIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
449
450
451
452
453
454
455
456
457
458
      double value = *dofIt;

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int index = 
	  (rStartInteriorDofs + 
	   nInteriorDofs +
	   mapGlobalBoundaryDof[globalRowDof]) * dispMult + dispAdd;
459
	VecSetValues(rhsInterior, 1, &index, &value, ADD_VALUES);
Thomas Witkowski's avatar
Thomas Witkowski committed
460
461
462
463
464
465
466
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Should not happen!\n");
	
	int index = 
	  (rStartBoundaryDofs + 
	   mapGlobalInteriorDof[globalRowDof]) * dispMult + dispAdd;
467
	VecSetValues(rhsInterior, 1, &index, &value, ADD_VALUES);
Thomas Witkowski's avatar
Thomas Witkowski committed
468
469
470
471
472
      }
    }
  }

}