PetscSolverSchur.cc 14.7 KB
Newer Older
Thomas Witkowski's avatar
Thomas Witkowski committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


#include "parallel/PetscSolverSchur.h"
#include "parallel/StdMpi.h"
#include "parallel/MpiHelper.h"

namespace AMDiS {

  using namespace std;

  void PetscSolverSchur::updateDofData(int nComponents)
  {
    FUNCNAME("PetscSolverSchur::updateDofData()");

    TEST_EXIT_DBG(meshDistributor)("No mesh distributor object defined!\n");
26
    TEST_EXIT_DBG(interiorMap)("No parallel DOF map defined!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
27

28
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
29
30
31
32
33
    typedef map<int, DofContainer> RankToDofContainer;
    typedef map<DegreeOfFreedom, bool> DofIndexToBool;

    boundaryDofs.clear();
    std::set<DegreeOfFreedom> boundaryLocalDofs;
34
    for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), feSpace);
35
36
37
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
	boundaryLocalDofs.insert(it.getDofIndex());	  
38
	boundaryDofs.insert((*interiorMap)[feSpace][it.getDofIndex()].global);
Thomas Witkowski's avatar
Thomas Witkowski committed
39
      }
40

41
      
Thomas Witkowski's avatar
Thomas Witkowski committed
42
    nBoundaryDofs = boundaryDofs.size();
43
    mpi::getDofNumbering(mpiCommGlobal, nBoundaryDofs, 
Thomas Witkowski's avatar
Thomas Witkowski committed
44
45
			 rStartBoundaryDofs, nOverallBoundaryDofs);

46

47
    DofContainerSet& edgeDofs = 
48
      meshDistributor->getBoundaryDofInfo(feSpace, 0).geoDofs[EDGE];
49
    DofContainerSet& vertexDofs = 
50
      meshDistributor->getBoundaryDofInfo(feSpace, 0).geoDofs[VERTEX];
51
52
53
54
55
56
57
    int nEdgeDofs = edgeDofs.size();
    int nVertexDofs = vertexDofs.size();

    TEST_EXIT_DBG(nEdgeDofs + nVertexDofs == nBoundaryDofs)
      ("Should not happen!\n");

    int rStartEdgeDofs, nOverallEdgeDofs;
58
    mpi::getDofNumbering(mpiCommGlobal, nEdgeDofs, 
59
60
61
			 rStartEdgeDofs, nOverallEdgeDofs);

    int rStartVertexDofs, nOverallVertexDofs;
62
    mpi::getDofNumbering(mpiCommGlobal, nVertexDofs, 
63
64
65
66
67
68
			 rStartVertexDofs, nOverallVertexDofs);

    TEST_EXIT_DBG(nOverallEdgeDofs + nOverallVertexDofs == nOverallBoundaryDofs)
      ("Should not happen!\n");
    

Thomas Witkowski's avatar
Thomas Witkowski committed
69
    mapGlobalBoundaryDof.clear();
70
71
72
73
74
#if 1
    {
      int counter = rStartEdgeDofs;
      for (DofContainerSet::iterator it = edgeDofs.begin();
	   it != edgeDofs.end(); ++it)
75
	mapGlobalBoundaryDof[(*interiorMap)[feSpace][**it].global] = 
76
77
78
79
80
81
	  counter++;
    }
    {
      int counter = nOverallEdgeDofs + rStartVertexDofs;
      for (DofContainerSet::iterator it = vertexDofs.begin();
	   it != vertexDofs.end(); ++it)
82
	mapGlobalBoundaryDof[(*interiorMap)[feSpace][**it].global] = 
83
84
85
86
87
	  counter++;
    }
#else
 {
    int counter = rStartBoundaryDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
88
89
90
    for (std::set<DegreeOfFreedom>::iterator it = boundaryDofs.begin();
	 it != boundaryDofs.end(); ++it)
      mapGlobalBoundaryDof[*it] = counter++;
91
92
 }
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
93
94
95


    std::set<DegreeOfFreedom> otherBoundaryLocalDofs;
96
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
97
98
99
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	otherBoundaryLocalDofs.insert(it.getDofIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
100
101
      
    interiorDofs.clear();
Thomas Witkowski's avatar
Thomas Witkowski committed
102
103
104

    ERROR_EXIT("Rewrite the following code block!\n");
#if 0
105
    DofIndexToBool& isRankDof = meshDistributor->getIsRankDof(feSpace);
Thomas Witkowski's avatar
Thomas Witkowski committed
106
107
108
109
110
    for (DofIndexToBool::iterator dofIt = isRankDof.begin(); 
	 dofIt != isRankDof.end(); ++dofIt) {
      if (dofIt->second && 
	  boundaryLocalDofs.count(dofIt->first) == 0 && 
	  otherBoundaryLocalDofs.count(dofIt->first) == 0)
111
	interiorDofs.insert(meshDistributor->mapDofToGlobal(feSpace, dofIt->first));
Thomas Witkowski's avatar
Thomas Witkowski committed
112
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
113
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
114
115
      
    nInteriorDofs = interiorDofs.size();
116
    mpi::getDofNumbering(mpiCommGlobal, nInteriorDofs, 
Thomas Witkowski's avatar
Thomas Witkowski committed
117
118
			 rStartInteriorDofs, nOverallInteriorDofs);

119
120
121
122
123
124
125
    {
      int counter = rStartInteriorDofs;
      mapGlobalInteriorDof.clear();
      for (std::set<DegreeOfFreedom>::iterator it = interiorDofs.begin();
	   it != interiorDofs.end(); ++it)
	mapGlobalInteriorDof[*it] = counter++;
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
126
127
128
129
130


    TEST_EXIT_DBG(nInteriorDofs > 0)("Should not happen!\n");


131
    StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiCommGlobal);
132
    for (DofComm::Iterator it(meshDistributor->getDofComm().getSendDofs(), feSpace);
133
134
135
136
137
	 !it.end(); it.nextRank()) {
      stdMpi.getSendData(it.getRank()).resize(0);
      stdMpi.getSendData(it.getRank()).reserve(it.getDofs().size());

      for (; !it.endDofIter(); it.nextDof()) {
138
	int globalSendDof = (*interiorMap)[feSpace][it.getDofIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
139
140
141
142

	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalSendDof))
	  ("No mapping for boundary DOF %d!\n", globalSendDof);

143
	stdMpi.getSendData(it.getRank()).push_back(mapGlobalBoundaryDof[globalSendDof]);
Thomas Witkowski's avatar
Thomas Witkowski committed
144
      }
145
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
146
147

    stdMpi.updateSendDataSize();
148

149
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
150
151
	 !it.end(); it.nextRank())
      stdMpi.recv(it.getRank());
Thomas Witkowski's avatar
Thomas Witkowski committed
152

153
    stdMpi.startCommunication();
154

155
    for (DofComm::Iterator it(meshDistributor->getDofComm().getRecvDofs(), feSpace);
156
157
	 !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof()) {
158
	int globalRecvDof = (*interiorMap)[feSpace][it.getDofIndex()].global;
159
160
	mapGlobalBoundaryDof[globalRecvDof] = 
	  stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
Thomas Witkowski's avatar
Thomas Witkowski committed
161
162
163
164
165
166
	boundaryDofs.insert(globalRecvDof);
      }


    // === Create PETSc IS structurs for interior and boundary DOFs. ===

167
    ISCreateStride(mpiCommGlobal, 
Thomas Witkowski's avatar
Thomas Witkowski committed
168
169
170
171
		   nInteriorDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs) * nComponents, 
		   1, &interiorIs);

172
    ISCreateStride(mpiCommGlobal,
Thomas Witkowski's avatar
Thomas Witkowski committed
173
174
175
176
177
178
		   nBoundaryDofs * nComponents,
		   (rStartInteriorDofs + rStartBoundaryDofs + nInteriorDofs) * nComponents, 
		   1, &boundaryIs);
  }


Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
179
  void PetscSolverSchur::fillPetscMatrix(Matrix<DOFMatrix*> *seqMat)
Thomas Witkowski's avatar
Thomas Witkowski committed
180
181
  {
    FUNCNAME("PetscSolverSchur::fillPetscMatrix()");
182

183
184
185
    petscData.init(interiorMap, coarseSpaceMap, 
		   subdomainLevel, mpiCommLocal, mpiCommGlobal,
		   meshDistributor);
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
186

187
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
188
    int nComponents = seqMat->getNumRows();
Thomas Witkowski's avatar
Thomas Witkowski committed
189
190
191
192
193
194
195
196
    updateDofData(nComponents);

    int nInteriorRows = nInteriorDofs * nComponents;
    int nOverallInteriorRows = nOverallInteriorDofs * nComponents;
    int nBoundaryRows = nBoundaryDofs * nComponents;
    int nOverallBoundaryRows = nOverallBoundaryDofs * nComponents;


197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
    MatCreateAIJ(mpiCommGlobal, 
		 nInteriorRows, nInteriorRows, 
		 nOverallInteriorRows, nOverallInteriorRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA11);

    MatCreateAIJ(mpiCommGlobal, 
		 nBoundaryRows, nBoundaryRows, 
		 nOverallBoundaryRows, nOverallBoundaryRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA22);

    MatCreateAIJ(mpiCommGlobal, 
		 nInteriorRows, nBoundaryRows, 
		 nOverallInteriorRows, nOverallBoundaryRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA12);    

    MatCreateAIJ(mpiCommGlobal, 
		 nBoundaryRows, nInteriorRows, 
		 nOverallBoundaryRows, nOverallInteriorRows,
		 100, PETSC_NULL, 100, PETSC_NULL, &matA21);
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217
218
219


    for (int i = 0; i < nComponents; i++)
      for (int j = 0; j < nComponents; j++)
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
220
221
	if ((*seqMat)[i][j])
	  setDofMatrix((*seqMat)[i][j], nComponents, i, j);
Thomas Witkowski's avatar
Thomas Witkowski committed
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244

    MatAssemblyBegin(matA11, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA11, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA12, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA12, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA21, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA21, MAT_FINAL_ASSEMBLY);

    MatAssemblyBegin(matA22, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(matA22, MAT_FINAL_ASSEMBLY);

    Mat tmpMat[2][2];
    tmpMat[0][0] = matA11;
    tmpMat[0][1] = matA12;
    tmpMat[1][0] = matA21;
    tmpMat[1][1] = matA22;

    IS tmpIS[2];
    tmpIS[0] = interiorIs;
    tmpIS[1] = boundaryIs;

245
246
247
    MatCreateNest(mpiCommGlobal, 2, &tmpIS[0], 2, &tmpIS[0], &tmpMat[0][0], 
		  &petscData.getInteriorMat());
    MatNestSetVecType(petscData.getInteriorMat(), VECNEST);
Thomas Witkowski's avatar
Thomas Witkowski committed
248

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
249
    petscData.matAssembly();
Thomas Witkowski's avatar
Thomas Witkowski committed
250

251
252
    int nRankRows = (*interiorMap)[feSpace].nRankDofs * nComponents;
    int nOverallRows = (*interiorMap)[feSpace].nOverallDofs * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
253

254
    VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &petscSolVec);
255
256
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
257

258
259
260
  void PetscSolverSchur::fillPetscRhs(SystemVector *vec)
  {
    FUNCNAME("PetscSolverSchur::fillPetscRhs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
261

262
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
263
    int nComponents = vec->getSize();
264
265
    int nRankRows = (*interiorMap)[feSpace].nRankDofs * nComponents;
    int nOverallRows = (*interiorMap)[feSpace].nOverallDofs * nComponents;
Thomas Witkowski's avatar
Thomas Witkowski committed
266

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
267
    VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &(getRhsInterior()));
Thomas Witkowski's avatar
Thomas Witkowski committed
268
269

    for (int i = 0; i < nComponents; i++)
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
270
      setDofVector(getRhsInterior(), vec->getDOFVector(i), nComponents, i);
Thomas Witkowski's avatar
Thomas Witkowski committed
271

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
272
    petscData.vecRhsAssembly();
Thomas Witkowski's avatar
Thomas Witkowski committed
273
274
275
  }


276
277
  void PetscSolverSchur::solvePetscMatrix(SystemVector &vec, 
					  AdaptInfo *adaptInfo)
Thomas Witkowski's avatar
Thomas Witkowski committed
278
279
280
  {
    FUNCNAME("PetscSolverSchur::solvePetscMatrix()");

281
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
282
283
    int nComponents = vec.getSize();

284
    KSPCreate(mpiCommGlobal, &kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
285

286
287
288
    KSPSetOperators(kspInterior, 
		    petscData.getInteriorMat(), 
		    petscData.getInteriorMat(), SAME_NONZERO_PATTERN); 
289
290
    KSPSetTolerances(kspInterior, 0.0, 1e-8, PETSC_DEFAULT, PETSC_DEFAULT);
    KSPSetFromOptions(kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
291

292
293
294
295
296
    KSPGetPC(kspInterior, &pcInterior);
    PCSetType(pcInterior, PCFIELDSPLIT);
    PCFieldSplitSetIS(pcInterior, "interior", interiorIs);
    PCFieldSplitSetIS(pcInterior, "boundary", boundaryIs);
    PCSetFromOptions(pcInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
297
298


Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
299
    KSPSolve(kspInterior, getRhsInterior(), petscSolVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
300
301
302
303
304
305
306
307
308

    // === Transfere values from PETSc's solution vectors to AMDiS vectors. ===

    PetscScalar *vecPointer;
    VecGetArray(petscSolVec, &vecPointer);

    for (int i = 0; i < nComponents; i++) {
      DOFVector<double>::Iterator dofIt(vec.getDOFVector(i), USED_DOFS);
      for (dofIt.reset(); !dofIt.end(); ++dofIt) {
309
	DegreeOfFreedom globalRowDof = (*interiorMap)[feSpace][dofIt.getDOFIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
	if (boundaryDofs.count(globalRowDof)) {
	  int index =
	    (mapGlobalBoundaryDof[globalRowDof] - rStartBoundaryDofs + nInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	} else {
	  int index = 
	    (mapGlobalInteriorDof[globalRowDof] - rStartInteriorDofs) * (i + 1);
	  *dofIt = vecPointer[index];
	}
      }
    }

    VecRestoreArray(petscSolVec, &vecPointer);


    // === Synchronize DOFs at common DOFs, i.e., DOFs that correspond to ===
    // === more than one partition.                                       ===
    meshDistributor->synchVector(vec);


    // === Destroy PETSC's variables. ===

332
333
334
335
    MatDestroy(&matA11);
    MatDestroy(&matA12);
    MatDestroy(&matA21);
    MatDestroy(&matA22);
336

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
337
338
    petscData.matDestroy();
    petscData.vecDestroy();
Thomas Witkowski's avatar
Thomas Witkowski committed
339

340
    KSPDestroy(&kspInterior);
Thomas Witkowski's avatar
Thomas Witkowski committed
341
342
343
  }
  

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
344
  void PetscSolverSchur::setDofMatrix(DOFMatrix* seqMat, int dispMult, 
Thomas Witkowski's avatar
Thomas Witkowski committed
345
346
347
348
				      int dispAddRow, int dispAddCol)
  {
    FUNCNAME("PetscSolverSchur::setDofMatrix()");

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
349
    TEST_EXIT(seqMat)("No DOFMatrix!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
350

351
    const FiniteElemSpace* feSpace = meshDistributor->getFeSpace(0);
Thomas Witkowski's avatar
Thomas Witkowski committed
352
353
354
355
    using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
356
357
    traits::col<Matrix>::type col(seqMat->getBaseMatrix());
    traits::const_value<Matrix>::type value(seqMat->getBaseMatrix());
Thomas Witkowski's avatar
Thomas Witkowski committed
358
359
360
361
362
363
364
365
366
367
368

    typedef traits::range_generator<row, Matrix>::type cursor_type;
    typedef traits::range_generator<nz, cursor_type>::type icursor_type;

    vector<int> colsBoundary, colsInterior;
    vector<double> valuesBoundary, valuesInterior;
    colsBoundary.reserve(300);
    colsInterior.reserve(300);
    valuesBoundary.reserve(300);
    valuesInterior.reserve(300);

Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
369
370
    for (cursor_type cursor = begin<row>(seqMat->getBaseMatrix()), 
	   cend = end<row>(seqMat->getBaseMatrix()); cursor != cend; ++cursor) {
Thomas Witkowski's avatar
Thomas Witkowski committed
371
372

      // Global index of the current row DOF.
373
      int globalRowDof = (*interiorMap)[feSpace][*cursor].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
374
375
376
377
378
379
380
381
     
      colsBoundary.clear();
      colsInterior.clear();
      valuesBoundary.clear();
      valuesInterior.clear();
      
      for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); 
	   icursor != icend; ++icursor) {
382
	int globalColDof = (*interiorMap)[feSpace][col(*icursor)].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436

	if (boundaryDofs.count(globalColDof)) {
	  TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalColDof))
	    ("Should not happen!\n");

	  int colIndex = 
	    mapGlobalBoundaryDof[globalColDof] * dispMult + dispAddCol;
	  
	  colsBoundary.push_back(colIndex);
	  valuesBoundary.push_back(value(*icursor));
	} else {
	  TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalColDof))
	    ("Cannot find global interior mapping for global column DOF %d!\n", 
	     globalColDof);

	  int colIndex = 
	    mapGlobalInteriorDof[globalColDof] * dispMult + dispAddCol;

	  colsInterior.push_back(colIndex);
	  valuesInterior.push_back(value(*icursor));
	}
      }

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int rowIndex = 
	  mapGlobalBoundaryDof[globalRowDof] * dispMult + dispAddRow;

 	MatSetValues(matA22, 1, &rowIndex, colsBoundary.size(), 
 		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
 	MatSetValues(matA21, 1, &rowIndex, colsInterior.size(), 
 		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Cannot find global interior mapping for global row DOF %d!\n", 
	   globalRowDof);

	int rowIndex = 
	  mapGlobalInteriorDof[globalRowDof] * dispMult + dispAddRow;

  	MatSetValues(matA11, 1, &rowIndex, colsInterior.size(), 
  		     &(colsInterior[0]), &(valuesInterior[0]), ADD_VALUES);
  	MatSetValues(matA12, 1, &rowIndex, colsBoundary.size(), 
  		     &(colsBoundary[0]), &(valuesBoundary[0]), ADD_VALUES);
      }
    }
  }


  void PetscSolverSchur::setDofVector(Vec& petscVec, DOFVector<double>* vec,
				      int dispMult, int dispAdd, bool rankOnly)
  {
437
438
    const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);

Thomas Witkowski's avatar
Thomas Witkowski committed
439
440
    DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
    for (dofIt.reset(); !dofIt.end(); ++dofIt) {
441
      if (rankOnly && !(*interiorMap)[feSpace].isRankDof(dofIt.getDOFIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
442
443
	continue;

444
      // Calculate global row index of the DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
445
      DegreeOfFreedom globalRowDof = 
446
	(*interiorMap)[feSpace][dofIt.getDOFIndex()].global;
Thomas Witkowski's avatar
Thomas Witkowski committed
447
448
449
450
451
452
453
454
455
456
      double value = *dofIt;

      if (boundaryDofs.count(globalRowDof)) {
	TEST_EXIT_DBG(mapGlobalBoundaryDof.count(globalRowDof))
	  ("Should not happen!\n");

	int index = 
	  (rStartInteriorDofs + 
	   nInteriorDofs +
	   mapGlobalBoundaryDof[globalRowDof]) * dispMult + dispAdd;
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
457
	VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
Thomas Witkowski's avatar
Thomas Witkowski committed
458
459
460
461
462
463
464
      } else {
	TEST_EXIT_DBG(mapGlobalInteriorDof.count(globalRowDof))
	  ("Should not happen!\n");
	
	int index = 
	  (rStartBoundaryDofs + 
	   mapGlobalInteriorDof[globalRowDof]) * dispMult + dispAdd;
Thomas Witkowski's avatar
Blub  
Thomas Witkowski committed
465
	VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
Thomas Witkowski's avatar
Thomas Witkowski committed
466
467
468
469
470
      }
    }
  }

}