ParallelDomainBase.cc 50.7 KB
Newer Older
Thomas Witkowski's avatar
Thomas Witkowski committed
1
2
#include <algorithm>

3
#include "ParallelDomainBase.h"
4
5
6
7
8
9
10
#include "ParMetisPartitioner.h"
#include "Mesh.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
#include "MacroElement.h"
#include "PartitionElementData.h"
11
12
#include "DOFMatrix.h"
#include "DOFVector.h"
13
#include "SystemVector.h"
14
#include "VtkWriter.h"
15
#include "ElementDofIterator.h"
16
17
#include "ProblemStatBase.h"
#include "StandardProblemIteration.h"
18
#include "ElementFileWriter.h"
19
20

#include "petscksp.h"
21
22
23

namespace AMDiS {

24
  PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
25
  {    
26
    if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0)
Thomas Witkowski's avatar
Thomas Witkowski committed
27
      std::cout << "  Petsc-Iteration " << iter << ": " << rnorm << std::endl;
28
29
30
31

    return 0;
  }

32
33
34
35
36
  inline bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2)
  {
    return (*dof1 < *dof2);
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
37
  ParallelDomainBase::ParallelDomainBase(std::string name,
38
39
40
41
					 ProblemIterationInterface *iIF,
					 ProblemTimeInterface *tIF,
					 FiniteElemSpace *fe,
					 RefinementManager *refineManager)
42
43
    : iterationIF(iIF),
      timeIF(tIF),
44
45
      feSpace(fe),
      mesh(fe->getMesh()),
46
      refinementManager(refineManager),
47
      initialPartitionMesh(true),
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
48
      nRankDOFs(0),
49
50
      rstart(0),
      nComponents(1)
51
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
52
53
54
55
56
57
58
    FUNCNAME("ParallelDomainBase::ParalleDomainBase()");

    TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
      ("Only meshes with one DOFAdmin are supported!\n");
    TEST_EXIT(mesh->getDOFAdmin(0).getNumberOfPreDOFs(0) == 0)
      ("Wrong pre dof number for DOFAdmin!\n");

59
60
61
62
63
64
    mpiRank = MPI::COMM_WORLD.Get_rank();
    mpiSize = MPI::COMM_WORLD.Get_size();
    mpiComm = MPI::COMM_WORLD;
    partitioner = new ParMetisPartitioner(mesh, &mpiComm);
  }

65
  void ParallelDomainBase::initParallelization(AdaptInfo *adaptInfo)
66
67
68
69
  {
    if (mpiSize <= 1)
      return;

70
71
72
73
74
    // Test, if the mesh is the macro mesh only! Paritioning of the mesh is supported
    // only for macro meshes, so it will not work yet if the mesh is already refined
    // in some way.
    testForMacroMesh();

75
76
77
78
79
80
81
    // create an initial partitioning of the mesh
    partitioner->createPartitionData();
    // set the element weights, which are 1 at the very first begin
    setElemWeights(adaptInfo);
    // and now partition the mesh
    partitionMesh(adaptInfo);   

Thomas Witkowski's avatar
Thomas Witkowski committed
82
83
84
85
#if (DEBUG != 0)
    ElementIdxToDofs elMap;
    DbgCreateElementMap(elMap);
#endif
86

87
    // === Create new global and local DOF numbering. ===
88

89
90
91
    // Set of all DOFs of the rank.
    std::vector<const DegreeOfFreedom*> rankDOFs;
    // Number of DOFs in ranks partition that are owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
92
    nRankDOFs = 0;
93
    // Number of all DOFs in the macro mesh.
94
    int nOverallDOFs = 0;
95

96
    createLocalGlobalNumbering(rankDOFs, nRankDOFs, nOverallDOFs);
97

Thomas Witkowski's avatar
Thomas Witkowski committed
98
99
    // === Create interior boundary information ===

100
    createInteriorBoundaryInfo(rankDOFs);
Thomas Witkowski's avatar
Thomas Witkowski committed
101

Thomas Witkowski's avatar
n    
Thomas Witkowski committed
102
103
104
105
    // === Remove all macro elements that are not part of the rank partition. ===

    removeMacroElements();

Thomas Witkowski's avatar
Thomas Witkowski committed
106
#if (DEBUG != 0)
Thomas Witkowski's avatar
Thomas Witkowski committed
107
    DbgTestElementMap(elMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
108
109
    DbgTestInteriorBoundary();
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
110

111

112
    // === Reset all DOFAdmins of the mesh. ===
113

114
    updateDofAdmins();
115

116

117
    // === Global refinements. ===
Thomas Witkowski's avatar
Thomas Witkowski committed
118

Thomas Witkowski's avatar
Thomas Witkowski committed
119
120
121
    int globalRefinement = 0;
    GET_PARAMETER(0, "testgr", "%d", &globalRefinement);

Thomas Witkowski's avatar
Thomas Witkowski committed
122
123
    if (globalRefinement > 0) {
      refinementManager->globalRefine(mesh, globalRefinement);
124

125
126
127
128
129
#if (DEBUG != 0)
      elMap.clear();
      DbgCreateElementMap(elMap);
#endif

Thomas Witkowski's avatar
Thomas Witkowski committed
130
      updateLocalGlobalNumbering(nRankDOFs, nOverallDOFs);
131
132
133
134
135
136

      updateDofAdmins();

#if (DEBUG != 0)
      DbgTestElementMap(elMap);
#endif
Thomas Witkowski's avatar
Thomas Witkowski committed
137
138
    }

139

Thomas Witkowski's avatar
Thomas Witkowski committed
140
#if (DEBUG != 0)
141
    DbgTestCommonDofs(true);
Thomas Witkowski's avatar
Thomas Witkowski committed
142
#endif
143

144

145
    // === Create petsc matrix. ===
146

147
148
149
    nRankRows = nRankDOFs * nComponents;
    nOverallRows = nOverallDOFs * nComponents;
    
150
151
152
153
154
155
156
    VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
    VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
    VecSetType(petscRhsVec, VECMPI);

    VecCreate(PETSC_COMM_WORLD, &petscSolVec);
    VecSetSizes(petscSolVec, nRankRows, nOverallRows);
    VecSetType(petscSolVec, VECMPI);
157
158
159
160

    VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
    VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
    VecSetType(petscTmpVec, VECMPI);
161
162
  }

163
164
165

  void ParallelDomainBase::exitParallelization(AdaptInfo *adaptInfo)
  {
166
167
    VecDestroy(petscRhsVec);
    VecDestroy(petscSolVec);
168
    VecDestroy(petscTmpVec);
169
  }
170

171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  
  void ParallelDomainBase::updateDofAdmins()
  {
    int nAdmins = mesh->getNumberOfDOFAdmin();
    for (int i = 0; i < nAdmins; i++) {
      DOFAdmin& admin = const_cast<DOFAdmin&>(mesh->getDOFAdmin(i));
      
      for (int j = 0; j < admin.getSize(); j++)
	admin.setDOFFree(j, true);
      for (int j = 0; j < static_cast<int>(mapLocalGlobalDOFs.size()); j++)
 	admin.setDOFFree(j, false);

      admin.setUsedSize(mapLocalGlobalDOFs.size());
      admin.setUsedCount(mapLocalGlobalDOFs.size());
      admin.setFirstHole(mapLocalGlobalDOFs.size());
    }
  }

189

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
  void ParallelDomainBase::testForMacroMesh()
  {
    FUNCNAME("ParallelDomainBase::testForMacroMesh()");

    int nMacroElements = 0;

    TraverseStack stack;
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
    while (elInfo) {
      TEST_EXIT(elInfo->getLevel() == 0)
	("Mesh is already refined! This does nor work with parallelization!\n");
      
      nMacroElements++;

      elInfo = stack.traverseNext(elInfo);
    }

    TEST_EXIT(nMacroElements >= mpiSize)
      ("The mesh has less macro elements than number of mpi processes!\n");
  }

211
212
  void ParallelDomainBase::setDofMatrix(DOFMatrix* mat, int dispMult, 
					int dispAddRow, int dispAddCol)
213
  {
214
215
216
217
    FUNCNAME("ParallelDomainBase::setDofMatrix()");

    TEST_EXIT(mat)("No DOFMatrix!\n");

218
    using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
219
220
221
222
223
224
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

    traits::col<Matrix>::type col(mat->getBaseMatrix());
    traits::const_value<Matrix>::type value(mat->getBaseMatrix());

225
    typedef traits::range_generator<row, Matrix>::type cursor_type;
226
227
    typedef traits::range_generator<nz, cursor_type>::type icursor_type;

228
229
230
231
232
233
234
235
236
237
238
239
240
    std::vector<int> cols;
    std::vector<double> values;
    cols.reserve(300);
    values.reserve(300);

    for (cursor_type cursor = begin<row>(mat->getBaseMatrix()), 
	   cend = end<row>(mat->getBaseMatrix()); cursor != cend; ++cursor) {

      cols.clear();
      values.clear();

      int r = mapLocalGlobalDOFs[*cursor] * dispMult + dispAddRow;

241
242
      for (icursor_type icursor = begin<nz>(cursor), 
	     icend = end<nz>(cursor); icursor != icend; ++icursor)
243
	if (value(*icursor) != 0.0) {
244
245
	  cols.push_back(mapLocalGlobalDOFs[col(*icursor)] * dispMult + dispAddCol);
	  values.push_back(value(*icursor));
246
	}
247
248
249
250

      MatSetValues(petscMatrix, 1, &r, cols.size(), &(cols[0]), &(values[0]), ADD_VALUES);
    }

251
  }
252

253

254
255
256
  void ParallelDomainBase::setDofVector(DOFVector<double>* vec, 
					int dispMult, int dispAdd)
  {
257
258
    DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
    for (dofIt.reset(); !dofIt.end(); ++dofIt) {
259
      int index = mapLocalGlobalDOFs[dofIt.getDOFIndex()] * dispMult + dispAdd;
260
261
262
      double value = *dofIt;

      VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES);
263
    }    
264
265
  }

266

267
268
  void ParallelDomainBase::fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec)
  {
269
270
271
272
    MatCreate(PETSC_COMM_WORLD, &petscMatrix);
    MatSetSizes(petscMatrix, nRankRows, nRankRows, nOverallRows, nOverallRows);
    MatSetType(petscMatrix, MATAIJ);

273
274
275
276
277
    setDofMatrix(mat);

    MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);

Thomas Witkowski's avatar
Thomas Witkowski committed
278
279
280
281
    setDofVector(vec);

    VecAssemblyBegin(petscRhsVec);
    VecAssemblyEnd(petscRhsVec);
282
283
284
285
286
  }

  
  void ParallelDomainBase::fillPetscMatrix(Matrix<DOFMatrix*> *mat, SystemVector *vec)
  {
287
288
289
290
    FUNCNAME("ParallelDomainBase::fillPetscMatrix()");

    clock_t first = clock();

291
    using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
292
293
294
295
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

    int d_nnz[nRankRows];
296
297
298
    int o_nnz[nRankRows];

    for (int i = 0; i < nRankRows; i++) {
299
      d_nnz[i] = 0;
300
301
      o_nnz[i] = 0;
    }
302
303
304
305
306
307
308
309
310

    for (int i = 0; i < nComponents; i++) 
      for (int j = 0; j < nComponents; j++) 
 	if ((*mat)[i][j]) {
	  Matrix bmat = (*mat)[i][j]->getBaseMatrix();

	  traits::col<Matrix>::type col(bmat);
	  traits::const_value<Matrix>::type value(bmat);
	  
311
	  typedef traits::range_generator<row, Matrix>::type cursor_type;
312
313
	  typedef traits::range_generator<nz, cursor_type>::type icursor_type;
	  
314
315
316
317
318
319
320
321
322
	  for (cursor_type cursor = begin<row>(bmat), 
		 cend = end<row>(bmat); cursor != cend; ++cursor) {

	    int r = mapLocalGlobalDOFs[*cursor] * nComponents + i;
	    r -= rstart * nComponents;

	    if (r >= nRankRows)
	      continue;
	      
323
324
325
	    for (icursor_type icursor = begin<nz>(cursor), 
		   icend = end<nz>(cursor); icursor != icend; ++icursor)
	      if (value(*icursor) != 0.0) {
326
327
		int c = mapLocalGlobalDOFs[col(*icursor)] * nComponents + j;

328
329
330
331
332
333
		if (c >= rstart * nComponents && 
		    c < rstart * nComponents + nRankRows)
		  d_nnz[r]++;
		else
		  o_nnz[r]++;		  
	      }    
334
335
336
337
338
	  }
	}


    MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
339
340
341
342
343
344
345
346
		    0, d_nnz, 0, o_nnz, &petscMatrix);

#if (DEBUG != 0)
    int a, b;
    MatGetOwnershipRange(petscMatrix, &a, &b);
    TEST_EXIT(a == rstart * nComponents)("Wrong matrix ownership range!\n");
    TEST_EXIT(b == rstart * nComponents + nRankRows)("Wrong matrix ownership range!\n");
#endif
347

348
349
350
351
352
353
354
355
356
357
358
359
360
361
    using mtl::tag::major; using mtl::tag::nz; using mtl::begin; using mtl::end;
    namespace traits= mtl::traits;
    typedef DOFMatrix::base_matrix_type Matrix;

    for (int i = 0; i < nComponents; i++)
      for (int j = 0; j < nComponents; j++)
	if ((*mat)[i][j]) 
	  setDofMatrix((*mat)[i][j], nComponents, i, j);

    MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
    MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);

    for (int i = 0; i < nComponents; i++)
      setDofVector(vec->getDOFVector(i), nComponents, i);
362

363
364
365
    VecAssemblyBegin(petscRhsVec);
    VecAssemblyEnd(petscRhsVec);

366
    INFO(info, 8)("Fill petsc matrix needed %.5f seconds\n", TIME_USED(first, clock()));
367
368
369
  }


370
  void ParallelDomainBase::solvePetscMatrix(DOFVector<double> *vec)
371
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
372
373
    FUNCNAME("ParallelDomainBase::solvePetscMatrix()");

374
375
376
377
    KSP ksp;
    PC pc;

    KSPCreate(PETSC_COMM_WORLD, &ksp);
378
    KSPSetOperators(ksp, petscMatrix, petscMatrix, SAME_NONZERO_PATTERN);
379
    KSPGetPC(ksp, &pc);
380
381
    //PCSetType(pc, PCJACOBI);
    PCSetType(pc, PCILU);
382
    KSPSetTolerances(ksp, 1.e-7, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT);
383
384
    KSPSetType(ksp, KSPBCGS);
    //KSPSetType(ksp, KSPCG);
385
    KSPMonitorSet(ksp, myKSPMonitor, PETSC_NULL, 0);
386
387
388
389
390
    KSPSolve(ksp, petscRhsVec, petscSolVec);

    PetscScalar *vecPointer;
    VecGetArray(petscSolVec, &vecPointer);

Thomas Witkowski's avatar
Thomas Witkowski committed
391
    for (int i = 0; i < nRankDOFs; i++)
392
      (*vec)[mapLocalToDofIndex[i]] = vecPointer[i];
393
394
395

    VecRestoreArray(petscSolVec, &vecPointer);

396
397
    std::vector<double*> sendBuffers(sendDofs.size());
    std::vector<double*> recvBuffers(recvDofs.size());
Thomas Witkowski's avatar
Thomas Witkowski committed
398
399
400

    MPI::Request request[sendDofs.size() + recvDofs.size()];
    int requestCounter = 0;
401
402
    
    int i = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
403
    for (RankToDofContainer::iterator sendIt = sendDofs.begin();
404
	 sendIt != sendDofs.end(); ++sendIt, i++) {
405
406
      int nSendDOFs = sendIt->second.size();
      sendBuffers[i] = new double[nSendDOFs];
407

408
      for (int j = 0; j < nSendDOFs; j++)
Thomas Witkowski's avatar
Thomas Witkowski committed
409
	sendBuffers[i][j] = (*vec)[*((sendIt->second)[j])];
410

Thomas Witkowski's avatar
Thomas Witkowski committed
411
412
      request[requestCounter++] =
	mpiComm.Isend(sendBuffers[i], nSendDOFs, MPI_DOUBLE, sendIt->first, 0);
413
414
415
    }

    i = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
416
    for (RankToDofContainer::iterator recvIt = recvDofs.begin();
417
	 recvIt != recvDofs.end(); ++recvIt, i++) {
418
419
      int nRecvDOFs = recvIt->second.size();
      recvBuffers[i] = new double[nRecvDOFs];
420

Thomas Witkowski's avatar
Thomas Witkowski committed
421
422
      request[requestCounter++] =
	mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_DOUBLE, recvIt->first, 0);
423
424
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
425
426
427

    MPI::Request::Waitall(requestCounter, request);

428
    i = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
429
    for (RankToDofContainer::iterator recvIt = recvDofs.begin();
430
	 recvIt != recvDofs.end(); ++recvIt, i++) {
Thomas Witkowski's avatar
Thomas Witkowski committed
431
      for (int j = 0; j < static_cast<int>(recvIt->second.size()); j++)
Thomas Witkowski's avatar
Thomas Witkowski committed
432
	(*vec)[*(recvIt->second)[j]] = recvBuffers[i][j];
433
434
435
436

      delete [] recvBuffers[i];
    }
    
437
    for (int i = 0; i < static_cast<int>(sendBuffers.size()); i++)
438
      delete [] sendBuffers[i];
439
440

    MatDestroy(petscMatrix);
441
442
  }

443

444
445
446
447
  void ParallelDomainBase::solvePetscMatrix(SystemVector *vec)
  {
    FUNCNAME("ParallelDomainBase::solvePetscMatrix()");

448
    KSP solver;
449

450
451
452
453
454
455
    KSPCreate(PETSC_COMM_WORLD, &solver);
    KSPSetOperators(solver, petscMatrix, petscMatrix, SAME_NONZERO_PATTERN); 

    KSPSetTolerances(solver, 1.e-8, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT);
    KSPSetType(solver, KSPBCGS);
    KSPMonitorSet(solver, myKSPMonitor, PETSC_NULL, 0);
456
    KSPSetFromOptions(solver);
457
458

    KSPSolve(solver, petscRhsVec, petscSolVec);
459
460
461
462
463
464
465
466
467
468
469
470

    PetscScalar *vecPointer;
    VecGetArray(petscSolVec, &vecPointer);

    for (int i = 0; i < nComponents; i++) {
      DOFVector<double> *dofvec = vec->getDOFVector(i);
      for (int j = 0; j < nRankDOFs; j++)
	(*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
    }

    VecRestoreArray(petscSolVec, &vecPointer);

471

472
473
474
475
476
477
478
479
480
    std::vector<double*> sendBuffers(sendDofs.size());
    std::vector<double*> recvBuffers(recvDofs.size());

    MPI::Request request[sendDofs.size() + recvDofs.size()];
    int requestCounter = 0;
    
    int i = 0;
    for (RankToDofContainer::iterator sendIt = sendDofs.begin();
	 sendIt != sendDofs.end(); ++sendIt, i++) {
481
482
      int nSendDOFs = sendIt->second.size();
      sendBuffers[i] = new double[nSendDOFs * nComponents];
483
484
485
486
487
488
489
490

      int counter = 0;
      for (int j = 0; j < nComponents; j++) {
	DOFVector<double> *dofvec = vec->getDOFVector(j);
	for (int k = 0; k < nSendDOFs; k++)
	  sendBuffers[i][counter++] = (*dofvec)[*((sendIt->second)[k])];
      }

491
492
      request[requestCounter++] = mpiComm.Isend(sendBuffers[i], nSendDOFs * nComponents,
						MPI_DOUBLE, sendIt->first, 0);
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
    }

    i = 0;
    for (RankToDofContainer::iterator recvIt = recvDofs.begin();
	 recvIt != recvDofs.end(); ++recvIt, i++) {
      int nRecvDOFs = recvIt->second.size() * nComponents;
      recvBuffers[i] = new double[nRecvDOFs];

      request[requestCounter++] =
	mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_DOUBLE, recvIt->first, 0);
    }


    MPI::Request::Waitall(requestCounter, request);

    i = 0;
    for (RankToDofContainer::iterator recvIt = recvDofs.begin();
	 recvIt != recvDofs.end(); ++recvIt, i++) {
      int nRecvDOFs = recvIt->second.size();

      int counter = 0;
      for (int j = 0; j < nComponents; j++) {
	DOFVector<double> *dofvec = vec->getDOFVector(j);
516
517
 	for (int k = 0; k < nRecvDOFs; k++)
 	  (*dofvec)[*(recvIt->second)[k]] = recvBuffers[i][counter++];
518
519
520
521
522
523
524
      }

      delete [] recvBuffers[i];
    }
    
    for (int i = 0; i < static_cast<int>(sendBuffers.size()); i++)
      delete [] sendBuffers[i];
525

526
527
528
529
530
531
532
533
534
535
    int iterations = 0;
    KSPGetIterationNumber(solver, &iterations);
    MSG("  Number of iterations: %d\n", iterations);
    
    double norm = 0.0;
    MatMult(petscMatrix, petscSolVec, petscTmpVec);
    VecAXPY(petscTmpVec, -1.0, petscRhsVec);
    VecNorm(petscTmpVec, NORM_2, &norm);
    MSG("  Residual norm: %e\n", norm);

536
    MatDestroy(petscMatrix);
537
    KSPDestroy(solver);
538
539
540
  }


541
  double ParallelDomainBase::setElemWeights(AdaptInfo *adaptInfo) 
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
  {
    double localWeightSum = 0.0;
    int elNum = -1;

    elemWeights.clear();

    TraverseStack stack;
    ElInfo *elInfo = stack.traverseFirst(mesh, -1,
					 Mesh::CALL_EVERY_EL_PREORDER);
    while (elInfo) {
      Element *element = elInfo->getElement();

      // get partition data
      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

      if (partitionData && partitionData->getPartitionStatus() == IN) {
	if (partitionData->getLevel() == 0) {
	  elNum = element->getIndex();
	}
Thomas Witkowski's avatar
Thomas Witkowski committed
562
	TEST_EXIT_DBG(elNum != -1)("invalid element number\n");
563
564
565
566
567
568
569
570
571
572
573
574
	if (element->isLeaf()) {
	  elemWeights[elNum] += 1.0;
	  localWeightSum += 1.0;
	}
      }

      elInfo = stack.traverseNext(elInfo);
    }

    return localWeightSum;
  }

575

576
  void ParallelDomainBase::partitionMesh(AdaptInfo *adaptInfo)
577
578
579
580
581
582
583
584
585
586
587
588
589
  {
    if (initialPartitionMesh) {
      initialPartitionMesh = false;
      partitioner->fillCoarsePartitionVec(&oldPartitionVec);
      partitioner->partition(&elemWeights, INITIAL);
    } else {
      oldPartitionVec = partitionVec;
      partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
    }    

    partitioner->fillCoarsePartitionVec(&partitionVec);
  }

590

591
  void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs)
592
  {
593
    FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()");
Thomas Witkowski's avatar
Thomas Witkowski committed
594
595
596

    // === First, create all the information about the interior boundaries. ===

597
    TraverseStack stack;
598
599
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, 
					 Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH);
600
601
602
603
604
605
606
607
608
609
610
611
    while (elInfo) {
      Element *element = elInfo->getElement();

      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));   
      if (partitionData->getPartitionStatus() == IN) {
	for (int i = 0; i < 3; i++) {
	  if (!elInfo->getNeighbour(i))
	    continue;

	  PartitionElementData *neighbourPartitionData =
	    dynamic_cast<PartitionElementData*>(elInfo->getNeighbour(i)->getElementData(PARTITION_ED));
612

613
 	  if (neighbourPartitionData->getPartitionStatus() == OUT) {
614
615
	    // We have found an element that is at an interior boundary. 

616
617
	    // === Find out, if the boundary part of the element corresponds to the  ===
	    // === rank or to the rank "on the other side" of the interoir boundary. ===
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642

	    const DegreeOfFreedom* boundDOF1 = NULL;
	    const DegreeOfFreedom* boundDOF2 = NULL;
	    
	    switch (i) {
	    case 0:
	      boundDOF1 = element->getDOF(1);
	      boundDOF2 = element->getDOF(2);
	      break;
	    case 1:
	      boundDOF1 = element->getDOF(0);
	      boundDOF2 = element->getDOF(2);
	      break;
	    case 2:
	      boundDOF1 = element->getDOF(0);
	      boundDOF2 = element->getDOF(1);
	      break;
	    default:
	      ERROR_EXIT("Should never happen!\n");
	    }

	    bool isRankDOF1 = (find(rankDOFs.begin(), rankDOFs.end(), boundDOF1) != rankDOFs.end());
	    bool isRankDOF2 = (find(rankDOFs.begin(), rankDOFs.end(), boundDOF2) != rankDOFs.end());
	    bool ranksBoundary = isRankDOF1 || isRankDOF2;

643
	    // === And add the part of the interior boundary. ===
644
645

	    AtomicBoundary& bound = 
Thomas Witkowski's avatar
Thomas Witkowski committed
646
647
648
649
	      (ranksBoundary ?
	       myIntBoundary.getNewAtomicBoundary(partitionVec[elInfo->getNeighbour(i)->getIndex()]) :
	       otherIntBoundary.getNewAtomicBoundary(partitionVec[elInfo->getNeighbour(i)->getIndex()]));

650
	    bound.rankObject.el = element;
651
	    bound.rankObject.elIndex = element->getIndex();
652
653
	    bound.rankObject.subObjAtBoundary = EDGE;
	    bound.rankObject.ithObjAtBoundary = i;
654
655
656
657
	    // Do not set a pointer to the element, because if will be deleted from
	    // mesh after partitioning and the pointer would become invalid.
	    bound.neighbourObject.el = NULL;
	    bound.neighbourObject.elIndex = elInfo->getNeighbour(i)->getIndex();
658
	    bound.neighbourObject.subObjAtBoundary = EDGE;
659
660
661
662
663
	    bound.neighbourObject.ithObjAtBoundary = elInfo->getSideOfNeighbour(i);

	    // i == 2  =>  getSideOfNeighbour(i) == 2
	    TEST_EXIT_DBG(i != 2 || elInfo->getSideOfNeighbour(i) == 2)
	      ("Should not happen!\n");
664
665
666
667
668
669
 	  }
	}
      }

      elInfo = stack.traverseNext(elInfo);
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
670
671
672


    // === Once we have this information, we must care about their order. Eventually ===
Thomas Witkowski's avatar
Thomas Witkowski committed
673
674
    // === all the boundaries have to be in the same order on both ranks that share  ===
    // === the bounday.                                                              ===
Thomas Witkowski's avatar
Thomas Witkowski committed
675

Thomas Witkowski's avatar
Thomas Witkowski committed
676
    std::vector<int*> sendBuffers, recvBuffers;
Thomas Witkowski's avatar
Thomas Witkowski committed
677

Thomas Witkowski's avatar
Thomas Witkowski committed
678
679
680
681
    MPI::Request request[myIntBoundary.boundary.size() + 
			 otherIntBoundary.boundary.size()];
    int requestCounter = 0;

Thomas Witkowski's avatar
Thomas Witkowski committed
682
683
    for (RankToBoundMap::iterator rankIt = myIntBoundary.boundary.begin();
	 rankIt != myIntBoundary.boundary.end(); ++rankIt) {    
Thomas Witkowski's avatar
Thomas Witkowski committed
684
      int nSendInt = rankIt->second.size();
685
686
687
688
689
      int* buffer = new int[nSendInt * 2];
      for (int i = 0; i < nSendInt; i++) {
	buffer[i * 2] = (rankIt->second)[i].rankObject.elIndex;
	buffer[i * 2 + 1] = (rankIt->second)[i].rankObject.ithObjAtBoundary;
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
690
691
      sendBuffers.push_back(buffer);
      
Thomas Witkowski's avatar
Thomas Witkowski committed
692
      request[requestCounter++] =
693
	mpiComm.Isend(buffer, nSendInt * 2, MPI_INT, rankIt->first, 0);
Thomas Witkowski's avatar
Thomas Witkowski committed
694
695
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
696
697
    for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
	 rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
698
      int nRecvInt = rankIt->second.size();
699
      int *buffer = new int[nRecvInt * 2];
Thomas Witkowski's avatar
Thomas Witkowski committed
700
      recvBuffers.push_back(buffer);
Thomas Witkowski's avatar
Thomas Witkowski committed
701

Thomas Witkowski's avatar
Thomas Witkowski committed
702
      request[requestCounter++] = 
703
	mpiComm.Irecv(buffer, nRecvInt * 2, MPI_INT, rankIt->first, 0);
Thomas Witkowski's avatar
Thomas Witkowski committed
704
705
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
706
707
708

    MPI::Request::Waitall(requestCounter, request);

Thomas Witkowski's avatar
Thomas Witkowski committed
709
710

    int i = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
711
712
    for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin();
	 rankIt != otherIntBoundary.boundary.end(); ++rankIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
713
714
715
716
717
718
719

      // === We have received from rank "rankIt->first" the ordered list of element ===
      // === indices. We now have to sort the corresponding list in this rank to    ===
      // === get the same order.                                                    ===
      
      for (int j = 0; j < static_cast<int>(rankIt->second.size()); j++) {
	// If the expected object is not at place, search for it.
720
721
	if ((rankIt->second)[j].neighbourObject.elIndex != recvBuffers[i][j * 2] ||
	    (rankIt->second)[j].neighbourObject.ithObjAtBoundary != recvBuffers[i][j * 2 + 1]) {
Thomas Witkowski's avatar
Thomas Witkowski committed
722
723
	  int k = j + 1;
	  for (; k < static_cast<int>(rankIt->second.size()); k++)
724
725
	    if ((rankIt->second)[k].neighbourObject.elIndex == recvBuffers[i][j * 2] && 
		(rankIt->second)[k].neighbourObject.ithObjAtBoundary == recvBuffers[i][j * 2 + 1])
Thomas Witkowski's avatar
Thomas Witkowski committed
726
727
728
	      break;

	  // The element must always be found, because the list is just in another order.
Thomas Witkowski's avatar
Thomas Witkowski committed
729
	  TEST_EXIT_DBG(k < static_cast<int>(rankIt->second.size()))
730
	    ("Should never happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
731
732
733
734
735
736
737
738
739
740
741
742
743

	  // Swap the current with the found element.
	  AtomicBoundary tmpBound = (rankIt->second)[k];
	  (rankIt->second)[k] = (rankIt->second)[j];
	  (rankIt->second)[j] = tmpBound;	
	}
      }

      delete [] recvBuffers[i++];
    }

    for (int i = 0; i < static_cast<int>(sendBuffers.size()); i++)
      delete [] sendBuffers[i];
744
745
746
  }


747
  void ParallelDomainBase::removeMacroElements()
748
749
750
751
752
753
754
755
756
757
758
759
  {
    std::vector<MacroElement*> macrosToRemove;
    for (std::deque<MacroElement*>::iterator it = mesh->firstMacroElement();
	 it != mesh->endOfMacroElements();
	 ++it) {
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>
	((*it)->getElement()->getElementData(PARTITION_ED));
      if (partitionData->getPartitionStatus() != IN)
	macrosToRemove.push_back(*it);
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
760
    mesh->removeMacroElements(macrosToRemove, feSpace);
761
762
763
  }


Thomas Witkowski's avatar
Thomas Witkowski committed
764
765
766
  void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs,
						      int& nRankDOFs, 
						      int& nOverallDOFs)
767
  {
768
    FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()");
769
770

    // === Get rank information about DOFs. ===
771
772
773

    // Stores to each DOF pointer the set of ranks the DOF is part of.
    std::map<const DegreeOfFreedom*, std::set<int> > partitionDOFs;
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
774
    DofContainer rankAllDofs;
775
    DofToRank boundaryDofs;
776

777
    createDOFMemberInfo(partitionDOFs, rankDOFs, rankAllDofs, boundaryDofs);
778
779
780
781

    nRankDOFs = rankDOFs.size();
    nOverallDOFs = partitionDOFs.size();

782

783
    // === Get starting position for global rank dof ordering. ====
784

785
    rstart = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
786
    mpiComm.Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM);
787
788
    rstart -= nRankDOFs;

789
790
791
792
    
    // === Create for all dofs in rank new indices. The new index must start at ===
    // === index 0, must be continues and have the same order as the indices    ===
    // === had before.                                                          ===
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
793

Thomas Witkowski's avatar
Thomas Witkowski committed
794

795
796
797
    // Do not change the indices now, but create a new indexing a store it here.
    DofIndexMap rankDofsNewLocalIndex;
    isRankDof.clear();
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
798
799
800
    int i = 0;
    for (DofContainer::iterator dofIt = rankAllDofs.begin();
	 dofIt != rankAllDofs.end(); ++dofIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
801

802
803
804
      rankDofsNewLocalIndex[*dofIt] = i;
      // First, we set all dofs in ranks partition to be owend by the rank. Later,
      // the dofs in ranks partition that are owned by other rank are set to false.
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
805
      isRankDof[i] = true;
806
      i++;
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
807
808
    }

809
810
811

    // === Create for all rank owned dofs a new global indexing.                ===

812
813
    // Stores for dofs in rank a new global index.
    DofIndexMap rankDofsNewGlobalIndex;
814
815
816
    // Stores for all rank owned dofs a continues local index.
    DofIndexMap rankOwnedDofsNewLocalIndex;

Thomas Witkowski's avatar
n    
Thomas Witkowski committed
817
818
819
    i = 0;
    for (DofContainer::iterator dofIt = rankDOFs.begin();
	 dofIt != rankDOFs.end(); ++dofIt) {
820
      rankDofsNewGlobalIndex[*dofIt] = i + rstart;
821
      rankOwnedDofsNewLocalIndex[*dofIt] = i;
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
822
823
824
      i++;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
825
 
826
827
828
829
830
831
    // === Create information which dof indices must be send and which must  ===
    // === be received.                                                      ===

    // Stores to each rank a map from DOF pointers (which are all owned by the rank
    // and lie on an interior boundary) to new global DOF indices.
    std::map<int, std::map<const DegreeOfFreedom*, DegreeOfFreedom> > sendNewDofs;
832

833
834
835
836
    // Maps to each rank the number of new DOF indices this rank will receive from.
    // All these DOFs are at an interior boundary on this rank, but are owned by
    // another rank.
    std::map<int, int> recvNewDofs;
837

838
    for (DofToRank::iterator it = boundaryDofs.begin(); it != boundaryDofs.end(); ++it) {
839
840
841
842
843
844
845
846

      if (it->second == mpiRank) {
	// If the boundary dof is a rank dof, it must be send to other ranks.

	// Search for all ranks that have this dof too.
	for (std::set<int>::iterator itRanks = partitionDOFs[it->first].begin();
	     itRanks != partitionDOFs[it->first].end();
	     ++itRanks) {
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
847
	  if (*itRanks != mpiRank) {
848
	    TEST_EXIT_DBG(rankDofsNewGlobalIndex.count(it->first) == 1)
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
849
850
	      ("DOF Key not found!\n");

851
	    sendNewDofs[*itRanks][it->first] = rankDofsNewGlobalIndex[it->first];
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
852
	  }
853
854
	}
      } else {
Thomas Witkowski's avatar
Thomas Witkowski committed
855
856
	// If the boundary dof is not a rank dof, its new dof index (and later
	// also the dof values) must be received from another rank.
857
858
859
860
	if (recvNewDofs.find(it->second) == recvNewDofs.end()) 
	  recvNewDofs[it->second] = 1;
	else
	  recvNewDofs[it->second]++;
861
862
863
      }
    }

Thomas Witkowski's avatar
n    
Thomas Witkowski committed
864

865
    // === Send and receive the dof indices at boundary. ===
866

Thomas Witkowski's avatar
Thomas Witkowski committed
867
    std::vector<int*> sendBuffers(sendNewDofs.size()), recvBuffers(recvNewDofs.size());
868
869
870

    sendDofs.clear();
    recvDofs.clear();
871
    
Thomas Witkowski's avatar
Thomas Witkowski committed
872
873
874
    MPI::Request request[sendNewDofs.size() + recvNewDofs.size()];
    int requestCounter = 0;

Thomas Witkowski's avatar
n    
Thomas Witkowski committed
875
    i = 0;
876
877
    for (std::map<int, std::map<const DegreeOfFreedom*, DegreeOfFreedom> >::iterator 
	   sendIt = sendNewDofs.begin();
878
879
	 sendIt != sendNewDofs.end();
	 ++sendIt, i++) {
880
881
      int nSendDofs = sendIt->second.size() * 2;
      sendBuffers[i] = new int[nSendDofs];
882

883
      int c = 0;
884
885
      for (std::map<const DegreeOfFreedom*, DegreeOfFreedom>::iterator 
	     dofIt = sendIt->second.begin();
886
	   dofIt != sendIt->second.end();
887
	   ++dofIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
888
	sendBuffers[i][c++] = *(dofIt->first);
889
	sendBuffers[i][c++] = dofIt->second;
890

891
	sendDofs[sendIt->first].push_back(dofIt->first);
892
893
      }

Thomas Witkowski's avatar
Thomas Witkowski committed
894
895
      request[requestCounter++] =
	mpiComm.Isend(sendBuffers[i], nSendDofs, MPI_INT, sendIt->first, 0);
896
897
898
    }

    i = 0;
899
    for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
900
901
	 recvIt != recvNewDofs.end();
	 ++recvIt, i++) {
902
903
      int nRecvDOFs = recvIt->second * 2;
      recvBuffers[i] = new int[nRecvDOFs];
904

Thomas Witkowski's avatar
Thomas Witkowski committed
905
906
      request[requestCounter++] =
	mpiComm.Irecv(recvBuffers[i], nRecvDOFs, MPI_INT, recvIt->first, 0);
907
908
909
    }


Thomas Witkowski's avatar
Thomas Witkowski committed
910
    MPI::Request::Waitall(requestCounter, request);
911
912

    
913
    // === Delete send buffers. ===
914

Thomas Witkowski's avatar
Thomas Witkowski committed
915
916
    for (int j = 0; j < static_cast<int>(sendBuffers.size()); j++)
      delete [] sendBuffers[j];
917
918


919
    // === Change dof indices at boundary from other ranks. ===
920

Thomas Witkowski's avatar
Thomas Witkowski committed
921
922
    // Within this small data structure we track which dof index was already changed.
    // This is used to avoid the following situation: Assume, there are two dof indices
923
    // a and b in boundaryDofs. Then we have to change index a to b and b to c. When
Thomas Witkowski's avatar
Thomas Witkowski committed
924
925
926
927
928
    // the second rule applies, we have to avoid that not the first b, resulted from
    // changing a to b, is set to c, but the second one. Therefore, after the first
    // rule was applied, the dof pointer is set to false in this data structure and 
    // is not allowed to be changed anymore.
    std::map<const DegreeOfFreedom*, bool> dofChanged;
929
    for (DofToRank::iterator dofIt = boundaryDofs.begin(); dofIt != boundaryDofs.end();
Thomas Witkowski's avatar
Thomas Witkowski committed
930
	 ++dofIt)
Thomas Witkowski's avatar
Thomas Witkowski committed
931
932
      dofChanged[dofIt->first] = false;

933
    i = 0;
934
    for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
935
936
937
	 recvIt != recvNewDofs.end();
	 ++recvIt, i++) {

938
      for (int j = 0; j < recvIt->second; j++) {
939

940
941
	DegreeOfFreedom oldDof = recvBuffers[i][j * 2];
	DegreeOfFreedom newGlobalDof = recvBuffers[i][j * 2 + 1];
942

943
	bool found = false;
944

Thomas Witkowski's avatar
Thomas Witkowski committed
945
946
	// Iterate over all boundary dofs to find the dof, which index we have to change.

947
948
	for (DofToRank::iterator dofIt = boundaryDofs.begin(); 
	     dofIt != boundaryDofs.end(); ++dofIt) {
949

Thomas Witkowski's avatar
Thomas Witkowski committed
950
951
952
	  if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
	    dofChanged[dofIt->first] = true;

953
	    recvDofs[recvIt->first].push_back(dofIt->first);
954
	    rankDofsNewGlobalIndex[dofIt->first] = newGlobalDof;
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
955
	    isRankDof[rankDofsNewLocalIndex[dofIt->first]] = false;
Thomas Witkowski's avatar
Thomas Witkowski committed
956

957
	    found = true;
958
959
960
	    break;
	  }
	}
961

Thomas Witkowski's avatar
Thomas Witkowski committed
962
	TEST_EXIT_DBG(found)("Should not happen!\n");
963
964
965
966
      }

      delete [] recvBuffers[i];
    }
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
967

968
    // === Create now the local to global index and local to dof index mappings.  ===
969

970
971
    createLocalMappings(rankDofsNewLocalIndex, rankOwnedDofsNewLocalIndex,
			rankDofsNewGlobalIndex);
972
973
974
  }


Thomas Witkowski's avatar
Thomas Witkowski committed
975
  void ParallelDomainBase::updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs)
976
  {
977
    FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()");
Thomas Witkowski's avatar
Thomas Witkowski committed
978

979
    typedef std::set<const DegreeOfFreedom*> DofSet;
Thomas Witkowski's avatar
Thomas Witkowski committed
980

981
    // === Get all DOFs in ranks partition. ===
982

Thomas Witkowski's avatar
Thomas Witkowski committed
983
    ElementDofIterator elDofIt(feSpace);
984
    DofSet rankDOFSet;
985
986
987
    TraverseStack stack;
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
    while (elInfo) {
Thomas Witkowski's avatar
Thomas Witkowski committed
988
      Element *element = elInfo->getElement();     
Thomas Witkowski's avatar
Thomas Witkowski committed
989
990
      elDofIt.reset(element);
      do {
991
	rankDOFSet.insert(elDofIt.getDofPtr());
Thomas Witkowski's avatar
Thomas Witkowski committed
992
      } while(elDofIt.next());
993
994
995
996

      elInfo = stack.traverseNext(elInfo);
    }

997
    DofContainer rankAllDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
998
999
    for (DofSet::iterator dofIt = rankDOFSet.begin(); dofIt != rankDOFSet.end(); ++dofIt)
      rankAllDofs.push_back(*dofIt);    
1000
1001
1002
1003
    sort(rankAllDofs.begin(), rankAllDofs.end(), cmpDofsByValue);
    DofContainer rankDOFs = rankAllDofs;


Thomas Witkowski's avatar
Thomas Witkowski committed
1004
    // === Traverse on interior boundaries and move all not ranked owned DOFs from ===
1005
    // === rankDOFs to boundaryDOFs.                                               ===
1006

1007
1008
    sendDofs.clear();
    recvDofs.clear();
1009

Thomas Witkowski's avatar
Thomas Witkowski committed
1010
1011
1012
    for (RankToBoundMap::iterator it =  myIntBoundary.boundary.begin();
	 it != myIntBoundary.boundary.end(); ++it) {

Thomas Witkowski's avatar
Thomas Witkowski committed
1013
      for (std::vector<AtomicBoundary>::iterator boundIt = it->second.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
1014
1015
	   boundIt != it->second.end(); ++boundIt) {

1016
1017
	DofContainer dofs;
	DofContainer &dofsToSend = sendDofs[it->first];
Thomas Witkowski's avatar
Thomas Witkowski committed
1018
1019
1020

	switch (boundIt->rankObject.ithObjAtBoundary) {
	case 0:
1021
1022
	  dofs.push_back(boundIt->rankObject.el->getDOF(1));
	  dofs.push_back(boundIt->rankObject.el->getDOF(2));
Thomas Witkowski's avatar
Thomas Witkowski committed
1023
1024
	  break;
	case 1:
1025
1026
	  dofs.push_back(boundIt->rankObject.el->getDOF(0));
	  dofs.push_back(boundIt->rankObject.el->getDOF(2));
Thomas Witkowski's avatar
Thomas Witkowski committed
1027
1028
	  break;
	case 2:
1029
1030
	  dofs.push_back(boundIt->rankObject.el->getDOF(0));
	  dofs.push_back(boundIt->rankObject.el->getDOF(1));
Thomas Witkowski's avatar
Thomas Witkowski committed
1031
1032
1033
1034
1035
	  break;
	default:
	  ERROR_EXIT("Should never happen!\n");
	}

1036
1037
1038
	for (DofContainer::iterator dofIt = dofs.begin(); dofIt != dofs.end(); ++dofIt)
	  if (find(dofsToSend.begin(), dofsToSend.end(), *dofIt) == dofsToSend.end())
	    dofsToSend.push_back(*dofIt);