ParallelProblem.cc 55.4 KB
Newer Older
1
#include "ParallelProblem.h"
2
3
4
5
6
7
8
9
10
11
#include "ProblemScal.h"
#include "ProblemVec.h"
#include "ProblemInstat.h"
#include "AdaptInfo.h"
#include "AdaptStationary.h"
#include "ConditionalEstimator.h"
#include "ConditionalMarker.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
12
#include "MacroElement.h"
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#include "PartitionElementData.h"
#include "ParMetisPartitioner.h"
#include "Mesh.h"
#include "DualTraverse.h"
#include "MeshStructure.h"
#include "DOFVector.h"
#include "FiniteElemSpace.h"
#include "RefinementManager.h"
#include "CoarseningManager.h"
#include "Lagrange.h"
#include "ElementFileWriter.h"
#include "MacroWriter.h"
#include "ValueWriter.h"
#include "SystemVector.h"
27
#include "VtkWriter.h"
28
29
#include "mpi.h"
#include <queue>
30
#include <time.h>
31
32
33
34
35

namespace AMDiS {

  bool elementInPartition(ElInfo *elInfo)
  {
36
37
38
39
40
41
42
    PartitionElementData *elementData = dynamic_cast<PartitionElementData*>
      (elInfo->getElement()->getElementData(PARTITION_ED));
    if (elementData && elementData->getPartitionStatus() == IN) {
      return true;
    } else {
      return false;
    }
43
44
45
46
47
48
49
50
51
52
53
54
55
  }

  class MyDualTraverse : public DualTraverse
  {
  public:
    MyDualTraverse(int coarseLevel)
      : coarseLevel_(coarseLevel)
    {};

    bool skipEl1(ElInfo *elInfo)
    {
      PartitionElementData *elementData = dynamic_cast<PartitionElementData*>
	(elInfo->getElement()->getElementData(PARTITION_ED));
56
57
      if (elementData) {
	if (elInfo->getElement()->isLeaf() && 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
	   elementData->getLevel() < coarseLevel_)
	  return false;
	if(elementData->getLevel() == coarseLevel_)
	  return false;
      }
      return true;
    };
  private:
    int coarseLevel_;
  };

  // =========================================================================
  // ===== class ParallelProblemBase =========================================
  // =========================================================================

73
74
  ParallelProblemBase::ParallelProblemBase(const std::string& name,
					   ProblemIterationInterface *iterationIF,
75
76
					   ProblemTimeInterface *timeIF)
    : iterationIF_(iterationIF),
77
      timeIF_(timeIF),
78
79
      debugMode(0),
      debugServerProcess(false)
80
  {
81
82
    mpiRank = MPI::COMM_WORLD.Get_rank();
    mpiSize = MPI::COMM_WORLD.Get_size();
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

    GET_PARAMETER(0, name + "->debug mode", "%d", &debugMode);

    if (debugMode) {
      MPI::Group group = MPI::COMM_WORLD.Get_group();
      
      int rankSize = mpiSize - 1;
      int *ranks = GET_MEMORY(int, rankSize);
      for (int i = 0; i < rankSize; i++) {
	ranks[i] = i + 1;
      }
      
      amdisGroup = group.Incl(rankSize, ranks);
      
      mpiComm = MPI::COMM_WORLD.Create(amdisGroup);
      
      if (mpiComm != MPI::COMM_NULL) {
	mpiRank = mpiComm.Get_rank();
	mpiSize = mpiComm.Get_size();
	debugServerProcess = false;
      } else {
	debugServerProcess = true;
      }
      
      FREE_MEMORY(ranks, int, rankSize);
    } else {
      mpiComm = MPI::COMM_WORLD;
    }
111
112
  }

113
114
115
116
117
118
  void ParallelProblemBase::exitParallelization(AdaptInfo *adaptInfo)
  {
    if (!timeIF_) 
      closeTimestep(adaptInfo);

    amdisGroup.Free();
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  }

  void ParallelProblemBase::closeTimestep(AdaptInfo *adaptInfo)
  {
    if (mpiSize > 1 && doBuildGlobalSolution(adaptInfo)) {
      if (debugMode && mpiRank == 0) {
	// Send adaptInfo inforamtion to debug server
	double sendTime = adaptInfo->getTime();
	double sendTimestep = adaptInfo->getTimestep();
	MPI::COMM_WORLD.Send(&sendTime, 1, MPI_DOUBLE, 0, 100);
	MPI::COMM_WORLD.Send(&sendTimestep, 1, MPI_DOUBLE, 0, 100);
      }
      synchronizeMeshes(adaptInfo);	
      exchangeRankSolutions(adaptInfo);
      buildGlobalSolution(adaptInfo);
    }
    
    if (timeIF_) 
      timeIF_->closeTimestep(adaptInfo);
  }
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184

  Flag ParallelProblemBase::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
  {
    Flag flag;
    
    if (mpiSize > 1 && toDo.isSet(MARK | ADAPT)) {
      flag = iterationIF_->oneIteration(adaptInfo, MARK | ADAPT);
      
      double localWeightSum = setElemWeights(adaptInfo);
      if (doPartitioning(adaptInfo, localWeightSum)) {
	clock_t partitioningStart = clock();

	synchronizeMeshes(adaptInfo);
	partitionMesh(adaptInfo);
	refineOverlap(adaptInfo);
	createOverlap(adaptInfo);
	synchronizeMeshes(adaptInfo);
	exchangeDOFVectors(adaptInfo);
	coarsenOutOfPartition(adaptInfo);
	
	clock_t partitioningEnd = clock();
	partitioningTime = TIME_USED(partitioningStart, 
				     partitioningEnd);
	computationStart = partitioningEnd;
      }
      
      flag |= iterationIF_->oneIteration(adaptInfo, toDo & ~(MARK | ADAPT));
    } else {
      flag = iterationIF_->oneIteration(adaptInfo, toDo);
    }
    
    // synchronize adaption flag
    unsigned long *flagBuffer = GET_MEMORY(unsigned long, mpiSize);
    
    unsigned long localFlag = flag.getFlags();
    
    mpiComm.Allgather(&localFlag, 1, MPI_UNSIGNED_LONG,
		      flagBuffer, 1, MPI_UNSIGNED_LONG);
    for (int i = 0; i < mpiSize; i++) {
      flag.setFlag(flagBuffer[i]);
    }
    FREE_MEMORY(flagBuffer, unsigned long, mpiSize);
    
    return flag;
  };

185
186
187
188
189
190
191
192
193
194
195
  // =========================================================================
  // ===== class ParallelProblem =============================================
  // =========================================================================

  ParallelProblem::ParallelProblem(const std::string& name,
				   ProblemIterationInterface *iterationIF,
				   ProblemTimeInterface *timeIF,
				   std::vector<DOFVector<double>*> vectors,
				   Mesh *mesh,
				   RefinementManager *rm,
				   CoarseningManager *cm)
196
    : ParallelProblemBase(name, iterationIF, timeIF),
197
      name_(name),
198
199
200
      mesh(mesh),
      refinementManager(rm),
      coarseningManager(cm),
201
202
203
204
205
206
207
208
209
210
211
      repartitionSteps_(1),
      puEveryTimestep_(false),
      dofVectors_(vectors),
      upperPartThreshold_(1.5),
      lowerPartThreshold_(2.0/3.0),
      globalCoarseGridLevel_(0),
      localCoarseGridLevel_(0),
      globalRefinements_(0),
      adaptiveThresholds_(0),
      thresholdIncFactor_(2.0),
      thresholdDecFactor_(0.5),
212
      repartTimeFactor_(10.0)
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
  {
    GET_PARAMETER(0, name_ + "->upper part threshold", "%f", 
		  &upperPartThreshold_);
    GET_PARAMETER(0, name_ + "->lower part threshold", "%f", 
		  &lowerPartThreshold_);
    GET_PARAMETER(0, name_ + "->global coarse grid level", "%d", 
		  &globalCoarseGridLevel_);
    GET_PARAMETER(0, name_ + "->local coarse grid level", "%d", 
		  &localCoarseGridLevel_);
    GET_PARAMETER(0, name_ + "->global refinements", "%d", 
		  &globalRefinements_);


    TEST_EXIT(localCoarseGridLevel_ >= globalCoarseGridLevel_)
      ("local coarse grid level < global coarse grid level\n");

229
    partitioner = NEW ParMetisPartitioner(mesh, &mpiComm);
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253

    GET_PARAMETER(0, name_ + "->adaptive thresholds", "%d", 
		  &adaptiveThresholds_);
    GET_PARAMETER(0, name_ + "->threshold inc factor", "%f", 
		  &thresholdIncFactor_);
    GET_PARAMETER(0, name_ + "->threshold dec factor", "%f", 
		  &thresholdDecFactor_);
    GET_PARAMETER(0, name_ + "->repart time factor", "%f", 
		  &repartTimeFactor_);


    TEST_EXIT(lowerPartThreshold_ <= 1.0)("invalid lower part threshold\n");
    TEST_EXIT(upperPartThreshold_ >= 1.0)("invalid upper part threshold\n");

    if (adaptiveThresholds_) {
      TEST_EXIT(thresholdDecFactor_ <= 1.0)("invalid threshold dec factor\n");
      TEST_EXIT(thresholdIncFactor_ >= 1.0)("invalid threshold inc factor\n");
    }
    minUpperTH_ = upperPartThreshold_;
    maxLowerTH_ = lowerPartThreshold_;
  }

  ParallelProblem::~ParallelProblem() 
  {
254
    DELETE partitioner;
255
256
257
258
259
260
  }

  bool ParallelProblem::doPartitioning(AdaptInfo *adaptInfo, double localWeightSum) 
  {
    FUNCNAME("ParallelProblem::doPartitioning()");

261
262
263
    double *weightSum = GET_MEMORY(double, mpiSize);
    int *partArray = GET_MEMORY(int, mpiSize);
    int part = 0;
264

265
    mpiComm.Gather(&localWeightSum, 1, MPI_DOUBLE, weightSum, 1, MPI_DOUBLE, 0);
266

267
    if (mpiRank == 0) {
268
269

      double average = 0.0;
270
      for (int i = 0; i < mpiSize; i++) {
271
272
273
	average += weightSum[i];
      }

274
      average /= mpiSize;
275

276
277
      for (int i = 0; i < mpiSize; i++) {
	if ((weightSum[i] / average) > upperPartThreshold_) {
278
279
280
	  part = 1;
	  break;
	}
281
	if ((weightSum[i] / average) < lowerPartThreshold_) {
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
	  part = 1;
	  break;
	}
      }

      double computationTime = TIME_USED(computationStart, clock());
      if (adaptiveThresholds_) {

	bool timeOver = ((computationTime / partitioningTime) >= repartTimeFactor_);

	if (part == 1 && !timeOver) {
	  // inc thresholds
	  upperPartThreshold_ *= thresholdIncFactor_;
	  lowerPartThreshold_ /= thresholdIncFactor_;

	  // avoid repartitioning
	  part = 0;
	}
      
	if (part == 0 && timeOver) {
	  // dec thresholds
	  upperPartThreshold_ *= thresholdDecFactor_;
	  lowerPartThreshold_ /= thresholdDecFactor_;

	  upperPartThreshold_ = max(minUpperTH_, upperPartThreshold_);
	  lowerPartThreshold_ = min(maxLowerTH_, lowerPartThreshold_);
	}
      }

311
      for (int i = 0; i < mpiSize; i++) {
312
313
314
315
	partArray[i] = part;
      }      
    }

316
317
    mpiComm.Scatter(partArray, 1, MPI_INT,
		    &part, 1, MPI_INT, 0);
318
    
319
320
    FREE_MEMORY(weightSum, double, mpiSize);
    FREE_MEMORY(partArray, int, mpiSize);
321
322
323
324
325
326
327
328
329
330
331

    return (part == 1);
  }

  bool ParallelProblem::doBuildGlobalSolution(AdaptInfo *adaptInfo) {
    return true;
  }

  void ParallelProblem::partitionMesh(AdaptInfo *adaptInfo)
  {
    static bool initial = true;
332
    if (initial) {
333
      initial = false;
334
335
      partitioner->fillCoarsePartitionVec(&oldPartitionVec);
      partitioner->partition(&elemWeights_, INITIAL);
336
    } else {
337
      oldPartitionVec = partitionVec;
338
      partitioner->partition(&elemWeights_, ADAPTIVE_REPART, 100.0 /*0.000001*/);
339
340
    }    

341
    partitioner->fillCoarsePartitionVec(&partitionVec);
342
343
344
345
  }

  void ParallelProblem::refineOverlap(AdaptInfo *adaptInfo)
  {
346
    int dim = mesh->getDim();
347
348
349
350

    bool finished = (localCoarseGridLevel_ == 0);

    //for(j = globalCoarseGridLevel_; j < localCoarseGridLevel_; j++) {
351
    while (!finished) {
352
353
354
355
      std::map<DegreeOfFreedom, int> inOut; // 1: in, 2: out, 3: border dof

      // mark in/out/border dofs
      TraverseStack stack;
356
      ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
357
      while (elInfo) {
358
359
360
361
362
363
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>(elInfo->getElement()->getElementData(PARTITION_ED));

	const DegreeOfFreedom **dofs = element->getDOF();

364
365
	if (partitionData->getPartitionStatus() == IN) {
	  for (int i = 0; i < dim + 1; i++) {
366
	    DegreeOfFreedom dof = dofs[i][0];
367
368
369
370
	    if (inOut[dof] == 2) 
	      inOut[dof] = 3;
	    if (inOut[dof] == 0) 
	      inOut[dof] = 1;
371
372
	  }
	} else {
373
	  for (int i = 0; i < dim + 1; i++) {
374
	    DegreeOfFreedom dof = dofs[i][0];
375
376
377
378
	    if (inOut[dof] == 1) 
	      inOut[dof] = 3;
	    if (inOut[dof] == 0) 
	      inOut[dof] = 2;
379
380
381
382
383
384
385
386
387
	  }
	}

	elInfo = stack.traverseNext(elInfo);
      }

      // refine overlap-border and inner elements
      finished = true;
      bool marked = false;
388
      elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
389
      while (elInfo) {
390
391
392
393
394
395
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>(elInfo->getElement()->getElementData(PARTITION_ED));

	int level = partitionData->getLevel();

396
397
	if (level < localCoarseGridLevel_) {
	  if (partitionData->getPartitionStatus() != IN) {
398
	    const DegreeOfFreedom **dofs = element->getDOF();
399
	    for (int i = 0; i < dim + 1; i++) {
400
	      DegreeOfFreedom dof = dofs[i][0];
401
	      if (inOut[dof] == 3) {
402
403
		element->setMark(1);
		marked = true;
404
405
		if ((level + 1) < localCoarseGridLevel_) 
		  finished = false;
406
407
408
409
410
411
		break;
	      }
	    }
	  } else {
	    element->setMark(1);
	    marked = true;
412
413
	    if ((level + 1) < localCoarseGridLevel_) 
	      finished = false;
414
415
416
417
418
	  }
	}

	elInfo = stack.traverseNext(elInfo);
      }
419
420
      if (marked) 
	refinementManager->refineMesh(mesh);
421
422
423
424
425
426
    }
  }

  void ParallelProblem::globalRefineOutOfPartition(AdaptInfo *adaptInfo)
  {
    TraverseStack stack;
427
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
428
    while (elInfo) {
429
430
431
432
433
434
435
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>(elInfo->getElement()->getElementData(PARTITION_ED));
      int refinements = globalCoarseGridLevel_ - partitionData->getLevel();
      elInfo->getElement()->setMark(max(0, refinements));
      elInfo = stack.traverseNext(elInfo);
    }

436
    refinementManager->refineMesh(mesh);
437
438
439
440
441
442
443
  }

  void ParallelProblem::coarsenOutOfPartition(AdaptInfo *adaptInfo)
  {
    Flag meshCoarsened = 1;    
    while(meshCoarsened.getFlags() != 0) {
      TraverseStack stack;
444
      ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
445
446
447
448
449
450
451
452
453
454
455
456
      while(elInfo) {
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>
	  (element->getElementData(PARTITION_ED));
	if(partitionData->getPartitionStatus() == OUT) {
	  int mark = min(0, -partitionData->getLevel() + globalCoarseGridLevel_);
	  //int mark = -partitionData->getLevel();
	  element->setMark(mark);
	}
	elInfo = stack.traverseNext(elInfo);
      }
457
      meshCoarsened = coarseningManager->coarsenMesh(mesh);
458
    }
459
    mpiComm.Barrier();
460
461
462
463
  }

  void ParallelProblem::exchangeMeshStructureCodes(MeshStructure *structures)
  {
464
465
466
    // every process creates a mesh structure code from its mesh.
    structures[mpiRank].init(mesh);
    const std::vector<unsigned long int>& myCode = structures[mpiRank].getCode();
467
468

    // broadcast code sizes
469
    int *codeSize = GET_MEMORY(int, mpiSize);
470
    int tmp = static_cast<int>(myCode.size());
471
472
473
474
475
    mpiComm.Allgather(&tmp, 1, MPI_INT, codeSize, 1, MPI_INT);
    if (debugMode) {
      // send code sizes also to debug server
      MPI::COMM_WORLD.Gather(&tmp, 1, MPI_INT, NULL, 1, MPI_INT, 0);
    }
476
477

    // broadcast number of elements
478
479
    int *elements = GET_MEMORY(int, mpiSize);
    tmp = structures[mpiRank].getNumElements();
480
481
482
483
484
    mpiComm.Allgather(&tmp, 1, MPI_INT, elements, 1, MPI_INT);
    if (debugMode) {
      // send number of elements also to debug server
      MPI::COMM_WORLD.Gather(&tmp, 1, MPI_INT, NULL, 1, MPI_INT, 0);
    }
485
486

    // broadcast codes
487
    int *codeOffset = GET_MEMORY(int, mpiSize);
488
    int codeSizeSum = 0;
489
    for (int rank = 0; rank < mpiSize; rank++) {
490
491
492
493
494
      codeOffset[rank] = codeSizeSum;
      codeSizeSum += codeSize[rank];
    }

    unsigned long int *code = GET_MEMORY(unsigned long int, codeSizeSum);
495
    unsigned long int *localCode = GET_MEMORY(unsigned long int, codeSize[mpiRank]);  
496
497
498
    unsigned long int *ptr;
    std::vector<unsigned long int>::const_iterator it, end = myCode.end();
  
499
500
    for (ptr = localCode, it = myCode.begin();
	 it != end; 
501
	 ++it, ++ptr) {
502
503
      *ptr = *it;
    }
504
505
506
507
508
509
510
511
512
513

    mpiComm.Allgatherv(localCode, codeSize[mpiRank], 
		       MPI_UNSIGNED_LONG, 
		       code, codeSize, codeOffset,
		       MPI_UNSIGNED_LONG);
    if (debugMode) {
      // send codes also to debug server
      MPI::COMM_WORLD.Send(localCode, codeSize[mpiRank],
			   MPI_UNSIGNED_LONG, 0, 100);
    }
514
    
515
516
    for (int rank = 0; rank < mpiSize; rank++) {
      if (rank != mpiRank) {
517
518
519
520
	std::vector<unsigned long int> remoteCode;
	unsigned long int *ptr;
	unsigned long int *begin = code + codeOffset[rank]; 
	unsigned long int *end = begin + codeSize[rank];
521
	for (ptr = begin; ptr != end; ++ptr) {
522
523
524
525
526
527
528
	  remoteCode.push_back(*ptr);
	}
	structures[rank].init(remoteCode, elements[rank]);
      }
    }

    // free memory
529
    FREE_MEMORY(elements, int, mpiSize);
530
    FREE_MEMORY(code, unsigned long int, codeSizeSum);
531
532
533
    FREE_MEMORY(localCode, unsigned long int, codeSize[mpiRank]);
    FREE_MEMORY(codeOffset, int, mpiSize);
    FREE_MEMORY(codeSize, int, mpiSize);
534
535
536
537
538
539
540
    
  }

  void ParallelProblem::synchronizeMeshes(AdaptInfo *adaptInfo)
  {
    FUNCNAME("ParallelProblem::synchronizeMeshes()");

541
    MeshStructure *structures = NEW MeshStructure[mpiSize];
542

543
    // build composite mesh structure
544
545
546
    exchangeMeshStructureCodes(structures);

    // merge codes
547
548
549
    for (int rank = 0; rank < mpiSize; rank++) {
      if (rank != mpiRank) {
	structures[mpiRank].merge(&structures[rank]);
550
551
      }
    }
552
553
554
555
556
557
558
  
    // build finest mesh on the rank partition
    structures[mpiRank].fitMeshToStructure(mesh,
					   refinementManager,
					   true);

    DELETE [] structures;
559
560
561
562
563
564
565
566
  }


  bool ParallelProblem::writeElement(ElInfo *elInfo)
  {
    Element *element = elInfo->getElement();
    PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
      (element->getElementData(PARTITION_ED));
567
    TEST_EXIT_DBG(partitionData)("no partition data\n");
568
    PartitionStatus status = partitionData->getPartitionStatus();
569
    if (status == IN) 
570
571
572
573
574
575
      return true;
    else
      return false;
  }

  void ParallelProblem::exchangeRankSolutions(AdaptInfo *adaptInfo,
576
					      Mesh *workMesh,
577
578
579
580
					      std::vector<DOFVector<double>*> rankSolutions)
  {
    FUNCNAME("ParallelProblem::exchangeRankSolutions()");

581
    ParallelProblem::fillVertexPartitions(localCoarseGridLevel_, 1, true, overlapDistance_);
582
583
584
    overlapDistance_.clear();

    const FiniteElemSpace *feSpace = rankSolutions[0]->getFESpace();
585
    int dim = workMesh->getDim();
586
587
588
589
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *coarseDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DegreeOfFreedom *fineDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
590
    DOFAdmin *admin = feSpace->getAdmin();
591

592
593
    std::vector<std::vector<DegreeOfFreedom> > sendOrder(mpiSize);
    std::vector<std::vector<DegreeOfFreedom> > recvOrder(mpiSize);
594
595
596
597
598
599

    elementPartitions_.clear();

    int elementPartition = -1;
    Element *coarseElement = NULL;
    TraverseStack stack;
600
    ElInfo *elInfo = stack.traverseFirst(workMesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
601
    while (elInfo) {
602
603
604
605
606
      Element *element = elInfo->getElement();

      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

607
608
      if (partitionData) {
	if (partitionData->getLevel() == 0) {
609
	  elementPartition = partitionVec[element->getIndex()];
610
611
612
613
	}

	PartitionStatus status = partitionData->getPartitionStatus();

614
615
 	if (status != OUT) {
	  if (partitionData->getLevel() == localCoarseGridLevel_) {
616
	    basFcts->getLocalIndices(element, admin, coarseDOFs);
617
618

	    // collect other partitions element belongs to
619
	    for (int i = 0; i < dim + 1; i++) {
620
621
	      std::set<int>::iterator setBegin = vertexPartitions[coarseDOFs[i]].begin();
	      std::set<int>::iterator setEnd = vertexPartitions[coarseDOFs[i]].end();
622
623
	      for (std::set<int>::iterator setIt = setBegin; setIt != setEnd; ++setIt) {
		elementPartitions_[element].insert(*setIt);
624
625
626
627
628
629
630
	      }	
	    }

	    coarseElement = element;
	  }


631
	  if (element->isLeaf()) {
632
	    basFcts->getLocalIndices(element, admin, fineDOFs);
633

634
635
	    for (int i = 0; i < numFcts; i++) {
	      if (status == OVERLAP) {
636
637
638
		// send dofs
		sendOrder[elementPartition].push_back(fineDOFs[i]);
	      } 
639
	      if (status == IN) {
640
		// recv dofs
641
642
643
644
645
		TEST_EXIT(elementPartition == mpiRank)("???\n");
		std::set<int>::iterator setBegin = elementPartitions_[coarseElement].begin();
		std::set<int>::iterator setEnd = elementPartitions_[coarseElement].end();
		for (std::set<int>::iterator setIt = setBegin; setIt != setEnd; ++setIt) {
		  if (*setIt != mpiRank) {
646
647
648
649
650
651
652
653
654
655
656
657
658
 		    recvOrder[*setIt].push_back(fineDOFs[i]);
		  }
		}
	      }
	    }
	  }
	}
      }
      
      elInfo = stack.traverseNext(elInfo);
    }

    // create send and recv buffers and fill send buffers
659
    DOFVector<double> *solution = rankSolutions[mpiRank];
660
661
662
663
664
665

    std::map<int, double*> sendBuffer;
    std::map<int, double*> recvBuffer;
    std::map<int, int> sendBufferSize;
    std::map<int, int> recvBufferSize;

666
667
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
668
669
670
671
672
	int sendSize = static_cast<int>(sendOrder[partition].size());
	int recvSize = static_cast<int>(recvOrder[partition].size());

	sendBufferSize[partition] = sendSize;	
	recvBufferSize[partition] = recvSize;
673
	if (sendSize > 0) {
674
675
676
677
678
679
	  sendBuffer[partition] = GET_MEMORY(double, sendSize);
	  std::vector<DegreeOfFreedom>::iterator dofIt;
	  dofIt = sendOrder[partition].begin();
	  double *bufferIt, *bufferBegin, *bufferEnd;
	  bufferBegin = sendBuffer[partition];
	  bufferEnd = bufferBegin + sendSize;
680
681
682
	  for (bufferIt = bufferBegin; 
	       bufferIt < bufferEnd; 
	       ++bufferIt, ++dofIt) {
683
684
685
	    *bufferIt = (*solution)[*dofIt];
	  }
	}
686
	if (recvSize > 0) {
687
	  recvBuffer[partition] = GET_MEMORY(double, recvSize);
688
	}
689
690
691
692
      }
    }

    // non-blocking sends
693
694
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
695
696
697
698
699
700
	if (sendBufferSize[partition] > 0) {
	  mpiComm.Isend(sendBuffer[partition],
			sendBufferSize[partition],
			MPI_DOUBLE,
			partition,
			0);
701
702
703
	}
      }
    }    
704
   
705
    // blocking recieves
706
707
708
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (recvBufferSize[partition] > 0) {
709
710
711
712
713
	  mpiComm.Recv(recvBuffer[partition],
		       recvBufferSize[partition],
		       MPI_DOUBLE,
		       partition,
		       0);
714
715
716
717
718
	}
      }
    }    

    // wait for end of communication
719
    mpiComm.Barrier();
720
721

    // copy values into rank solutions
722
723
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
724
	std::vector<DegreeOfFreedom>::iterator dofIt = recvOrder[partition].begin();
725
726
	for (int i = 0; i < recvBufferSize[partition]; i++) {
	  (*(rankSolutions[partition]))[*dofIt] = recvBuffer[partition][i];
727
728
729
730
	  ++dofIt;
	}
      }
    }    
731

732
    // free send and recv buffers
733
734
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
735
736
737
738
739
740
	if (sendBufferSize[partition] > 0) {
	  FREE_MEMORY(sendBuffer[partition], double, sendBufferSize[partition]);
	}
	if (recvBufferSize[partition] > 0) {
	  FREE_MEMORY(recvBuffer[partition], double, recvBufferSize[partition]);
	}
741
742
      }
    }    
743

744
745
746
747
748
749
750
    FREE_MEMORY(coarseDOFs, DegreeOfFreedom, numFcts);
    FREE_MEMORY(fineDOFs, DegreeOfFreedom, numFcts);
  }

  void ParallelProblem::exchangeDOFVector(AdaptInfo *adaptInfo,
					  DOFVector<double> *values)
  {
751
752
    partitioner->fillLeafPartitionVec(&oldPartitionVec, &oldPartitionVec);
    partitioner->fillLeafPartitionVec(&partitionVec, &partitionVec);
753
754
755
756

    // === get send and recieve orders ===
    std::vector<std::vector<DegreeOfFreedom> > sendOrder;
    std::vector<std::vector<DegreeOfFreedom> > recvOrder;
757
758
    sendOrder.resize(mpiSize);
    recvOrder.resize(mpiSize);
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774

    int i;
    const FiniteElemSpace *feSpace = values->getFESpace();
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *dofs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DOFAdmin *admin =  feSpace->getAdmin();

    Mesh *mesh = feSpace->getMesh();
    TraverseStack stack;
    ElInfo *elInfo;

    elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
    while(elInfo) {
      Element *element = elInfo->getElement();
      int index = element->getIndex();
775
776
      int oldPartition = oldPartitionVec[index];
      int newPartition = partitionVec[index];
777
778
779
780
781

      if(oldPartition != newPartition) {
	// get dof indices
	basFcts->getLocalIndices(element, admin, dofs);

782
	if(oldPartition == mpiRank) {
783
784
785
786
787
	  for(i = 0; i < numFcts; i++) {
	    // send element values to new partition
	    sendOrder[newPartition].push_back(dofs[i]);
	  }
	}
788
	if(newPartition == mpiRank) {
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
	  for(i = 0; i < numFcts; i++) {
	    // recv element values from old partition
	    recvOrder[oldPartition].push_back(dofs[i]);
	  }
	}
      }

      elInfo = stack.traverseNext(elInfo);
    }

    FREE_MEMORY(dofs, DegreeOfFreedom, numFcts);

    // === create send and recv buffers and fill send buffers ===
    std::map<int, double*> sendBuffer;
    std::map<int, double*> recvBuffer;
    std::map<int, int> sendBufferSize;
    std::map<int, int> recvBufferSize;

    int partition;
808
809
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
810
811
812
813
814
	int sendSize = static_cast<int>(sendOrder[partition].size());
	int recvSize = static_cast<int>(recvOrder[partition].size());
      
	sendBufferSize[partition] = sendSize;	
	recvBufferSize[partition] = recvSize;
815
	if (sendSize > 0) {
816
817
818
819
820
821
	  sendBuffer[partition] = GET_MEMORY(double, sendSize);
	  std::vector<DegreeOfFreedom>::iterator dofIt;
	  dofIt = sendOrder[partition].begin();
	  double *bufferIt, *bufferBegin, *bufferEnd;
	  bufferBegin = sendBuffer[partition];
	  bufferEnd = bufferBegin + sendSize;
822
823
824
	  for (bufferIt = bufferBegin; 
	       bufferIt < bufferEnd; 
	       ++bufferIt, ++dofIt) {
825
826
827
	    *bufferIt = (*values)[*dofIt];
	  }
	}
828
	if (recvSize > 0)
829
830
831
832
833
	  recvBuffer[partition] = GET_MEMORY(double, recvSize);
      }
    }

    // === non-blocking sends ===
834
835
836
837
838
839
840
841
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (sendBufferSize[partition] > 0) {
	  mpiComm.Isend(sendBuffer[partition],
			  sendBufferSize[partition],
			  MPI_DOUBLE,
			  partition,
			  0);
842
843
844
845
846
	}
      }
    }    
    
    // === blocking receives ===
847
848
849
850
851
852
853
854
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (recvBufferSize[partition] > 0) {
	  mpiComm.Recv(recvBuffer[partition],
			 recvBufferSize[partition],
			 MPI_DOUBLE,
			 partition,
			 0);
855
856
857
858
859
	}
      }
    }    

    // === wait for end of MPI communication ===
860
    mpiComm.Barrier();
861
862

    // === copy received values into DOFVector ===
863
864
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
865
	std::vector<DegreeOfFreedom>::iterator dofIt = recvOrder[partition].begin();
866
	for (i = 0; i < recvBufferSize[partition]; i++) {
867
868
869
870
871
872
873
	  (*values)[*dofIt] = recvBuffer[partition][i];
	  ++dofIt;
	}
      }
    }    

    // === free send and receive buffers ===
874
875
876
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (sendBufferSize[partition] > 0)
877
878
879
	  FREE_MEMORY(sendBuffer[partition], 
		      double,
		      sendBufferSize[partition]);
880
	if (recvBufferSize[partition] > 0)
881
882
883
884
885
886
887
888
889
890
891
892
893
894
	  FREE_MEMORY(recvBuffer[partition], 
		      double,
		      recvBufferSize[partition]);
      }
    }
  }

  void ParallelProblem::buildGlobalSolution(AdaptInfo *adaptInfo,
					    std::vector<DOFVector<double>*> rankSolutions,
					    DOFVector<double> *globalSolution)
  {
    FUNCNAME("ParallelProblem::buildGlobalSolution()");

    const FiniteElemSpace *feSpace = globalSolution->getFESpace();
895
    int dim = mesh->getDim();
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *coarseDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DegreeOfFreedom *fineDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DOFAdmin *admin =  feSpace->getAdmin();

    Lagrange *linearFunctions = Lagrange::getLagrange(dim, 1);

    MSG("Building global solution\n");

    // compute w[DOF][partition]->value
    std::map<DegreeOfFreedom, std::map<int, double> > w;
    std::map<DegreeOfFreedom, std::map<int, double> >::iterator wIt, wBegin, wEnd;

    std::map<DegreeOfFreedom, double> sumW;

    Element *lastCoarseElement = NULL;

    WorldVector<double> worldCoord;
    DimVec<double> baryCoord(dim, NO_INIT);

    std::set<int>::iterator partIt, partBegin, partEnd;

    std::map<DegreeOfFreedom, bool> visited;


922
    MyDualTraverse dualTraverse(localCoarseGridLevel_);
923
924
925
    ElInfo *elInfo1, *elInfo2;
    ElInfo *large, *small;

926
927
928
929
930
931
932
933
934
    bool cont = dualTraverse.traverseFirst(mesh, mesh,
					   -1, -1,
					   Mesh::CALL_EVERY_EL_PREORDER |
					   Mesh::FILL_COORDS | 
					   Mesh::FILL_DET,
					   Mesh::CALL_LEAF_EL | 
					   Mesh::FILL_COORDS,
					   &elInfo1, &elInfo2,
					   &small, &large);
935
936
937
938
939
940
941

    while (cont) {
      Element *element1 = elInfo1->getElement();
      Element *element2 = elInfo2->getElement();
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>
	(element1->getElementData(PARTITION_ED));
942
      if (partitionData->getPartitionStatus() == IN) {
943
944

	// get coarse dofs
945
	if (element1 != lastCoarseElement) {
946
947
948
949
	  basFcts->getLocalIndices(element1, admin, coarseDOFs);
	  lastCoarseElement = element1;
	}
      
950
	if (elementPartitions_[element1].size() > 1) {
951
952
953
954
	  // get fine dofs
	  basFcts->getLocalIndices(element2, admin, fineDOFs);
      
	  // for all fine DOFs
955
956
	  for (int i = 0; i < numFcts; i++) {
	    if (!visited[fineDOFs[i]]) {
957
958
	      visited[fineDOFs[i]] = true;

959
	      elInfo2->coordToWorld(*(basFcts->getCoords(i)), worldCoord);
960
961
962
	      elInfo1->worldToCoord(worldCoord, &baryCoord);

	      // for all coarse vertex DOFs
963
	      for (int j = 0; j < dim + 1; j++) {
964
965
		partBegin = vertexPartitions[coarseDOFs[j]].begin();
		partEnd = vertexPartitions[coarseDOFs[j]].end();
966
		for (partIt = partBegin; partIt != partEnd; ++partIt) {
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
		  int partition = *partIt/* - 1*/;
		  double val = (*(linearFunctions->getPhi(j)))(baryCoord);
		  w[fineDOFs[i]][partition] += val;

		  sumW[fineDOFs[i]] += val;
		}
	      }
	    }
	  }
	}
      }
      cont = dualTraverse.traverseNext(&elInfo1, &elInfo2,
				       &small, &large);
    }

    FREE_MEMORY(coarseDOFs, DegreeOfFreedom, numFcts);
    FREE_MEMORY(fineDOFs, DegreeOfFreedom, numFcts);

    MSG("PU ...\n");

    wBegin = w.begin();
    wEnd = w.end();

990
    for (wIt = wBegin; wIt != wEnd; ++wIt) {
991
992
993
994
      DegreeOfFreedom dof = wIt->first;
      (*globalSolution)[dof] = 0.0;
    }
    
995
    for (wIt = wBegin; wIt != wEnd; ++wIt) {
996
997
998
999
1000
      DegreeOfFreedom dof = wIt->first;
      std::map<int, double>::iterator partIt, partBegin, partEnd;
      partBegin = wIt->second.begin();
      partEnd = wIt->second.end();
    
1001
      for (partIt = partBegin; partIt != partEnd; ++partIt) {
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
	int partition = partIt->first;
	double wDOF = partIt->second;
	(*globalSolution)[dof] += wDOF / sumW[dof] * (*(rankSolutions[partition]))[dof];
      }
    }
  }

  double ParallelProblem::errors2map(std::map<int, double> &errVec, 
				     int comp,
				     bool add)
  {
    int elNum = -1;
    double totalErr, error;

1016
1017
    if (!add) 
      errVec.clear();
1018
1019

    TraverseStack stack;
1020
    ElInfo *elInfo = stack.traverseFirst(mesh, 
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
					 -1,
					 Mesh::CALL_EVERY_EL_PREORDER);
    totalErr = 0.0;
    while(elInfo) {
      Element *element = elInfo->getElement();

      // get partition data
      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

      if(partitionData && partitionData->getPartitionStatus() == IN) {
	if(partitionData->getLevel() == 0) {
	  elNum = element->getIndex();
	}
	TEST_EXIT(elNum != -1)("invalid element number\n");
	if(element->isLeaf()) {
	  error = element->getEstimation(comp);
	  errVec[elNum] += error;
	  totalErr += error;
	}
      }
      elInfo = stack.traverseNext(elInfo);
    }

    return totalErr;
  }

  double ParallelProblem::setElemWeights(AdaptInfo *adaptInfo) 
  {
    double localWeightSum = 0.0;
    int elNum = -1;

    elemWeights_.clear();

    TraverseStack stack;
1056
    ElInfo *elInfo = stack.traverseFirst(mesh, 
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
					 -1,
					 Mesh::CALL_EVERY_EL_PREORDER);
    while(elInfo) {
      Element *element = elInfo->getElement();

      // get partition data
      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

      if(partitionData && partitionData->getPartitionStatus() == IN) {
	if(partitionData->getLevel() == 0) {
	  elNum = element->getIndex();
	}
	TEST_EXIT(elNum != -1)("invalid element number\n");
	if(element->isLeaf()) {
	  elemWeights_[elNum] += 1.0;
	  localWeightSum += 1.0;
	}
      }

      elInfo = stack.traverseNext(elInfo);
    }

    return localWeightSum;
  }


  void ParallelProblem::globalRefinements() 
  {
    if (globalRefinements_ <= 0) 
      return;

    TraverseStack stack;
1090
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
    while (elInfo) {
      Element *element = elInfo->getElement();
      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

      if (partitionData && partitionData->getPartitionStatus() == IN) {
	element->setMark(globalRefinements_);
      }
      
      elInfo = stack.traverseNext(elInfo);
    }

1103
    refinementManager->refineMesh(mesh);
1104
1105
1106
1107
1108
1109
  }


  void ParallelProblem::createOverlap(int level, int overlap, bool openOverlap,
				      std::map<Element*, int> &overlapDistance)
  {
1110
    int dim = mesh->getDim();
1111
1112
1113
1114
1115
1116

    // === create dual graph (one common node) and prepare breadth-first search ===
    std::map<DegreeOfFreedom, std::vector<Element*> > vertexElements;
    std::queue<Element*> overlapQueue;

    TraverseStack stack;
1117
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
1118
    while (elInfo) {
1119
1120
1121
1122
      Element *element = elInfo->getElement();
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
      
1123
1124
1125
      if (partitionData) {
	if (partitionData->getLevel() == level) {
	  for (int i = 0; i < dim + 1; i++) {	    
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
	    vertexElements[element->getDOF(i, 0)].push_back(element);
	  }

	  if(partitionData->getPartitionStatus() == IN) {
	    overlapDistance[element] = 0;
	    overlapQueue.push(element);
	  } else {
	    overlapDistance[element] = -1; // still unknown
	  }
	}
      }
      elInfo = stack.traverseNext(elInfo);
    }

    // === breadth-first search on dual graph ===
    std::vector<Element*>::iterator it, itBegin, itEnd;

1143
    while (!overlapQueue.empty()) {
1144
1145
1146
1147
1148
1149
1150
1151
      // get first element in queue
      Element *element = overlapQueue.front();
      overlapQueue.pop();

      // get distance
      int distance = overlapDistance[element];
      TEST_EXIT(distance >= 0)("invalid distance\n");

1152
1153
      if (distance >= overlap) 
	continue;
1154
1155
      
      // get all adjacent elements
1156
      for (int i = 0; i < dim + 1; i++) {
1157
1158
	itBegin = vertexElements[element->getDOF(i, 0)].begin();
	itEnd = vertexElements[element->getDOF(i, 0)].end();
1159
1160
	for (it = itBegin; it != itEnd; ++it) {
	  if (overlapDistance[*it] == -1) {
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
	    // set distance for new member
	    overlapDistance[*it] = distance + 1;
	    // push neighbour to queue
	    overlapQueue.push(*it);
	    // mark as overlap on AMDiS mesh	    
	    PartitionElementData *partitionData = 
	      dynamic_cast<PartitionElementData*>((*it)->getElementData(PARTITION_ED));
	    TEST_EXIT(partitionData)("no partition data\n");
	    partitionData->setPartitionStatus(OVERLAP);
	    partitionData->descend(*it);
	  }
	}
      }
    }
  }

  void ParallelProblem::fillVertexPartitions(int level, int overlap, 
					     bool openOverlap,
					     std::map<Element*, int> &overlapDistance)
  {
1181
    int dim = mesh->getDim();
1182
1183

    // clear partition dof vector
1184
    vertexPartitions.clear();
1185
1186

    // first: partition elements ...
1187
1188
1189
    int partition;
    TraverseStack stack;
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
1190
    while (elInfo) {
1191
1192
1193
      Element *element = elInfo->getElement();
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
1194
1195
      if (partitionData) {
	if (partitionData->getLevel() == 0) {
1196
	  partition = partitionVec[element->getIndex()];
1197
	}
1198
1199
	if (partitionData->getLevel() == level) {
	  for (int i = 0; i < dim + 1; i++) {
1200
	    vertexPartitions[element->getDOF(i, 0)].insert(partition);
1201
1202
1203
1204
1205
1206
	  }
	}
      }
      elInfo = stack.traverseNext(elInfo);
    }

1207
    if (overlap > 1 || openOverlap == false) {
1208
      // exchange mesh structure codes
1209
      MeshStructure *structures = NEW MeshStructure[mpiSize];
1210
1211
1212
      exchangeMeshStructureCodes(structures);

      // merge codes
1213
1214
      for (int rank = 0; rank < mpiSize; rank++) {
	if (rank != mpiRank) {
1215
	  structures[mpiRank].merge(&structures[rank]);
1216
1217
1218
	}
      }
    
1219
      MeshStructure &compositeStructure = structures[mpiRank];
1220
1221
1222
1223
1224
1225
      compositeStructure.reset();

      // get composite indices of local overlap elements
      std::map<int, Element*> indexElement;
      std::vector<int> innerOverlapElements; // not at open overlap boundary

1226
1227
      elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
      while (elInfo) {
1228
1229
1230
1231
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>(element->getElementData(PARTITION_ED));
	
1232
	if (partitionData && partitionData->getLevel() == level) {
1233
1234
	  int compositeIndex = compositeStructure.getCurrentElement();
	  indexElement[compositeIndex] = element;
1235
	  if (partitionData->getPartitionStatus() == OVERLAP) {
1236
	    int distance = overlapDistance[element];
1237
	    if (distance < overlap || !openOverlap) {
1238
1239
1240
1241
1242
	      innerOverlapElements.push_back(compositeIndex);
	    } 
	  }
	}
	
1243
	if (element->isLeaf()) {
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
	  compositeStructure.skipBranch();
	} else {
	  compositeStructure.nextElement();
	}
	elInfo = stack.traverseNext(elInfo);
      }
    
      // === exchange 'inner' overlap elements ===

      // exchange number of overlap elements
1254
      int *numOverlapElements = GET_MEMORY(int, mpiSize);
1255
      int tmp = static_cast<int>(innerOverlapElements.size());
1256
      mpiComm.Allgather(&tmp, 1, MPI_INT, numOverlapElements, 1, MPI_INT);
1257
1258
      
      // exchange overlap elements
1259
      int *offset = GET_MEMORY(int, mpiSize);
1260
      int sum = 0;
1261
      for (int rank = 0; rank < mpiSize; rank++) {
1262
1263
1264
1265
1266
	offset[rank] = sum;
	sum += numOverlapElements[rank];
      }

      int *recvBuffer = GET_MEMORY(int, sum);
1267
      int *sendBuffer = GET_MEMORY(int, numOverlapElements[mpiRank]);
1268
1269
1270
1271
   
      int *ptr;
      std::vector<int>::iterator elemIt, elemEnd = innerOverlapElements.end();
  
1272
1273
1274
      for (ptr = sendBuffer, elemIt = innerOverlapElements.begin();
	   elemIt != elemEnd; 
	   ++elemIt, ++ptr) {
1275
1276
1277
	*ptr = *elemIt;
      }
  
1278
1279
      mpiComm.Allgatherv(sendBuffer, numOverlapElements[mpiRank], MPI_INT, 
			 recvBuffer, numOverlapElements, offset, MPI_INT);
1280
1281
1282


      // fill vertexPartitions for 'inner' overlap elements
1283
      for (int rank = 0; rank < mpiSize; rank++) {
1284
1285
	int numElements = numOverlapElements[rank];
	int *elements = recvBuffer + offset[rank];
1286
	for (int el = 0; el < numElements; el++) {
1287
	  Element *element = indexElement[elements[el]];
1288
	  for (int i = 0; i < dim + 1; i++) {
1289
	    vertexPartitions[element->getDOF(i, 0)].insert(rank/* + 1*/);
1290
1291
1292
1293
1294
1295
1296
	  }
	}
      }

      // free memory
      DELETE [] structures;
      FREE_MEMORY(recvBuffer, int, sum);
1297
1298
      FREE_MEMORY(sendBuffer, int, numOverlapElements[mpiRank]);
      FREE_MEMORY(numOverlapElements, int, mpiSize);
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
    }
  }

  void ParallelProblem::createOverlap(AdaptInfo *adaptInfo)
  {
    int level = localCoarseGridLevel_, overlap = 1;
    bool openOverlap = true;

    ParallelProblem::createOverlap(level, overlap, openOverlap, overlapDistance_);
  }

  // =========================================================================
  // ===== class ParallelProblemScal =========================================
  // =========================================================================
  
  ParallelProblemScal::ParallelProblemScal(const std::string& name,
1315
					   ProblemScal *prob,
1316
1317
1318
					   ProblemInstatScal *problemInstat,
					   std::vector<DOFVector<double>*> vectors)
    : ParallelProblem(name,