ParallelProblem.cc 55.5 KB
Newer Older
1
#include "ParallelProblem.h"
2
3
4
5
6
7
8
9
10
11
#include "ProblemScal.h"
#include "ProblemVec.h"
#include "ProblemInstat.h"
#include "AdaptInfo.h"
#include "AdaptStationary.h"
#include "ConditionalEstimator.h"
#include "ConditionalMarker.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
12
#include "MacroElement.h"
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#include "PartitionElementData.h"
#include "ParMetisPartitioner.h"
#include "Mesh.h"
#include "DualTraverse.h"
#include "MeshStructure.h"
#include "DOFVector.h"
#include "FiniteElemSpace.h"
#include "RefinementManager.h"
#include "CoarseningManager.h"
#include "Lagrange.h"
#include "ElementFileWriter.h"
#include "MacroWriter.h"
#include "ValueWriter.h"
#include "SystemVector.h"
27
#include "VtkWriter.h"
28
29
#include "mpi.h"
#include <queue>
30
#include <time.h>
31
32
33
34
35

namespace AMDiS {

  bool elementInPartition(ElInfo *elInfo)
  {
36
37
38
39
40
41
42
    PartitionElementData *elementData = dynamic_cast<PartitionElementData*>
      (elInfo->getElement()->getElementData(PARTITION_ED));
    if (elementData && elementData->getPartitionStatus() == IN) {
      return true;
    } else {
      return false;
    }
43
44
45
46
47
48
49
50
51
52
53
54
55
  }

  class MyDualTraverse : public DualTraverse
  {
  public:
    MyDualTraverse(int coarseLevel)
      : coarseLevel_(coarseLevel)
    {};

    bool skipEl1(ElInfo *elInfo)
    {
      PartitionElementData *elementData = dynamic_cast<PartitionElementData*>
	(elInfo->getElement()->getElementData(PARTITION_ED));
56
57
      if (elementData) {
	if (elInfo->getElement()->isLeaf() && 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
	   elementData->getLevel() < coarseLevel_)
	  return false;
	if(elementData->getLevel() == coarseLevel_)
	  return false;
      }
      return true;
    };
  private:
    int coarseLevel_;
  };

  // =========================================================================
  // ===== class ParallelProblemBase =========================================
  // =========================================================================

73
74
  ParallelProblemBase::ParallelProblemBase(const std::string& name,
					   ProblemIterationInterface *iterationIF,
75
76
					   ProblemTimeInterface *timeIF)
    : iterationIF_(iterationIF),
77
      timeIF_(timeIF),
78
79
      debugMode(0),
      debugServerProcess(false)
80
  {
81
82
    mpiRank = MPI::COMM_WORLD.Get_rank();
    mpiSize = MPI::COMM_WORLD.Get_size();
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

    GET_PARAMETER(0, name + "->debug mode", "%d", &debugMode);

    if (debugMode) {
      MPI::Group group = MPI::COMM_WORLD.Get_group();
      
      int rankSize = mpiSize - 1;
      int *ranks = GET_MEMORY(int, rankSize);
      for (int i = 0; i < rankSize; i++) {
	ranks[i] = i + 1;
      }
      
      amdisGroup = group.Incl(rankSize, ranks);
      
      mpiComm = MPI::COMM_WORLD.Create(amdisGroup);
      
      if (mpiComm != MPI::COMM_NULL) {
	mpiRank = mpiComm.Get_rank();
	mpiSize = mpiComm.Get_size();
	debugServerProcess = false;
      } else {
	debugServerProcess = true;
      }
      
      FREE_MEMORY(ranks, int, rankSize);
    } else {
      mpiComm = MPI::COMM_WORLD;
    }
111
112
  }

113
114
115
116
117
118
  void ParallelProblemBase::exitParallelization(AdaptInfo *adaptInfo)
  {
    if (!timeIF_) 
      closeTimestep(adaptInfo);

    amdisGroup.Free();
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  }

  void ParallelProblemBase::closeTimestep(AdaptInfo *adaptInfo)
  {
    if (mpiSize > 1 && doBuildGlobalSolution(adaptInfo)) {
      if (debugMode && mpiRank == 0) {
	// Send adaptInfo inforamtion to debug server
	double sendTime = adaptInfo->getTime();
	double sendTimestep = adaptInfo->getTimestep();
	MPI::COMM_WORLD.Send(&sendTime, 1, MPI_DOUBLE, 0, 100);
	MPI::COMM_WORLD.Send(&sendTimestep, 1, MPI_DOUBLE, 0, 100);
      }
      synchronizeMeshes(adaptInfo);	
      exchangeRankSolutions(adaptInfo);
      buildGlobalSolution(adaptInfo);
    }
    
    if (timeIF_) 
      timeIF_->closeTimestep(adaptInfo);
  }
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184

  Flag ParallelProblemBase::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
  {
    Flag flag;
    
    if (mpiSize > 1 && toDo.isSet(MARK | ADAPT)) {
      flag = iterationIF_->oneIteration(adaptInfo, MARK | ADAPT);
      
      double localWeightSum = setElemWeights(adaptInfo);
      if (doPartitioning(adaptInfo, localWeightSum)) {
	clock_t partitioningStart = clock();

	synchronizeMeshes(adaptInfo);
	partitionMesh(adaptInfo);
	refineOverlap(adaptInfo);
	createOverlap(adaptInfo);
	synchronizeMeshes(adaptInfo);
	exchangeDOFVectors(adaptInfo);
	coarsenOutOfPartition(adaptInfo);
	
	clock_t partitioningEnd = clock();
	partitioningTime = TIME_USED(partitioningStart, 
				     partitioningEnd);
	computationStart = partitioningEnd;
      }
      
      flag |= iterationIF_->oneIteration(adaptInfo, toDo & ~(MARK | ADAPT));
    } else {
      flag = iterationIF_->oneIteration(adaptInfo, toDo);
    }
    
    // synchronize adaption flag
    unsigned long *flagBuffer = GET_MEMORY(unsigned long, mpiSize);
    
    unsigned long localFlag = flag.getFlags();
    
    mpiComm.Allgather(&localFlag, 1, MPI_UNSIGNED_LONG,
		      flagBuffer, 1, MPI_UNSIGNED_LONG);
    for (int i = 0; i < mpiSize; i++) {
      flag.setFlag(flagBuffer[i]);
    }
    FREE_MEMORY(flagBuffer, unsigned long, mpiSize);
    
    return flag;
  };

185
186
187
188
189
190
191
192
  // =========================================================================
  // ===== class ParallelProblem =============================================
  // =========================================================================

  ParallelProblem::ParallelProblem(const std::string& name,
				   ProblemIterationInterface *iterationIF,
				   ProblemTimeInterface *timeIF,
				   std::vector<DOFVector<double>*> vectors,
193
				   Mesh *mesh_,
194
195
				   RefinementManager *rm,
				   CoarseningManager *cm)
196
    : ParallelProblemBase(name, iterationIF, timeIF),
197
      name_(name),
198
      mesh(mesh_),
199
200
      refinementManager(rm),
      coarseningManager(cm),
201
202
203
204
205
206
207
208
209
210
211
      repartitionSteps_(1),
      puEveryTimestep_(false),
      dofVectors_(vectors),
      upperPartThreshold_(1.5),
      lowerPartThreshold_(2.0/3.0),
      globalCoarseGridLevel_(0),
      localCoarseGridLevel_(0),
      globalRefinements_(0),
      adaptiveThresholds_(0),
      thresholdIncFactor_(2.0),
      thresholdDecFactor_(0.5),
212
      repartTimeFactor_(10.0)
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
  {
    GET_PARAMETER(0, name_ + "->upper part threshold", "%f", 
		  &upperPartThreshold_);
    GET_PARAMETER(0, name_ + "->lower part threshold", "%f", 
		  &lowerPartThreshold_);
    GET_PARAMETER(0, name_ + "->global coarse grid level", "%d", 
		  &globalCoarseGridLevel_);
    GET_PARAMETER(0, name_ + "->local coarse grid level", "%d", 
		  &localCoarseGridLevel_);
    GET_PARAMETER(0, name_ + "->global refinements", "%d", 
		  &globalRefinements_);


    TEST_EXIT(localCoarseGridLevel_ >= globalCoarseGridLevel_)
      ("local coarse grid level < global coarse grid level\n");

229
    partitioner = NEW ParMetisPartitioner(mesh, &mpiComm);
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253

    GET_PARAMETER(0, name_ + "->adaptive thresholds", "%d", 
		  &adaptiveThresholds_);
    GET_PARAMETER(0, name_ + "->threshold inc factor", "%f", 
		  &thresholdIncFactor_);
    GET_PARAMETER(0, name_ + "->threshold dec factor", "%f", 
		  &thresholdDecFactor_);
    GET_PARAMETER(0, name_ + "->repart time factor", "%f", 
		  &repartTimeFactor_);


    TEST_EXIT(lowerPartThreshold_ <= 1.0)("invalid lower part threshold\n");
    TEST_EXIT(upperPartThreshold_ >= 1.0)("invalid upper part threshold\n");

    if (adaptiveThresholds_) {
      TEST_EXIT(thresholdDecFactor_ <= 1.0)("invalid threshold dec factor\n");
      TEST_EXIT(thresholdIncFactor_ >= 1.0)("invalid threshold inc factor\n");
    }
    minUpperTH_ = upperPartThreshold_;
    maxLowerTH_ = lowerPartThreshold_;
  }

  ParallelProblem::~ParallelProblem() 
  {
254
    DELETE partitioner;
255
256
257
258
259
260
  }

  bool ParallelProblem::doPartitioning(AdaptInfo *adaptInfo, double localWeightSum) 
  {
    FUNCNAME("ParallelProblem::doPartitioning()");

261
262
263
    double *weightSum = GET_MEMORY(double, mpiSize);
    int *partArray = GET_MEMORY(int, mpiSize);
    int part = 0;
264

265
    mpiComm.Gather(&localWeightSum, 1, MPI_DOUBLE, weightSum, 1, MPI_DOUBLE, 0);
266

267
    if (mpiRank == 0) {
268
269

      double average = 0.0;
270
      for (int i = 0; i < mpiSize; i++) {
271
272
273
	average += weightSum[i];
      }

274
      average /= mpiSize;
275

276
277
      for (int i = 0; i < mpiSize; i++) {
	if ((weightSum[i] / average) > upperPartThreshold_) {
278
279
280
	  part = 1;
	  break;
	}
281
	if ((weightSum[i] / average) < lowerPartThreshold_) {
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
	  part = 1;
	  break;
	}
      }

      double computationTime = TIME_USED(computationStart, clock());
      if (adaptiveThresholds_) {

	bool timeOver = ((computationTime / partitioningTime) >= repartTimeFactor_);

	if (part == 1 && !timeOver) {
	  // inc thresholds
	  upperPartThreshold_ *= thresholdIncFactor_;
	  lowerPartThreshold_ /= thresholdIncFactor_;

	  // avoid repartitioning
	  part = 0;
	}
      
	if (part == 0 && timeOver) {
	  // dec thresholds
	  upperPartThreshold_ *= thresholdDecFactor_;
	  lowerPartThreshold_ /= thresholdDecFactor_;

	  upperPartThreshold_ = max(minUpperTH_, upperPartThreshold_);
	  lowerPartThreshold_ = min(maxLowerTH_, lowerPartThreshold_);
	}
      }

311
      for (int i = 0; i < mpiSize; i++) {
312
313
314
315
	partArray[i] = part;
      }      
    }

316
317
    mpiComm.Scatter(partArray, 1, MPI_INT,
		    &part, 1, MPI_INT, 0);
318
    
319
320
    FREE_MEMORY(weightSum, double, mpiSize);
    FREE_MEMORY(partArray, int, mpiSize);
321
322
323
324
325
326
327
328
329
330
331

    return (part == 1);
  }

  bool ParallelProblem::doBuildGlobalSolution(AdaptInfo *adaptInfo) {
    return true;
  }

  void ParallelProblem::partitionMesh(AdaptInfo *adaptInfo)
  {
    static bool initial = true;
332

333
    if (initial) {
334
      initial = false;
335
      partitioner->fillCoarsePartitionVec(&oldPartitionVec);
Thomas Witkowski's avatar
Thomas Witkowski committed
336
      partitioner->partition(&elemWeights, INITIAL);
337
    } else {
338
      oldPartitionVec = partitionVec;
Thomas Witkowski's avatar
Thomas Witkowski committed
339
      partitioner->partition(&elemWeights, ADAPTIVE_REPART, 100.0 /*0.000001*/);
340
341
    }    

342
    partitioner->fillCoarsePartitionVec(&partitionVec);
343
344
345
346
  }

  void ParallelProblem::refineOverlap(AdaptInfo *adaptInfo)
  {
347
    int dim = mesh->getDim();
348
349
    bool finished = (localCoarseGridLevel_ == 0);

350
    while (!finished) {
351
352
353
354
      std::map<DegreeOfFreedom, int> inOut; // 1: in, 2: out, 3: border dof

      // mark in/out/border dofs
      TraverseStack stack;
355
      ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
356
      while (elInfo) {
357
358
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
359
360
	  dynamic_cast<PartitionElementData*>(elInfo->getElement()->
					      getElementData(PARTITION_ED));
361
362
363

	const DegreeOfFreedom **dofs = element->getDOF();

364
365
	if (partitionData->getPartitionStatus() == IN) {
	  for (int i = 0; i < dim + 1; i++) {
366
	    DegreeOfFreedom dof = dofs[i][0];
367
368
369
370
	    if (inOut[dof] == 2) 
	      inOut[dof] = 3;
	    if (inOut[dof] == 0) 
	      inOut[dof] = 1;
371
372
	  }
	} else {
373
	  for (int i = 0; i < dim + 1; i++) {
374
	    DegreeOfFreedom dof = dofs[i][0];
375
376
377
378
	    if (inOut[dof] == 1) 
	      inOut[dof] = 3;
	    if (inOut[dof] == 0) 
	      inOut[dof] = 2;
379
380
381
382
383
384
385
386
387
	  }
	}

	elInfo = stack.traverseNext(elInfo);
      }

      // refine overlap-border and inner elements
      finished = true;
      bool marked = false;
388
      elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
389
      while (elInfo) {
390
391
392
393
394
395
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>(elInfo->getElement()->getElementData(PARTITION_ED));

	int level = partitionData->getLevel();

396
397
	if (level < localCoarseGridLevel_) {
	  if (partitionData->getPartitionStatus() != IN) {
398
	    const DegreeOfFreedom **dofs = element->getDOF();
399
	    for (int i = 0; i < dim + 1; i++) {
400
	      DegreeOfFreedom dof = dofs[i][0];
401
	      if (inOut[dof] == 3) {
402
403
		element->setMark(1);
		marked = true;
404
405
		if ((level + 1) < localCoarseGridLevel_) 
		  finished = false;
406
407
408
409
410
411
		break;
	      }
	    }
	  } else {
	    element->setMark(1);
	    marked = true;
412
413
	    if ((level + 1) < localCoarseGridLevel_) 
	      finished = false;
414
415
416
417
418
	  }
	}

	elInfo = stack.traverseNext(elInfo);
      }
419
420
      if (marked) 
	refinementManager->refineMesh(mesh);
421
422
423
424
425
426
    }
  }

  void ParallelProblem::globalRefineOutOfPartition(AdaptInfo *adaptInfo)
  {
    TraverseStack stack;
427
    ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
428
    while (elInfo) {
429
      PartitionElementData *partitionData = 
430
431
	dynamic_cast<PartitionElementData*>(elInfo->getElement()->
					    getElementData(PARTITION_ED));
432
433
434
435
436
      int refinements = globalCoarseGridLevel_ - partitionData->getLevel();
      elInfo->getElement()->setMark(max(0, refinements));
      elInfo = stack.traverseNext(elInfo);
    }

437
    refinementManager->refineMesh(mesh);
438
439
440
441
442
  }

  void ParallelProblem::coarsenOutOfPartition(AdaptInfo *adaptInfo)
  {
    Flag meshCoarsened = 1;    
443
    while (meshCoarsened.getFlags() != 0) {
444
      TraverseStack stack;
445
      ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
446
      while (elInfo) {
447
448
449
450
	Element *element = elInfo->getElement();
	PartitionElementData *partitionData = 
	  dynamic_cast<PartitionElementData*>
	  (element->getElementData(PARTITION_ED));
451
	if (partitionData->getPartitionStatus() == OUT) {
452
453
454
455
456
	  int mark = min(0, -partitionData->getLevel() + globalCoarseGridLevel_);
	  element->setMark(mark);
	}
	elInfo = stack.traverseNext(elInfo);
      }
457
      meshCoarsened = coarseningManager->coarsenMesh(mesh);
458
    }
459
    mpiComm.Barrier();
460
461
462
463
  }

  void ParallelProblem::exchangeMeshStructureCodes(MeshStructure *structures)
  {
464
465
466
    // every process creates a mesh structure code from its mesh.
    structures[mpiRank].init(mesh);
    const std::vector<unsigned long int>& myCode = structures[mpiRank].getCode();
467
468

    // broadcast code sizes
469
    int *codeSize = GET_MEMORY(int, mpiSize);
470
    int tmp = static_cast<int>(myCode.size());
471
472
473
474
475
    mpiComm.Allgather(&tmp, 1, MPI_INT, codeSize, 1, MPI_INT);
    if (debugMode) {
      // send code sizes also to debug server
      MPI::COMM_WORLD.Gather(&tmp, 1, MPI_INT, NULL, 1, MPI_INT, 0);
    }
476
477

    // broadcast number of elements
478
479
    int *elements = GET_MEMORY(int, mpiSize);
    tmp = structures[mpiRank].getNumElements();
480
481
482
483
484
    mpiComm.Allgather(&tmp, 1, MPI_INT, elements, 1, MPI_INT);
    if (debugMode) {
      // send number of elements also to debug server
      MPI::COMM_WORLD.Gather(&tmp, 1, MPI_INT, NULL, 1, MPI_INT, 0);
    }
485
486

    // broadcast codes
487
    int *codeOffset = GET_MEMORY(int, mpiSize);
488
    int codeSizeSum = 0;
489
    for (int rank = 0; rank < mpiSize; rank++) {
490
491
492
493
494
      codeOffset[rank] = codeSizeSum;
      codeSizeSum += codeSize[rank];
    }

    unsigned long int *code = GET_MEMORY(unsigned long int, codeSizeSum);
495
    unsigned long int *localCode = GET_MEMORY(unsigned long int, codeSize[mpiRank]);  
496
497
498
    unsigned long int *ptr;
    std::vector<unsigned long int>::const_iterator it, end = myCode.end();
  
499
500
    for (ptr = localCode, it = myCode.begin();
	 it != end; 
501
	 ++it, ++ptr) {
502
503
      *ptr = *it;
    }
504
505
506
507
508
509
510
511
512
513

    mpiComm.Allgatherv(localCode, codeSize[mpiRank], 
		       MPI_UNSIGNED_LONG, 
		       code, codeSize, codeOffset,
		       MPI_UNSIGNED_LONG);
    if (debugMode) {
      // send codes also to debug server
      MPI::COMM_WORLD.Send(localCode, codeSize[mpiRank],
			   MPI_UNSIGNED_LONG, 0, 100);
    }
514
    
515
516
    for (int rank = 0; rank < mpiSize; rank++) {
      if (rank != mpiRank) {
517
518
519
520
	std::vector<unsigned long int> remoteCode;
	unsigned long int *ptr;
	unsigned long int *begin = code + codeOffset[rank]; 
	unsigned long int *end = begin + codeSize[rank];
521
	for (ptr = begin; ptr != end; ++ptr) {
522
523
524
525
526
527
528
	  remoteCode.push_back(*ptr);
	}
	structures[rank].init(remoteCode, elements[rank]);
      }
    }

    // free memory
529
    FREE_MEMORY(elements, int, mpiSize);
530
    FREE_MEMORY(code, unsigned long int, codeSizeSum);
531
532
533
    FREE_MEMORY(localCode, unsigned long int, codeSize[mpiRank]);
    FREE_MEMORY(codeOffset, int, mpiSize);
    FREE_MEMORY(codeSize, int, mpiSize);
534
535
536
537
538
539
540
    
  }

  void ParallelProblem::synchronizeMeshes(AdaptInfo *adaptInfo)
  {
    FUNCNAME("ParallelProblem::synchronizeMeshes()");

541
    MeshStructure *structures = NEW MeshStructure[mpiSize];
542

543
    // build composite mesh structure
544
545
546
    exchangeMeshStructureCodes(structures);

    // merge codes
547
548
549
    for (int rank = 0; rank < mpiSize; rank++) {
      if (rank != mpiRank) {
	structures[mpiRank].merge(&structures[rank]);
550
551
      }
    }
552
553
554
555
556
557
558
  
    // build finest mesh on the rank partition
    structures[mpiRank].fitMeshToStructure(mesh,
					   refinementManager,
					   true);

    DELETE [] structures;
559
560
561
562
563
564
565
566
  }


  bool ParallelProblem::writeElement(ElInfo *elInfo)
  {
    Element *element = elInfo->getElement();
    PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
      (element->getElementData(PARTITION_ED));
567
    TEST_EXIT_DBG(partitionData)("no partition data\n");
568
    PartitionStatus status = partitionData->getPartitionStatus();
569
    if (status == IN) 
570
571
572
573
574
575
      return true;
    else
      return false;
  }

  void ParallelProblem::exchangeRankSolutions(AdaptInfo *adaptInfo,
576
					      Mesh *workMesh,
577
578
579
580
					      std::vector<DOFVector<double>*> rankSolutions)
  {
    FUNCNAME("ParallelProblem::exchangeRankSolutions()");

581
    ParallelProblem::fillVertexPartitions(localCoarseGridLevel_, 1, true, overlapDistance_);
582
583
584
    overlapDistance_.clear();

    const FiniteElemSpace *feSpace = rankSolutions[0]->getFESpace();
585
    int dim = workMesh->getDim();
586
587
588
589
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *coarseDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DegreeOfFreedom *fineDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
590
    DOFAdmin *admin = feSpace->getAdmin();
591

592
593
    std::vector<std::vector<DegreeOfFreedom> > sendOrder(mpiSize);
    std::vector<std::vector<DegreeOfFreedom> > recvOrder(mpiSize);
594
595
596
597
598
599

    elementPartitions_.clear();

    int elementPartition = -1;
    Element *coarseElement = NULL;
    TraverseStack stack;
600
    ElInfo *elInfo = stack.traverseFirst(workMesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
601
    while (elInfo) {
602
603
604
605
606
      Element *element = elInfo->getElement();

      PartitionElementData *partitionData = dynamic_cast<PartitionElementData*>
	(element->getElementData(PARTITION_ED));

607
608
      if (partitionData) {
	if (partitionData->getLevel() == 0) {
609
	  elementPartition = partitionVec[element->getIndex()];
610
611
612
613
	}

	PartitionStatus status = partitionData->getPartitionStatus();

614
615
 	if (status != OUT) {
	  if (partitionData->getLevel() == localCoarseGridLevel_) {
616
	    basFcts->getLocalIndices(element, admin, coarseDOFs);
617
618

	    // collect other partitions element belongs to
619
	    for (int i = 0; i < dim + 1; i++) {
620
621
	      std::set<int>::iterator setBegin = vertexPartitions[coarseDOFs[i]].begin();
	      std::set<int>::iterator setEnd = vertexPartitions[coarseDOFs[i]].end();
622
623
	      for (std::set<int>::iterator setIt = setBegin; setIt != setEnd; ++setIt) {
		elementPartitions_[element].insert(*setIt);
624
625
626
627
628
629
630
	      }	
	    }

	    coarseElement = element;
	  }


631
	  if (element->isLeaf()) {
632
	    basFcts->getLocalIndices(element, admin, fineDOFs);
633

634
635
	    for (int i = 0; i < numFcts; i++) {
	      if (status == OVERLAP) {
636
637
638
		// send dofs
		sendOrder[elementPartition].push_back(fineDOFs[i]);
	      } 
639
	      if (status == IN) {
640
		// recv dofs
641
642
643
644
645
		TEST_EXIT(elementPartition == mpiRank)("???\n");
		std::set<int>::iterator setBegin = elementPartitions_[coarseElement].begin();
		std::set<int>::iterator setEnd = elementPartitions_[coarseElement].end();
		for (std::set<int>::iterator setIt = setBegin; setIt != setEnd; ++setIt) {
		  if (*setIt != mpiRank) {
646
647
648
649
650
651
652
653
654
655
656
657
658
 		    recvOrder[*setIt].push_back(fineDOFs[i]);
		  }
		}
	      }
	    }
	  }
	}
      }
      
      elInfo = stack.traverseNext(elInfo);
    }

    // create send and recv buffers and fill send buffers
659
    DOFVector<double> *solution = rankSolutions[mpiRank];
660
661
662
663
664
665

    std::map<int, double*> sendBuffer;
    std::map<int, double*> recvBuffer;
    std::map<int, int> sendBufferSize;
    std::map<int, int> recvBufferSize;

666
667
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
668
669
670
671
672
	int sendSize = static_cast<int>(sendOrder[partition].size());
	int recvSize = static_cast<int>(recvOrder[partition].size());

	sendBufferSize[partition] = sendSize;	
	recvBufferSize[partition] = recvSize;
673
	if (sendSize > 0) {
674
675
676
677
678
679
	  sendBuffer[partition] = GET_MEMORY(double, sendSize);
	  std::vector<DegreeOfFreedom>::iterator dofIt;
	  dofIt = sendOrder[partition].begin();
	  double *bufferIt, *bufferBegin, *bufferEnd;
	  bufferBegin = sendBuffer[partition];
	  bufferEnd = bufferBegin + sendSize;
680
681
682
	  for (bufferIt = bufferBegin; 
	       bufferIt < bufferEnd; 
	       ++bufferIt, ++dofIt) {
683
684
685
	    *bufferIt = (*solution)[*dofIt];
	  }
	}
686
	if (recvSize > 0) {
687
	  recvBuffer[partition] = GET_MEMORY(double, recvSize);
688
	}
689
690
691
692
      }
    }

    // non-blocking sends
693
694
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
695
696
697
698
699
700
	if (sendBufferSize[partition] > 0) {
	  mpiComm.Isend(sendBuffer[partition],
			sendBufferSize[partition],
			MPI_DOUBLE,
			partition,
			0);
701
702
703
	}
      }
    }    
704
   
705
    // blocking recieves
706
707
708
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (recvBufferSize[partition] > 0) {
709
710
711
712
713
	  mpiComm.Recv(recvBuffer[partition],
		       recvBufferSize[partition],
		       MPI_DOUBLE,
		       partition,
		       0);
714
715
716
717
718
	}
      }
    }    

    // wait for end of communication
719
    mpiComm.Barrier();
720
721

    // copy values into rank solutions
722
723
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
724
	std::vector<DegreeOfFreedom>::iterator dofIt = recvOrder[partition].begin();
725
726
	for (int i = 0; i < recvBufferSize[partition]; i++) {
	  (*(rankSolutions[partition]))[*dofIt] = recvBuffer[partition][i];
727
728
729
730
	  ++dofIt;
	}
      }
    }    
731

732
    // free send and recv buffers
733
734
    for (int partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
735
736
737
738
739
740
	if (sendBufferSize[partition] > 0) {
	  FREE_MEMORY(sendBuffer[partition], double, sendBufferSize[partition]);
	}
	if (recvBufferSize[partition] > 0) {
	  FREE_MEMORY(recvBuffer[partition], double, recvBufferSize[partition]);
	}
741
742
      }
    }    
743

744
745
746
747
748
749
750
    FREE_MEMORY(coarseDOFs, DegreeOfFreedom, numFcts);
    FREE_MEMORY(fineDOFs, DegreeOfFreedom, numFcts);
  }

  void ParallelProblem::exchangeDOFVector(AdaptInfo *adaptInfo,
					  DOFVector<double> *values)
  {
751
752
    partitioner->fillLeafPartitionVec(&oldPartitionVec, &oldPartitionVec);
    partitioner->fillLeafPartitionVec(&partitionVec, &partitionVec);
753
754
755
756

    // === get send and recieve orders ===
    std::vector<std::vector<DegreeOfFreedom> > sendOrder;
    std::vector<std::vector<DegreeOfFreedom> > recvOrder;
757
758
    sendOrder.resize(mpiSize);
    recvOrder.resize(mpiSize);
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774

    int i;
    const FiniteElemSpace *feSpace = values->getFESpace();
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *dofs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DOFAdmin *admin =  feSpace->getAdmin();

    Mesh *mesh = feSpace->getMesh();
    TraverseStack stack;
    ElInfo *elInfo;

    elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
    while(elInfo) {
      Element *element = elInfo->getElement();
      int index = element->getIndex();
775
776
      int oldPartition = oldPartitionVec[index];
      int newPartition = partitionVec[index];
777
778
779
780
781

      if(oldPartition != newPartition) {
	// get dof indices
	basFcts->getLocalIndices(element, admin, dofs);

782
	if(oldPartition == mpiRank) {
783
784
785
786
787
	  for(i = 0; i < numFcts; i++) {
	    // send element values to new partition
	    sendOrder[newPartition].push_back(dofs[i]);
	  }
	}
788
	if(newPartition == mpiRank) {
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
	  for(i = 0; i < numFcts; i++) {
	    // recv element values from old partition
	    recvOrder[oldPartition].push_back(dofs[i]);
	  }
	}
      }

      elInfo = stack.traverseNext(elInfo);
    }

    FREE_MEMORY(dofs, DegreeOfFreedom, numFcts);

    // === create send and recv buffers and fill send buffers ===
    std::map<int, double*> sendBuffer;
    std::map<int, double*> recvBuffer;
    std::map<int, int> sendBufferSize;
    std::map<int, int> recvBufferSize;

    int partition;
808
809
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
810
811
812
813
814
	int sendSize = static_cast<int>(sendOrder[partition].size());
	int recvSize = static_cast<int>(recvOrder[partition].size());
      
	sendBufferSize[partition] = sendSize;	
	recvBufferSize[partition] = recvSize;
815
	if (sendSize > 0) {
816
817
818
819
820
821
	  sendBuffer[partition] = GET_MEMORY(double, sendSize);
	  std::vector<DegreeOfFreedom>::iterator dofIt;
	  dofIt = sendOrder[partition].begin();
	  double *bufferIt, *bufferBegin, *bufferEnd;
	  bufferBegin = sendBuffer[partition];
	  bufferEnd = bufferBegin + sendSize;
822
823
824
	  for (bufferIt = bufferBegin; 
	       bufferIt < bufferEnd; 
	       ++bufferIt, ++dofIt) {
825
826
827
	    *bufferIt = (*values)[*dofIt];
	  }
	}
828
	if (recvSize > 0)
829
830
831
832
833
	  recvBuffer[partition] = GET_MEMORY(double, recvSize);
      }
    }

    // === non-blocking sends ===
834
835
836
837
838
839
840
841
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (sendBufferSize[partition] > 0) {
	  mpiComm.Isend(sendBuffer[partition],
			  sendBufferSize[partition],
			  MPI_DOUBLE,
			  partition,
			  0);
842
843
844
845
846
	}
      }
    }    
    
    // === blocking receives ===
847
848
849
850
851
852
853
854
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (recvBufferSize[partition] > 0) {
	  mpiComm.Recv(recvBuffer[partition],
			 recvBufferSize[partition],
			 MPI_DOUBLE,
			 partition,
			 0);
855
856
857
858
859
	}
      }
    }    

    // === wait for end of MPI communication ===
860
    mpiComm.Barrier();
861
862

    // === copy received values into DOFVector ===
863
864
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
865
	std::vector<DegreeOfFreedom>::iterator dofIt = recvOrder[partition].begin();
866
	for (i = 0; i < recvBufferSize[partition]; i++) {
867
868
869
870
871
872
873
	  (*values)[*dofIt] = recvBuffer[partition][i];
	  ++dofIt;
	}
      }
    }    

    // === free send and receive buffers ===
874
875
876
    for (partition = 0; partition < mpiSize; partition++) {
      if (partition != mpiRank) {
	if (sendBufferSize[partition] > 0)
877
878
879
	  FREE_MEMORY(sendBuffer[partition], 
		      double,
		      sendBufferSize[partition]);
880
	if (recvBufferSize[partition] > 0)
881
882
883
884
885
886
887
888
889
890
891
892
893
894
	  FREE_MEMORY(recvBuffer[partition], 
		      double,
		      recvBufferSize[partition]);
      }
    }
  }

  void ParallelProblem::buildGlobalSolution(AdaptInfo *adaptInfo,
					    std::vector<DOFVector<double>*> rankSolutions,
					    DOFVector<double> *globalSolution)
  {
    FUNCNAME("ParallelProblem::buildGlobalSolution()");

    const FiniteElemSpace *feSpace = globalSolution->getFESpace();
895
    int dim = mesh->getDim();
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
    const BasisFunction *basFcts = feSpace->getBasisFcts();
    int numFcts = basFcts->getNumber();
    DegreeOfFreedom *coarseDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DegreeOfFreedom *fineDOFs = GET_MEMORY(DegreeOfFreedom, numFcts);
    DOFAdmin *admin =  feSpace->getAdmin();

    Lagrange *linearFunctions = Lagrange::getLagrange(dim, 1);

    MSG("Building global solution\n");

    // compute w[DOF][partition]->value
    std::map<DegreeOfFreedom, std::map<int, double> > w;
    std::map<DegreeOfFreedom, std::map<int, double> >::iterator wIt, wBegin, wEnd;

    std::map<DegreeOfFreedom, double> sumW;

    Element *lastCoarseElement = NULL;

    WorldVector<double> worldCoord;
    DimVec<double> baryCoord(dim, NO_INIT);

    std::set<int>::iterator partIt, partBegin, partEnd;

    std::map<DegreeOfFreedom, bool> visited;


922
    MyDualTraverse dualTraverse(localCoarseGridLevel_);
923
924
925
    ElInfo *elInfo1, *elInfo2;
    ElInfo *large, *small;

926
927
928
929
930
931
932
933
934
    bool cont = dualTraverse.traverseFirst(mesh, mesh,
					   -1, -1,
					   Mesh::CALL_EVERY_EL_PREORDER |
					   Mesh::FILL_COORDS | 
					   Mesh::FILL_DET,
					   Mesh::CALL_LEAF_EL | 
					   Mesh::FILL_COORDS,
					   &elInfo1, &elInfo2,
					   &small, &large);
935
936
937
938
939
940
941

    while (cont) {
      Element *element1 = elInfo1->getElement();
      Element *element2 = elInfo2->getElement();
      PartitionElementData *partitionData = 
	dynamic_cast<PartitionElementData*>
	(element1->getElementData(PARTITION_ED));
942
      if (partitionData->getPartitionStatus() == IN) {
943
944

	// get coarse dofs
945
	if (element1 != lastCoarseElement) {
946
947
948
949
	  basFcts->getLocalIndices(element1, admin, coarseDOFs);
	  lastCoarseElement = element1;
	}
      
950
	if (elementPartitions_[element1].size() > 1) {
951
952
953
954
	  // get fine dofs
	  basFcts->getLocalIndices(element2, admin, fineDOFs);
      
	  // for all fine DOFs
955
956
	  for (int i = 0; i < numFcts; i++) {
	    if (!visited[fineDOFs[i]]) {
957
958
	      visited[fineDOFs[i]] = true;

959
	      elInfo2->coordToWorld(*(basFcts->getCoords(i)), worldCoord);
960
961
962
	      elInfo1->worldToCoord(worldCoord, &baryCoord);

	      // for all coarse vertex DOFs
963
	      for (int j = 0; j < dim + 1; j++) {
964
965
		partBegin = vertexPartitions[coarseDOFs[j]].begin();
		partEnd = vertexPartitions[coarseDOFs[j]].end();
966
		for (partIt = partBegin; partIt != partEnd; ++partIt) {
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
		  int partition = *partIt/* - 1*/;
		  double val = (*(linearFunctions->getPhi(j)))(baryCoord);
		  w[fineDOFs[i]][partition] += val;

		  sumW[fineDOFs[i]] += val;
		}
	      }
	    }
	  }
	}
      }
      cont = dualTraverse.traverseNext(&elInfo1, &elInfo2,
				       &small, &large);
    }

    FREE_MEMORY(coarseDOFs, DegreeOfFreedom, numFcts);
    FREE_MEMORY(fineDOFs, DegreeOfFreedom, numFcts);

    MSG("PU ...\n");

    wBegin = w.begin();
    wEnd = w.end();

990
    for (wIt = wBegin; wIt != wEnd; ++wIt) {
991
992
993
994
      DegreeOfFreedom dof = wIt->first;
      (*globalSolution)[dof] = 0.0;
    }
    
995
    for (wIt = wBegin; wIt != wEnd; ++wIt) {
996
997
998
999
1000
      DegreeOfFreedom dof = wIt->first;
      std::map<int, double>::iterator partIt, partBegin, partEnd;
      partBegin = wIt->second.begin();
      partEnd = wIt->second.end();