ParMetisPartitioner.cc 16.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include <queue>
14
15
#include "parallel/ParMetisPartitioner.h"
#include "parallel/MpiHelper.h"
16
#include "Serializer.h"
17
18
19
20
21
22
23
24
25
26
#include "Mesh.h"
#include "Traverse.h"
#include "ElInfo.h"
#include "Element.h"
#include "FixVec.h"
#include "DOFVector.h"
#include "mpi.h"

namespace AMDiS {

27
  ParMetisMesh::ParMetisMesh(Mesh *mesh, MPI::Intracomm *comm, 
28
			     std::map<int, bool>& elementInRank,
29
			     std::map<DegreeOfFreedom, DegreeOfFreedom> *mapLocalGlobal)
Thomas Witkowski's avatar
Thomas Witkowski committed
30
31
    : dim(mesh->getDim()),
      nElements(0),
32
      mpiComm(comm)
33
34
  {
    FUNCNAME("ParMetisMesh::ParMetisMesh()");
35
36

    int mpiSize = mpiComm->Get_size();
37
38
39
40
    int elementCounter = 0;
    int dow = Global::getGeo(WORLD);

    TraverseStack stack;
41
    ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
42
    while (elInfo) {
43
      if (elementInRank[elInfo->getElement()->getIndex()])
44
	elementCounter++;      
45
46
47
48

      elInfo = stack.traverseNext(elInfo);
    }

49
    nElements = elementCounter;
50

51
    TEST_EXIT(nElements > 0)("No elements in ParMETIS mesh!\n");
52
53

    // allocate memory
Thomas Witkowski's avatar
Thomas Witkowski committed
54
    eptr = new int[nElements + 1];
55
    eind = new int[nElements * (mesh->getGeo(VERTEX))];
Thomas Witkowski's avatar
Thomas Witkowski committed
56
57
    elmdist = new int[mpiSize + 1];
    elem_p2a = new int[nElements];
58

59
    if (dim == dow)
Thomas Witkowski's avatar
Thomas Witkowski committed
60
      xyz = new float[nElements * dim];
61
62
    else
      xyz = NULL;    
63

Thomas Witkowski's avatar
Thomas Witkowski committed
64
    eptr[0] = 0;
65

Thomas Witkowski's avatar
Thomas Witkowski committed
66
67
68
    int *ptr_eptr = eptr + 1;
    int *ptr_eind = eind;
    float *ptr_xyz = xyz;
69
70
    
    // gather element numbers and create elmdist
71
    mpiComm->Allgather(&nElements, 1, MPI_INT, elmdist + 1, 1, MPI_INT);
72

73
    elmdist[0] = 0;
74
    for (int i = 2; i < mpiSize + 1; i++)
75
      elmdist[i] += elmdist[i - 1];
76
77

    // traverse mesh and fill distributed ParMETIS data
78
    DimVec<double> bary(dim, DEFAULT_VALUE, 1.0 / mesh->getGeo(VERTEX));
79
80
81
    WorldVector<double> world;

    elementCounter = 0;
82
    int nodeCounter = 0;
83

84
    elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL | Mesh::FILL_COORDS);
85
    while (elInfo) {
86
87
88
89
      Element *element = elInfo->getElement();
      int index = element->getIndex();

      // if element in partition
90
      if (elementInRank[index]) {
91
92
93
94
95
	// remember index
	setParMetisIndex(index, elementCounter);
	setAMDiSIndex(elementCounter, index);

	// write eptr entry
96
	nodeCounter += mesh->getGeo(VERTEX);
97
98
99
100
	*ptr_eptr = nodeCounter;
	ptr_eptr++;

	// write eind entries (element nodes)
Thomas Witkowski's avatar
Thomas Witkowski committed
101
	for (int i = 0; i < dim + 1; i++) {
102
	  if (mapLocalGlobal)
103
	    *ptr_eind = (*mapLocalGlobal)[element->getDof(i, 0)];
104
105
	  else
	    *ptr_eind = element->getDof(i, 0);	  
106

107
108
109
110
	  ptr_eind++;
	}

	// write xyz element coordinates
111
	if (ptr_xyz) {
112
	  elInfo->coordToWorld(bary, world);
Thomas Witkowski's avatar
Thomas Witkowski committed
113
	  for (int i = 0; i < dim; i++) {
114
	    *ptr_xyz = static_cast<float>(world[i]);
115
116
117
118
119
120
	    ptr_xyz++;
	  }
	}

	elementCounter++;
      }
121

122
123
124
125
      elInfo = stack.traverseNext(elInfo);
    }
  }

126

127
128
  ParMetisMesh::~ParMetisMesh()
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
129
    if (eptr)
Thomas Witkowski's avatar
Thomas Witkowski committed
130
      delete [] eptr;
131
    
Thomas Witkowski's avatar
Thomas Witkowski committed
132
    if (eind)     
Thomas Witkowski's avatar
Thomas Witkowski committed
133
      delete [] eind;
134
    
135
    if (elmdist)
Thomas Witkowski's avatar
Thomas Witkowski committed
136
      delete [] elmdist;
137
    
Thomas Witkowski's avatar
Thomas Witkowski committed
138
    if (xyz)
Thomas Witkowski's avatar
Thomas Witkowski committed
139
      delete [] xyz;
140
    
Thomas Witkowski's avatar
Thomas Witkowski committed
141
    if (elem_p2a) 
Thomas Witkowski's avatar
Thomas Witkowski committed
142
      delete [] elem_p2a;
143
144
  }

145

146
  ParMetisGraph::ParMetisGraph(ParMetisMesh *parMesh,
147
			       MPI::Intracomm *comm,
148
			       int ncommonnodes)
149
    : parMetisMesh(parMesh)
150
  {
151
152
153
154
155
    FUNCNAME("ParMetisGraph::ParMetisGraph()");

    TEST_EXIT(parMesh)("No ParMetisMesh defined!\n");
    TEST_EXIT(comm)("No MPI communicator defined!\n");

156
157
    int numflag = 0;

158
159
    if (ncommonnodes == -1) 
      ncommonnodes = parMetisMesh->getDim();
160

161
    MPI_Comm tmpComm = MPI_Comm(*comm);
162

163
164
165
    ParMETIS_V3_Mesh2Dual(parMetisMesh->getElementDist(),
			  parMetisMesh->getElementPtr(),
			  parMetisMesh->getElementInd(),
166
167
			  &numflag,
			  &ncommonnodes,
Thomas Witkowski's avatar
Thomas Witkowski committed
168
169
			  &xadj,
			  &adjncy,
170
			  &tmpComm);
171
172
  }

173

174
175
  ParMetisGraph::~ParMetisGraph()
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
176
177
    free(xadj);
    free(adjncy);
178
179
  }

180

181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
  void ParMetisGraph::print()
  {
    FUNCNAME("ParMetisGraph::print()");

    std::stringstream oss;
    for (int i = 0; i <= MPI::COMM_WORLD.Get_size(); i++)
      oss << parMetisMesh->getElementDist()[i] << " ";
    
    MSG("Element dist = %s\n", oss.str().c_str());

    int mpiRank = MPI::COMM_WORLD.Get_rank();
    int nElements = parMetisMesh->getElementDist()[mpiRank + 1] - 
      parMetisMesh->getElementDist()[mpiRank];

    MSG("nElements = %d   in index range %d - %d\n", 
	nElements, 
	parMetisMesh->getElementDist()[mpiRank],
	parMetisMesh->getElementDist()[mpiRank + 1]);

    oss.str("");
    oss.clear();
    
    for (int i = 0; i <= nElements; i++)
      oss << xadj[i] << ", ";

    MSG("xadj = {%s}\n", oss.str().c_str());

    oss.str("");
    oss.clear();

    for (int i = 0; i <= xadj[nElements] - 1; i++) 
      oss << adjncy[i] << ", ";

    MSG("adjncy = {%s}\n", oss.str().c_str());
  }


  ParMetisPartitioner::~ParMetisPartitioner()
  {
    if (parMetisMesh) 
      delete parMetisMesh;
  }


225
  void ParMetisPartitioner::createInitialPartitioning() 
226
  {
227
    FUNCNAME("ParMetrisPartitioner::createInitialPartitioning()");
228

229
230
    int mpiRank = mpiComm->Get_rank();
    int mpiSize = mpiComm->Get_size();
231
232
    int nLeaves = mesh->getNumberOfLeaves();
    int elPerRank = nLeaves / mpiSize;
233

234
    // === Create initial partitioning of the AMDiS mesh. ===
235

236
237
    elementInRank.clear();

238
    TraverseStack stack;
239
    ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
240
    while (elInfo) {
241
242
      Element *element = elInfo->getElement();

243
244
245
246
      if ((element->getIndex() >= mpiRank * elPerRank &&
	   element->getIndex() < (mpiRank + 1) * elPerRank) ||
	  (element->getIndex() >= mpiSize * elPerRank &&
	   mpiRank == mpiSize - 1))
247
	elementInRank[element->getIndex()] = true;
248
      else
249
	elementInRank[element->getIndex()] = false;
250
      
251
252
253
254
      elInfo = stack.traverseNext(elInfo);
    }
  }

255

256
  bool ParMetisPartitioner::partition(std::map<int, double> &elemWeights,
257
258
259
				      PartitionMode mode,
				      float itr) 
  {
260
    FUNCNAME("ParMetisPartitioner::partition()");
261

262
    int mpiSize = mpiComm->Get_size();
263

264
265
266

    // === Create parmetis mesh ===

267
    if (parMetisMesh) 
Thomas Witkowski's avatar
Thomas Witkowski committed
268
      delete parMetisMesh;
269

270
271
    TEST_EXIT_DBG(elementInRank.size() != 0)("Should not happen!\n");

272
    parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal);
273

274
    int nElements = parMetisMesh->getNumElements();
275

276
277
278

    // === Create weight array ===

279
280
281
    std::vector<int> wgts(nElements);
    std::vector<float> floatWgts(nElements);
    unsigned int floatWgtsPos = 0;
282
283
    float maxWgt = 0.0;

284
    TraverseStack stack;
285
    ElInfo *elInfo = stack.traverseFirst(mesh, 0, Mesh::CALL_EL_LEVEL);
286
    while (elInfo) {
287
      int index = elInfo->getElement()->getIndex();
288

289
      if (elementInRank[index]) {
290
	// get weight 
291
	float wgt = static_cast<float>(elemWeights[index]);
292
	maxWgt = std::max(wgt, maxWgt);
293
294

	// write float weight
295
296
	TEST_EXIT_DBG(floatWgtsPos < floatWgts.size())("Should not happen!\n");
	floatWgts[floatWgtsPos++] = wgt;
297
298
299
300
      }
      elInfo = stack.traverseNext(elInfo);
    }

301
302
    TEST_EXIT_DBG(floatWgtsPos == floatWgts.size())("Should not happen!\n");

303
    float tmp;
304
    mpiComm->Allreduce(&maxWgt, &tmp, 1, MPI_FLOAT, MPI_MAX);
305
306
    maxWgt = tmp;

307
308
309

    // === Create dual graph ===

310
    ParMetisGraph parMetisGraph(parMetisMesh, mpiComm);
311

312
313
314

    // === Partitioning of dual graph ===

315
    int wgtflag = 2; // weights at vertices only!
316
    int numflag = 0; // c numbering style!
317
    int ncon = 1; // one weight at each vertex!
318
    int nparts = mpiSize; // number of partitions
319

320
    std::vector<float> tpwgts(mpiSize);
321
    float ubvec = 1.05;
322
    int options[4] = {0, 0, 15, 1}; // default options
323
    int edgecut = -1;
324
    std::vector<int> part(nElements);
325

326
327
328
    // set tpwgts
    for (int i = 0; i < mpiSize; i++)
      tpwgts[i] = 1.0 / nparts;
329
   
330
    float scale = 10000.0 / maxWgt;
331
332
333
334


    // === Scale element weights. ===

335
336
337
338
339
    int smin = 9999999;
    int smax = 0;
    int ssum = 0;

    for (int i = 0; i < nElements; i++) {
340
      wgts[i] = static_cast<int>(floatWgts[i] * scale);
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
      smin = std::min(smin, wgts[i]);
      smax = std::max(smax, wgts[i]);
      ssum += wgts[i];
    }

    mpi::globalMin(smin);
    mpi::globalMax(smax);
    mpi::globalAdd(ssum);

    MSG("DATA   SMIN = %d   SMAX = %d    SSUM = %d\n", smin, smax, ssum);

    int kpart = ssum / mpiSize;
    int kpartMax = 0;
    for (int i = 0; i < nElements; i++)
      if (wgts[i] < kpart)
356
	kpartMax = std::max(kpartMax, wgts[i]);
357
358
359
360
361
362
363
364
365
366

    mpi::globalMax(kpartMax);

    MSG("KPART MAX: %d\n", kpartMax);

    smin = 9999999;
    smax = 0;
    ssum = 0;

    for (int i = 0; i < nElements; i++) {
367
368
      wgts[i] = std::min(wgts[i], kpartMax);
      wgts[i] = std::max(wgts[i], 1);
369

370
371
372
373
374
375
376
377
378
379
      smin = std::min(smin, wgts[i]);
      smax = std::max(smax, wgts[i]);
      ssum += wgts[i];
    }

    mpi::globalMin(smin);
    mpi::globalMax(smax);
    mpi::globalAdd(ssum);

    MSG("DATA   SMIN = %d   SMAX = %d    SSUM = %d\n", smin, smax, ssum);
380
381
382


    // === Start ParMETIS. ===
383

384
385
    MPI_Comm tmpComm = MPI_Comm(*mpiComm);

386
    switch (mode) {
387
    case INITIAL:
388
      ParMETIS_V3_PartKway(parMetisMesh->getElementDist(),
389
390
			   parMetisGraph.getXAdj(),
			   parMetisGraph.getAdjncy(),
391
			   &(wgts[0]),
392
			   NULL,
393
			   &wgtflag,
394
395
396
			   &numflag,
			   &ncon,
			   &nparts,
397
			   &(tpwgts[0]),
398
399
400
			   &ubvec,
			   options,
			   &edgecut,
401
			   &(part[0]),
402
			   &tmpComm);
403
404
405
      break;
    case ADAPTIVE_REPART:
      {
406
407
	//	parMetisGraph.print();

408
	std::vector<int> vsize(nElements);
Thomas Witkowski's avatar
Thomas Witkowski committed
409
	for (int i = 0; i < nElements; i++)
410
411
	  vsize[i] = static_cast<int>(floatWgts[i]);

412
	ParMETIS_V3_AdaptiveRepart(parMetisMesh->getElementDist(),
413
414
				   parMetisGraph.getXAdj(),
				   parMetisGraph.getAdjncy(),
415
				   &(wgts[0]),
416
				   NULL,
417
				   &(vsize[0]),
418
419
420
421
				   &wgtflag,
				   &numflag,
				   &ncon,
				   &nparts,
422
				   &(tpwgts[0]),
423
424
425
426
				   &ubvec,
				   &itr,
				   options,
				   &edgecut,
427
				   &(part[0]),
428
				   &tmpComm);
429
430
431
      }
      break;
    case REFINE_PART:
432
      ParMETIS_V3_RefineKway(parMetisMesh->getElementDist(),
433
434
			     parMetisGraph.getXAdj(),
			     parMetisGraph.getAdjncy(),
435
			     &(wgts[0]),
436
437
438
439
440
			     NULL,
			     &wgtflag,
			     &numflag,
			     &ncon,
			     &nparts,
441
			     &(tpwgts[0]),
442
443
444
			     &ubvec,
			     options,
			     &edgecut,
445
			     &(part[0]),
446
			     &tmpComm);
447

448
449
450
451
452
      break;
    default: 
      ERROR_EXIT("unknown partitioning mode\n");
    }

453
454
455

    // === Distribute new partition data. ===

456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
    bool b = distributePartitioning(&(part[0]));

    if (!b) {
      MSG("RETRY ParMETIS!\n");

      std::vector<float> testub(14);
      
      testub[0] = 1.001;
      testub[1] = 1.01;
      testub[2] = 1.02;
      testub[3] = 1.03;
      testub[4] = 1.04;
      testub[5] = 1.06;
      testub[6] = 1.07;
      testub[7] = 1.08;
      testub[8] = 1.09;
      testub[9] = 1.1;
      testub[10] = 1.25;
      testub[11] = 1.5;
      testub[12] = 2.0;
      testub[13] = 2.5;

      for (int jj = 0; jj < testub.size(); jj++) {
      ubvec = testub[jj];

      std::vector<int> vsize(nElements);
      for (int i = 0; i < nElements; i++)
	vsize[i] = static_cast<int>(floatWgts[i]);


      ParMETIS_V3_AdaptiveRepart(parMetisMesh->getElementDist(),
				 parMetisGraph.getXAdj(),
				 parMetisGraph.getAdjncy(),
				 &(wgts[0]),
				 NULL,
				 &(vsize[0]),
				 &wgtflag,
				 &numflag,
				 &ncon,
				 &nparts,
				 &(tpwgts[0]),
				 &ubvec,
				 &itr,
				 options,
				 &edgecut,
				 &(part[0]),
				 &tmpComm);


      b = distributePartitioning(&(part[0]));

      MSG("ParMETIS RETRY with %f: %d\n", ubvec, b);
      }
    }

    return b;
512
513
  }

514

515
516
  void ParMetisPartitioner::fillCoarsePartitionVec(std::map<int, int> *partitionVec)
  {
517
518
    FUNCNAME("ParMetisPartitioner::fillCoarsePartitionVec()");

519
    TEST_EXIT_DBG(partitionVec)("No partition vector!\n");
520
521
522
523

    partitionVec->clear();

    // update ParMETIS mesh to new partitioning
524
    if (!parMetisMesh)
525
      parMetisMesh = new ParMetisMesh(mesh, mpiComm, elementInRank, mapLocalGlobal);
526

527
528
    int mpiRank = mpiComm->Get_rank();
    int mpiSize = mpiComm->Get_size();
529
    std::vector<int> nPartitionElements(mpiSize);
530
    int *elmdist = parMetisMesh->getElementDist();
Thomas Witkowski's avatar
Thomas Witkowski committed
531

532
    for (int i = 0; i < mpiSize; i++)
533
      nPartitionElements[i] = elmdist[i + 1] - elmdist[i];
534
535

    // === count number of elements ===
536
537
538
    int nElements = 0;
    int localElements = parMetisMesh->getNumElements();
    mpiComm->Allreduce(&localElements, &nElements, 1, MPI_INT, MPI_SUM);
539

540
    std::vector<int> partitionElements(nElements);
541
542

    // distribute partition elements
543
544
    mpiComm->Allgatherv(parMetisMesh->getAMDiSIndices(),
			nPartitionElements[mpiRank], 
545
			MPI_INT, 
546
547
			&(partitionElements[0]), 
			&(nPartitionElements[0]), 
548
549
			elmdist, 
			MPI_INT);
550
551

    // fill partitionVec
Thomas Witkowski's avatar
Thomas Witkowski committed
552
553
    for (int i = 0; i < mpiSize; i++)
      for (int j = 0; j < nPartitionElements[i]; j++)
554
555
556
	(*partitionVec)[partitionElements[elmdist[i] + j]] = i;
  }

557

558
  bool ParMetisPartitioner::distributePartitioning(int *part) 
559
  {
560
561
    FUNCNAME("ParMetisPartitioner::distributePartitioning()");

562
563
    int mpiSize = mpiComm->Get_size();
    int mpiRank = mpiComm->Get_rank();
564
    int nElements = parMetisMesh->getNumElements();
565

566
    // nPartitionElements[i] is the number of elements for the i-th partition
Thomas Witkowski's avatar
Thomas Witkowski committed
567
    int *nPartitionElements = new int[mpiSize];
568
    for (int i = 0; i < mpiSize; i++) 
569
      nPartitionElements[i] = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
570
    for (int i = 0; i < nElements; i++)
571
      nPartitionElements[part[i]]++;    
572
573

    // collect number of partition elements from all ranks for this rank
Thomas Witkowski's avatar
Thomas Witkowski committed
574
    int *nRankElements = new int[mpiSize];
575
    mpiComm->Alltoall(nPartitionElements, 1, MPI_INT, nRankElements, 1, MPI_INT);
576

577

578
    // sum up partition elements over all ranks
Thomas Witkowski's avatar
Thomas Witkowski committed
579
    int *sumPartitionElements = new int[mpiSize];
580
    mpiComm->Allreduce(nPartitionElements, sumPartitionElements, mpiSize,
581
		       MPI_INT, MPI_SUM);
582
583
584
585
586
587
588
589
590

    // Test if there exists an empty partition
    bool emptyPartition = false;
    for (int i = 0; i < mpiSize; i++) 
      emptyPartition |= (sumPartitionElements[i] == 0);

    if (emptyPartition)
      return false;    

591
    // prepare distribution (fill partitionElements with AMDiS indices)
Thomas Witkowski's avatar
Thomas Witkowski committed
592
    int *bufferOffset = new int[mpiSize];
593
    bufferOffset[0] = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
594
    for (int i = 1; i < mpiSize; i++)
595
      bufferOffset[i] = bufferOffset[i - 1] + nPartitionElements[i - 1];
596

Thomas Witkowski's avatar
Thomas Witkowski committed
597
598
    int *partitionElements = new int[nElements];
    int **partitionPtr = new int*[mpiSize];
599

600
    for (int i = 0; i < mpiSize; i++)
601
602
      partitionPtr[i] = partitionElements + bufferOffset[i];

603
    sendElements.clear();
604
    for (int i = 0; i < nElements; i++) {
605
      int partition = part[i];
606
      int amdisIndex = parMetisMesh->getAMDiSIndex(i);
607

608
609
610
      if (partition != mpiRank)
	sendElements[partition].push_back(amdisIndex);

611
612
613
614
615
      *(partitionPtr[partition]) = amdisIndex;
      ++(partitionPtr[partition]);
    }

    // all to all: partition elements to rank elements
Thomas Witkowski's avatar
Thomas Witkowski committed
616
617
    int *rankElements = new int[sumPartitionElements[mpiRank]];
    int *recvBufferOffset = new int[mpiSize];
618
    recvBufferOffset[0] = 0;
Thomas Witkowski's avatar
Thomas Witkowski committed
619
    for (int i = 1; i < mpiSize; i++)
620
      recvBufferOffset[i] = recvBufferOffset[i - 1] + nRankElements[i - 1];
621

622
    mpiComm->Alltoallv(partitionElements, 
623
		       nPartitionElements,
624
625
626
		       bufferOffset,
		       MPI_INT,
		       rankElements,
627
		       nRankElements,
628
629
		       recvBufferOffset,
		       MPI_INT);
630
    
631
632
633
634
    TEST_EXIT(elementInRank.size() != 0)("Should not happen!\n");
    for (std::map<int, bool>::iterator it = elementInRank.begin();
	 it != elementInRank.end(); ++it)
      elementInRank[it->first] = false;
635

636
637
    // Create map which stores for each element index on ther partitioning level
    // if the element is in the partition of this rank.
638
    recvElements.clear();
639
    for (int i = 0; i < mpiSize; i++) {
640
      int *rankStart = rankElements + recvBufferOffset[i];
641
      int *rankEnd = rankStart + nRankElements[i];
642
643
      for (int *rankPtr = rankStart; rankPtr < rankEnd; ++rankPtr) {
	elementInRank[*rankPtr] = true;
644
645
	if (i != mpiRank)
	  recvElements[i].push_back(*rankPtr);
646
647
648
      }
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
649
    delete parMetisMesh;
650
    parMetisMesh = NULL;
651

Thomas Witkowski's avatar
Thomas Witkowski committed
652
653
654
655
656
657
658
    delete [] rankElements;
    delete [] nPartitionElements;
    delete [] nRankElements;
    delete [] sumPartitionElements;
    delete [] partitionElements;
    delete [] partitionPtr;
    delete [] bufferOffset;
Thomas Witkowski's avatar
huh    
Thomas Witkowski committed
659
    delete [] recvBufferOffset;
660
661

    return true;
662
663
  }

664
665
666
667
668
669
670
671
672
673
674
  
  void ParMetisPartitioner::serialize(std::ostream &out)
  {
    SerUtil::serialize(out, elementInRank);
  }


  void ParMetisPartitioner::deserialize(std::istream &in)
  {
    SerUtil::deserialize(in, elementInRank);
  }
675
}