StdMpi.h 18 KB
Newer Older
1
2
3
4
5
6
7
/******************************************************************************
 *
 * AMDiS - Adaptive multidimensional simulations
 *
 * Copyright (C) 2013 Dresden University of Technology. All Rights Reserved.
 * Web: https://fusionforge.zih.tu-dresden.de/projects/amdis
 *
8
 * Authors:
9
10
11
12
13
14
15
16
17
 * Simon Vey, Thomas Witkowski, Andreas Naumann, Simon Praetorius, et al.
 *
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *
 * This file is part of AMDiS
 *
 * See also license.opensource.txt in the distribution.
18
 *
19
 ******************************************************************************/
20
21


22
23
24
25
26
27
28

/** \file StdMpi.h */

#ifndef AMDIS_STDMPI_H
#define AMDIS_STDMPI_H

#include <map>
29
#include <stdint.h>
30
#include <mpi.h>
31
#include "MeshStructure.h"
32
#include "parallel/InteriorBoundary.h"
33

34
namespace AMDiS { namespace Parallel {
35

36
  /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
37
38
   * The class StdMpiHelper defines for a type a set of variables, types and
   * functions  that makes it able to transfer objects of the original type to
39
   * a buffer which is suitable to be send and received using MPI
Thomas Witkowski's avatar
Thomas Witkowski committed
40
   * communication.
41
   *
42
   *    mpiDataType     Specifies the MPI data type that should be used for
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
   *                    communication.
   *    cppDataType     Specifies the C++ data type of the buffer.
   *    getBufferSize   This functions calculates the size of the buffer for a
   *                    specific object.
   *    createBuffer    Create the buffer for a specific object.
   *    makeFromBuffer  Create an object of a buffer.
   */
  template<typename T>
  struct StdMpiHelper {};

  template<>
  struct StdMpiHelper<int> {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(int &data);

    static void createBuffer(int &data, int *buf);
62

63
64
65
66
67
    static void makeFromBuffer(int &data, int *buf, int bufSize);
  };


  template<>
68
  struct StdMpiHelper<std::vector<int> > {
69
    static MPI_Datatype mpiDataType;
70

71
    typedef int cppDataType;
72

73
    static int getBufferSize(std::vector<int> &data);
74

75
    static int getBufferSize(std::vector<const int*> &data);
76

77
    static void createBuffer(std::vector<int> &data, int *buf);
78

79
    static void makeFromBuffer(std::vector<int> &data, int *buf, int bufSize);
80
81
  };

82

83
84
85
86
87
88
89
  template<>
  struct StdMpiHelper<std::set<int> > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(std::set<int> &data);
90

91
    static void createBuffer(std::set<int> &data, int *buf);
92

93
    static void makeFromBuffer(std::set<int> &data, int *buf, int bufSize);
94
  };
95

96

97
  template<>
98
  struct StdMpiHelper<std::vector<std::set<int> > > {
99
100
101
102
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

103
    static int getBufferSize(std::vector<std::set<int> > &data);
104

105
    static void createBuffer(std::vector<std::set<int> > &data, int *buf);
106
107

    static void makeFromBuffer(std::vector<std::set<int> > &data,
108
109
			       int *buf, int bufSize);
  };
110

111
112
113
  template<>
  struct StdMpiHelper<std::set<std::pair<int, int> > > {
    static MPI_Datatype mpiDataType;
114

115
    typedef int cppDataType;
116

117
    static int getBufferSize(std::set<std::pair<int, int> > &data);
118

119
    static void createBuffer(std::set<std::pair<int, int> > &data, int *buf);
120

121
122
123
    static void makeFromBuffer(std::set<std::pair<int, int> > &data,
			       int *buf, int bufSize);
  };
124

125

126
  template<>
127
  struct StdMpiHelper<std::vector<double> > {
128
129
130
131
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

132
    static int getBufferSize(std::vector<double> &data);
133

134
    static void createBuffer(std::vector<double> &data, double *buf);
135

136
    static void makeFromBuffer(std::vector<double> &data, double *buf, int bufSize);
137
  };
138

Thomas Witkowski's avatar
Thomas Witkowski committed
139

140
  template<>
141
  struct StdMpiHelper<std::vector<std::vector<double> > > {
142
143
144
145
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

146
    static int getBufferSize(std::vector<std::vector<double> > &data);
147

148
    static void createBuffer(std::vector<std::vector<double> > &data, double *buf);
149

150
    static void makeFromBuffer(std::vector<std::vector<double> > &data, double *buf, int bufSize);
151
152
  };

153

154
155
  // MeshDistributor::MeshCodeVec
  template<>
156
  struct StdMpiHelper<std::vector<MeshStructure> > {
157
158
159
160
    static MPI_Datatype mpiDataType;

    typedef uint64_t cppDataType;

161
    static int getBufferSize(std::vector<MeshStructure> &data);
162

163
    static void createBuffer(std::vector<MeshStructure> &data, uint64_t *buf);
164
165

    static void makeFromBuffer(std::vector<MeshStructure> &data,
166
167
			       uint64_t *buf, int bufSize);
  };
168

169

170
  template<>
171
  struct StdMpiHelper<std::vector<AtomicBoundary> > {
172
    static MPI_Datatype mpiDataType;
173

174
    typedef int cppDataType;
175

176
    static int getBufferSize(std::vector<AtomicBoundary> &data);
177

178
    static void createBuffer(std::vector<AtomicBoundary> &data, int *buf);
179

180
    static void makeFromBuffer(std::vector<AtomicBoundary> &data, int *buf, int bufSize);
181
  };
182
183


184
  // PeriodicMap::PeriodicDofMap
185
  template<>
186
  struct StdMpiHelper<std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > > {
187
    static MPI_Datatype mpiDataType;
188

189
    typedef int cppDataType;
190

191
    static int getBufferSize(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data);
192

193
    static void createBuffer(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf);
194

195
    static void makeFromBuffer(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize);
196
  };
197

198

199
200
  // PetscSolver::createPetscNnzStructure::MatrixNnzEntry
  template<>
201
  struct StdMpiHelper<std::vector<std::pair<int, int> > > {
202
    static MPI_Datatype mpiDataType;
203

204
    typedef int cppDataType;
205

206
    static int getBufferSize(std::vector<std::pair<int, int> > &data);
207

208
    static void createBuffer(std::vector<std::pair<int, int> > &data, int *buf);
209

210
    static void makeFromBuffer(std::vector<std::pair<int, int> > &data,
211
212
			       int *buf, int bufSize);
  };
213
214


215
216
  // ParallelDebug::CoordsVec
  template<>
217
  struct StdMpiHelper<std::vector<WorldVector<double> > > {
218
    static MPI_Datatype mpiDataType;
Thomas Witkowski's avatar
Thomas Witkowski committed
219

220
    typedef double cppDataType;
221

222
    static int getBufferSize(std::vector<WorldVector<double> > &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
223

224
    static void createBuffer(std::vector<WorldVector<double> > &data, double *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
225

226
    static void makeFromBuffer(std::vector<WorldVector<double> > &data, double *buf, int bufSize);
227
  };
228

Praetorius, Simon's avatar
Praetorius, Simon committed
229
230
231
232
233
  template<>
  struct StdMpiHelper<std::vector<WorldVector<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;
234

Praetorius, Simon's avatar
Praetorius, Simon committed
235
236
237
238
239
240
    static int getBufferSize(std::vector<WorldVector<int> > &data);

    static void createBuffer(std::vector<WorldVector<int> > &data, int *buf);

    static void makeFromBuffer(std::vector<WorldVector<int> > &data, int *buf, int bufSize);
  };
241

Praetorius, Simon's avatar
Praetorius, Simon committed
242
243
244
245
246
  template<>
  struct StdMpiHelper<std::vector<Vector<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;
247

Praetorius, Simon's avatar
Praetorius, Simon committed
248
249
250
251
252
253
    static int getBufferSize(std::vector<Vector<int> > &data);

    static void createBuffer(std::vector<Vector<int> > &data, int *buf);

    static void makeFromBuffer(std::vector<Vector<int> > &data, int *buf, int bufSize);
  };
254

255
256
257
258
259
  template<>
  struct StdMpiHelper<std::vector<WorldMatrix<double> > > {
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;
260

261
262
263
264
265
266
    static int getBufferSize(std::vector<WorldMatrix<double> > &data);

    static void createBuffer(std::vector<WorldMatrix<double> > &data, double *buf);

    static void makeFromBuffer(std::vector<WorldMatrix<double> > &data, double *buf, int bufSize);
  };
267

Praetorius, Simon's avatar
Praetorius, Simon committed
268
269
270
271
272
  template<>
  struct StdMpiHelper<std::vector<WorldMatrix<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;
273

Praetorius, Simon's avatar
Praetorius, Simon committed
274
275
276
277
278
279
    static int getBufferSize(std::vector<WorldMatrix<int> > &data);

    static void createBuffer(std::vector<WorldMatrix<int> > &data, int *buf);

    static void makeFromBuffer(std::vector<WorldMatrix<int> > &data, int *buf, int bufSize);
  };
280

Praetorius, Simon's avatar
Praetorius, Simon committed
281
282
283
284
285
  template<>
  struct StdMpiHelper<std::vector<WorldVector<WorldVector<double> > > > {
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;
286

Praetorius, Simon's avatar
Praetorius, Simon committed
287
288
289
290
291
292
    static int getBufferSize(std::vector<WorldVector<WorldVector<double> > > &data);

    static void createBuffer(std::vector<WorldVector<WorldVector<double> > > &data, double *buf);

    static void makeFromBuffer(std::vector<WorldVector<WorldVector<double> > > &data, double *buf, int bufSize);
  };
293

Praetorius, Simon's avatar
Praetorius, Simon committed
294
295
296
297
298
  template<>
  struct StdMpiHelper<std::vector<Matrix<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;
299

Praetorius, Simon's avatar
Praetorius, Simon committed
300
301
302
303
304
305
    static int getBufferSize(std::vector<Matrix<int> > &data);

    static void createBuffer(std::vector<Matrix<int> > &data, int *buf);

    static void makeFromBuffer(std::vector<Matrix<int> > &data, int *buf, int bufSize);
  };
Thomas Witkowski's avatar
Thomas Witkowski committed
306
307


308
309
  // ParallelDebug::testGlobalIndexByCoords::CoordsIndexMap
  template<>
310
  struct StdMpiHelper<std::map<WorldVector<double>, int> > {
311
    static MPI_Datatype mpiDataType;
312

313
    typedef double cppDataType;
314

315
    static int getBufferSize(std::map<WorldVector<double>, int> &data);
316

317
    static void createBuffer(std::map<WorldVector<double>, int> &data, double* buf);
318

319
    static void makeFromBuffer(std::map<WorldVector<double>, int> &data, double* buf, int bufSize);
320
  };
321
  template<>
322
  struct StdMpiHelper<std::map<int, WorldVector<double> > > {
323
324
325
326
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

327
    static int getBufferSize(std::map<int, WorldVector<double> > &data);
328

329
    static void createBuffer(std::map<int, WorldVector<double> > &data, double* buf);
330

331
    static void makeFromBuffer(std::map<int, WorldVector<double> > &data, double* buf, int bufSize);
332
  };
333

334

335

336
  template<>
337
  struct StdMpiHelper<std::vector<std::vector<WorldVector<double> > > > {
338
339
340
341
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

342
    typedef std::vector<std::vector<WorldVector<double> > >  DataType;
343
344
345
346
347
348
349
350
351

    static int getBufferSize(DataType &data);

    static void createBuffer(DataType &data, double* buf);

    static void makeFromBuffer(DataType &data, double* buf, int bufSize);
  };


352
353
354
  /** \brief
   * This class is used to easily send and receive STL containers using MPI.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
355
  template<typename SendT, typename RecvT=SendT>
356
  class StdMpi
357
358
  {
  public:
359
360
361
362
363
364
365
366
367
368
    /** \brief
     * Constructor of the class.
     *
     * \param[in]  comm   Reference to the MPI communicator that should be used.
     * \param[in]  b      If true, the size of the data that will be communicated
     *                    between the ranks will be communicated before to achieve
     *                    correct communication. If the value is false, the data size
     *                    must be set by hand and is assumed to be correct on all
     *                    sending and receiving ranks.
     */
369
    StdMpi(MPI::Intracomm &comm, bool b = true) :
370
      mpiComm(comm),
371
372
      commPrepared(true),
      exchangeDataSize(b)
373
374
    {}

375
    /// Reset all variables.
376
    void clear()
377
378
379
    {
      sendData.clear();
      recvData.clear();
380
381
382
      sendDataSize.clear();
      recvDataSize.clear();

383
384
385
      commPrepared = true;
    }

386
387
388
389
390
391
392

    /** \brief
     * Send data to rank.
     *
     * \param[in]  toRank   Rank number to which the data is send to.
     * \param[in]  data     Data to be send.
     */
393
394
395
    void send(int toRank, SendT& data)
    {
      FUNCNAME("StdMpi::send()");
396

397
398
399
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      sendData[toRank] = data;
400
      sendDataSize[toRank] = StdMpiHelper<SendT>::getBufferSize(data);
401
402
    }

403
404
405
406
407
408
409

    /** \brief
     * Send data to rank.
     *
     * \param[in]  data    Maps rank number to the data that should be send to
     *                     these ranks.
     */
410
    void send(std::map<int, SendT>& data)
411
412
    {
      FUNCNAME("StdMpi::send()");
413

414
415
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

416
      for (typename std::map<int, SendT>::iterator it = data.begin();
417
418
	   it != data.end(); ++it) {
	sendData[it->first] = it->second;
419
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
420
421
422
      }
    }

423
424

    /// Returns sending data, see \ref sendData.
425
    inline std::map<int, SendT>& getSendData()
426
427
428
429
    {
      return sendData;
    }

430
431

    /// Returns the data that should be send to a specific rank, see \ref sendData.
432
    inline SendT& getSendData(int rank)
433
434
435
436
    {
      return sendData[rank];
    }

437
438

    /// Updates the buffer sizes for all sending data.
439
440
    void updateSendDataSize()
    {
441
      for (typename std::map<int, SendT>::iterator it = sendData.begin();
442
	   it != sendData.end(); ++it)
443
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
444
445
    }

446
447
448

    /** \brief
     * Is used to specify from which rank data should be received.
449
     *
450
     * \param[in]  fromRank   Rank number.
451
     * \param[in]  size
452
     */
453
454
455
    void recv(int fromRank, int size = -1)
    {
      FUNCNAME("StdMpi::recv()");
456

457
458
459
460
461
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      recvDataSize[fromRank] = size;
    }

462
463

    /// Is used to specify that data is received from all ranks.
464
465
466
    void recvFromAll(int size = -1)
    {
      FUNCNAME("StdMpi::recvFromAll()");
467

468
469
470
471
472
473
      TEST_EXIT_DBG(commPrepared)("Communication is no prepared!\n");

      for (int i = 0; i < mpiComm.Get_size(); i++)
	recvDataSize[i] = size;
    }

474
475

    /// Is used to specify from which rank data will be received.
476
    template<class T>
477
    void recv(std::map<int, T> &fromRanks)
478
479
    {
      FUNCNAME("StdMpi::recv()");
480

481
482
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

483
      for (typename std::map<int, T>::iterator it = fromRanks.begin();
484
	   it != fromRanks.end(); ++it)
485
	recvDataSize[it->first] =
486
	  (exchangeDataSize ? -1 : StdMpiHelper<SendT>::getBufferSize(it->second));
487
488
    }

489
490

    /// Returns received data, see \ref recvData.
491
    std::map<int, RecvT>& getRecvData()
492
493
494
495
    {
      return recvData;
    }

496
497

    /// Returns received data from a specific rank, see \ref recvData.
498
499
    RecvT& getRecvData(int rank)
    {
500
501
      FUNCNAME("StdMpi::getRecvData()");
      TEST_EXIT_DBG(recvData.count(rank))("No recv data from rank %d\n", rank);
502
503
504
      return recvData[rank];
    }

505
506
507

    /// If data sizes should be exchanged before the data itself is communicated,
    /// this will be done in this function.
508
509
    void commDataSize()
    {
510
511
      FUNCNAME("StdMpi::commDataSize()");

512
      MPI::Request* request= new MPI::Request[sendData.size() + recvDataSize.size()];
513
      std::vector<int> sendBuffers;
Thomas Witkowski's avatar
Thomas Witkowski committed
514
515
      sendBuffers.resize(sendDataSize.size());

516
517
      int requestCounter = 0;

518
      for (std::map<int, int>::iterator sendIt = sendDataSize.begin();
519
	   sendIt != sendDataSize.end(); ++sendIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
520
	sendBuffers[requestCounter] = sendIt->second;
521
522
 	request[requestCounter] =
 	  mpiComm.Isend(&(sendBuffers[requestCounter]), 1,
523
524
			MPI_INT, sendIt->first, 0);
	requestCounter++;
525
      }
526

527
      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
528
	   recvIt != recvDataSize.end(); ++recvIt) {
529
        request[requestCounter++] =
Thomas Witkowski's avatar
Thomas Witkowski committed
530
          mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0);
531
      }
532

Thomas Witkowski's avatar
Thomas Witkowski committed
533
      MPI::Request::Waitall(requestCounter, request);
534

535
#ifndef NDEBUG
536
537
538
      bool testall = MPI::Request::Testall(requestCounter, request);
      TEST_EXIT(testall)("Should not happen!\n");
#endif
539
540

      delete[] request;
541
542
    }

543
544
545
546

    /// Main function of the class, makes the communication. When the function
    /// returns, all data is send and received.
    void startCommunication()
547
548
549
550
551
    {
      FUNCNAME("StdMpi::startCommunication()");

      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

552
553
554
      typedef typename StdMpiHelper<SendT>::cppDataType cppDataType;
      MPI_Datatype mpiDataType = StdMpiHelper<SendT>::mpiDataType;

555
556
557
      if (exchangeDataSize)
	commDataSize();

Thomas Witkowski's avatar
Thomas Witkowski committed
558
      // === Remove empty data communication. ===
559

Thomas Witkowski's avatar
Thomas Witkowski committed
560
      {
561
	std::map<int, int>::iterator it = sendDataSize.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
562
563
564
565
566
567
568
569
570
571
572
573
	while (it != sendDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    sendData.erase(it->first);
	    sendDataSize.erase(it++);
	  } else
	    ++it;
	}
      }

      {
574
	std::map<int, int>::iterator it = recvDataSize.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
575
576
577
578
579
580
581
582
583
584
585
586
587
588
	while (it != recvDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    recvData.erase(it->first);
	    recvDataSize.erase(it++);
	  } else
	    ++it;
	}
      }


      // === Start communication. ===

589
      MPI::Request *request = new MPI::Request[sendData.size() + recvDataSize.size()];
590
      int requestCounter = 0;
591
      std::vector<cppDataType*> sendBuffers, recvBuffers;
592

593
      for (typename std::map<int, SendT>::iterator sendIt = sendData.begin();
594
	   sendIt != sendData.end(); ++sendIt) {
595
	int bufferSize = sendDataSize[sendIt->first];
Thomas Witkowski's avatar
Thomas Witkowski committed
596
597
598
599
600

	// Ommit sending empty buffers
	if (bufferSize == 0)
	  continue;

601
602
	cppDataType* buf = new cppDataType[bufferSize];
	StdMpiHelper<SendT>::createBuffer(sendIt->second, buf);
603

604
  	request[requestCounter++] =
605
  	  mpiComm.Isend(buf, bufferSize, mpiDataType, sendIt->first, 0);
606
607
608
609

	sendBuffers.push_back(buf);
      }

610
      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
611
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
612
613
614
	// Ommit receiving empty buffers
	if (recvIt->second == 0)
	  continue;
615

616
	cppDataType* buf = new cppDataType[recvIt->second];
617
618

  	request[requestCounter++] =
619
  	  mpiComm.Irecv(buf, recvIt->second, mpiDataType, recvIt->first, 0);
620
621

	recvBuffers.push_back(buf);
622
623
624
625
      }

      MPI::Request::Waitall(requestCounter, request);

Thomas Witkowski's avatar
Thomas Witkowski committed
626
      for (unsigned int i = 0; i < sendBuffers.size(); i++)
627
628
629
630
	delete [] sendBuffers[i];
      sendBuffers.clear();

      int i = 0;
631
      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
632
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
633
634
	if (recvIt->second == 0)
	  continue;
635

636
637
	StdMpiHelper<SendT>::makeFromBuffer(recvData[recvIt->first],
					    recvBuffers[i],
638
					    recvIt->second);
639
640
	delete [] recvBuffers[i];
	i++;
641
      }
642
      delete[] request;
643
644
645
      commPrepared = false;
    }

646
647
648
649
650
  protected:
    ///
    MPI::Intracomm mpiComm;

    ///
651
    std::map<int, SendT> sendData;
652
653

    ///
654
    std::map<int, RecvT> recvData;
655

656
    std::map<int, int> sendDataSize;
657

658
    std::map<int, int> recvDataSize;
659
660
661
662

    bool commPrepared;

    bool exchangeDataSize;
663
664

    static int ccc;
665
666
  };

667
} }
668
669

#endif