StdMpi.h 15.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/******************************************************************************
 *
 * AMDiS - Adaptive multidimensional simulations
 *
 * Copyright (C) 2013 Dresden University of Technology. All Rights Reserved.
 * Web: https://fusionforge.zih.tu-dresden.de/projects/amdis
 *
 * Authors: 
 * Simon Vey, Thomas Witkowski, Andreas Naumann, Simon Praetorius, et al.
 *
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *
 * This file is part of AMDiS
 *
 * See also license.opensource.txt in the distribution.
 * 
 ******************************************************************************/
20
21


22
23
24
25
26
27
28

/** \file StdMpi.h */

#ifndef AMDIS_STDMPI_H
#define AMDIS_STDMPI_H

#include <map>
29
#include <stdint.h>
30
#include <mpi.h>
31
#include "MeshStructure.h"
32
#include "parallel/InteriorBoundary.h"
33

34
namespace AMDiS { namespace Parallel {
35
36

  using namespace std;
37
38
   
  /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
39
40
41
42
   * The class StdMpiHelper defines for a type a set of variables, types and
   * functions  that makes it able to transfer objects of the original type to
   * a buffer which is suitable to be send and received using MPI 
   * communication.
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
   *
   *    mpiDataType     Specifies the MPI data type that should be used for 
   *                    communication.
   *    cppDataType     Specifies the C++ data type of the buffer.
   *    getBufferSize   This functions calculates the size of the buffer for a
   *                    specific object.
   *    createBuffer    Create the buffer for a specific object.
   *    makeFromBuffer  Create an object of a buffer.
   */
  template<typename T>
  struct StdMpiHelper {};

  template<>
  struct StdMpiHelper<int> {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(int &data);

    static void createBuffer(int &data, int *buf);
    
    static void makeFromBuffer(int &data, int *buf, int bufSize);
  };


  template<>
70
  struct StdMpiHelper<vector<int> > {
71
    static MPI_Datatype mpiDataType;
72

73
    typedef int cppDataType;
74

75
76
77
78
79
80
81
82
83
    static int getBufferSize(vector<int> &data);

    static int getBufferSize(vector<const int*> &data);

    static void createBuffer(vector<int> &data, int *buf);
    
    static void makeFromBuffer(vector<int> &data, int *buf, int bufSize);
  };

84

85
86
87
88
89
90
91
  template<>
  struct StdMpiHelper<std::set<int> > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(std::set<int> &data);
92

93
    static void createBuffer(std::set<int> &data, int *buf);
94
    
95
    static void makeFromBuffer(std::set<int> &data, int *buf, int bufSize);
96
  };
97

98

99
100
101
102
103
104
105
106
107
108
109
110
111
112
  template<>
  struct StdMpiHelper<vector<std::set<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(vector<std::set<int> > &data);

    static void createBuffer(vector<std::set<int> > &data, int *buf);
    
    static void makeFromBuffer(vector<std::set<int> > &data, 
			       int *buf, int bufSize);
  };

113

114
  template<>
115
  struct StdMpiHelper<vector<double> > {
116
117
118
119
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

120
    static int getBufferSize(vector<double> &data);
121

122
    static void createBuffer(vector<double> &data, double *buf);
123

124
    static void makeFromBuffer(vector<double> &data, double *buf, int bufSize);
125
  };
126

Thomas Witkowski's avatar
Thomas Witkowski committed
127

128
  template<>
129
  struct StdMpiHelper<vector<vector<double> > > {
130
131
132
133
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

134
    static int getBufferSize(vector<vector<double> > &data);
135

136
    static void createBuffer(vector<vector<double> > &data, double *buf);
137

138
    static void makeFromBuffer(vector<vector<double> > &data, double *buf, int bufSize);
139
140
  };

Thomas Witkowski's avatar
Thomas Witkowski committed
141
  
142
143
  // MeshDistributor::MeshCodeVec
  template<>
144
  struct StdMpiHelper<vector<MeshStructure> > {
145
146
147
148
    static MPI_Datatype mpiDataType;

    typedef uint64_t cppDataType;

149
    static int getBufferSize(vector<MeshStructure> &data);
150

151
    static void createBuffer(vector<MeshStructure> &data, uint64_t *buf);
152
    
153
    static void makeFromBuffer(vector<MeshStructure> &data, 
154
155
			       uint64_t *buf, int bufSize);
  };
156

157

158
  template<>
159
  struct StdMpiHelper<vector<AtomicBoundary> > {
160
    static MPI_Datatype mpiDataType;
161

162
    typedef int cppDataType;
163

164
    static int getBufferSize(vector<AtomicBoundary> &data);
165

166
    static void createBuffer(vector<AtomicBoundary> &data, int *buf);
167

168
    static void makeFromBuffer(vector<AtomicBoundary> &data, int *buf, int bufSize);
169
  };
170
171


172
  // PeriodicMap::PeriodicDofMap
173
  template<>
174
  struct StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > > {
175
    static MPI_Datatype mpiDataType;
176

177
    typedef int cppDataType;
178

179
    static int getBufferSize(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data);
180

181
    static void createBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf);
182
    
183
    static void makeFromBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize);
184
  };
185

186

187
188
  // PetscSolver::createPetscNnzStructure::MatrixNnzEntry
  template<>
189
  struct StdMpiHelper<vector<std::pair<int, int> > > {
190
    static MPI_Datatype mpiDataType;
191

192
    typedef int cppDataType;
193

194
    static int getBufferSize(vector<std::pair<int, int> > &data);
195

196
    static void createBuffer(vector<std::pair<int, int> > &data, int *buf);
197

198
    static void makeFromBuffer(vector<std::pair<int, int> > &data, 
199
200
			       int *buf, int bufSize);
  };
201
202


203
204
  // ParallelDebug::CoordsVec
  template<>
205
  struct StdMpiHelper<vector<WorldVector<double> > > {
206
    static MPI_Datatype mpiDataType;
Thomas Witkowski's avatar
Thomas Witkowski committed
207

208
209
    typedef double cppDataType;
    
210
    static int getBufferSize(vector<WorldVector<double> > &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
211

212
    static void createBuffer(vector<WorldVector<double> > &data, double *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
213

214
    static void makeFromBuffer(vector<WorldVector<double> > &data, double *buf, int bufSize);
215
  };
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217


218
219
  // ParallelDebug::testGlobalIndexByCoords::CoordsIndexMap
  template<>
220
  struct StdMpiHelper<map<WorldVector<double>, int> > {
221
    static MPI_Datatype mpiDataType;
222

223
    typedef double cppDataType;
224

225
    static int getBufferSize(map<WorldVector<double>, int> &data);
226

227
    static void createBuffer(map<WorldVector<double>, int> &data, double* buf);
228

229
    static void makeFromBuffer(map<WorldVector<double>, int> &data, double* buf, int bufSize);
230
  };
231
232
233
234
235
236
237
238
239
240
241
242
  template<>
  struct StdMpiHelper<map<int, WorldVector<double> > > {
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

    static int getBufferSize(map<int, WorldVector<double> > &data);

    static void createBuffer(map<int, WorldVector<double> > &data, double* buf);

    static void makeFromBuffer(map<int, WorldVector<double> > &data, double* buf, int bufSize);
  };
243

244

245

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
  template<>
  struct StdMpiHelper<vector<vector<WorldVector<double> > > > {
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

    typedef vector<vector<WorldVector<double> > >  DataType;

    static int getBufferSize(DataType &data);

    static void createBuffer(DataType &data, double* buf);

    static void makeFromBuffer(DataType &data, double* buf, int bufSize);
  };


262
263
264
  /** \brief
   * This class is used to easily send and receive STL containers using MPI.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
265
  template<typename SendT, typename RecvT=SendT>
266
267
268
  class StdMpi 
  {
  public:
269
270
271
272
273
274
275
276
277
278
    /** \brief
     * Constructor of the class.
     *
     * \param[in]  comm   Reference to the MPI communicator that should be used.
     * \param[in]  b      If true, the size of the data that will be communicated
     *                    between the ranks will be communicated before to achieve
     *                    correct communication. If the value is false, the data size
     *                    must be set by hand and is assumed to be correct on all
     *                    sending and receiving ranks.
     */
279
    StdMpi(MPI::Intracomm &comm, bool b = true) :
280
      mpiComm(comm),
281
282
      commPrepared(true),
      exchangeDataSize(b)
283
284
    {}

285
    /// Reset all variables.
286
    void clear()
287
288
289
    {
      sendData.clear();
      recvData.clear();
290
291
292
      sendDataSize.clear();
      recvDataSize.clear();

293
294
295
      commPrepared = true;
    }

296
297
298
299
300
301
302

    /** \brief
     * Send data to rank.
     *
     * \param[in]  toRank   Rank number to which the data is send to.
     * \param[in]  data     Data to be send.
     */
303
304
305
306
307
308
309
    void send(int toRank, SendT& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      sendData[toRank] = data;
310
      sendDataSize[toRank] = StdMpiHelper<SendT>::getBufferSize(data);
311
312
    }

313
314
315
316
317
318
319

    /** \brief
     * Send data to rank.
     *
     * \param[in]  data    Maps rank number to the data that should be send to
     *                     these ranks.
     */
320
    void send(map<int, SendT>& data)
321
322
323
324
325
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

326
      for (typename map<int, SendT>::iterator it = data.begin(); 
327
328
	   it != data.end(); ++it) {
	sendData[it->first] = it->second;
329
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
330
331
332
      }
    }

333
334

    /// Returns sending data, see \ref sendData.
335
    inline map<int, SendT>& getSendData()
336
337
338
339
    {
      return sendData;
    }

340
341

    /// Returns the data that should be send to a specific rank, see \ref sendData.
342
    inline SendT& getSendData(int rank)
343
344
345
346
    {
      return sendData[rank];
    }

347
348

    /// Updates the buffer sizes for all sending data.
349
350
    void updateSendDataSize()
    {
351
      for (typename map<int, SendT>::iterator it = sendData.begin(); 
352
	   it != sendData.end(); ++it)
353
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
354
355
    }

356
357
358
359
360
361
362

    /** \brief
     * Is used to specify from which rank data should be received.
     * 
     * \param[in]  fromRank   Rank number.
     * \param[in]  size       
     */
363
364
365
366
367
368
369
370
371
    void recv(int fromRank, int size = -1)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      recvDataSize[fromRank] = size;
    }

372
373

    /// Is used to specify that data is received from all ranks.
374
375
376
377
378
379
380
381
382
383
    void recvFromAll(int size = -1)
    {
      FUNCNAME("StdMpi::recvFromAll()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is no prepared!\n");

      for (int i = 0; i < mpiComm.Get_size(); i++)
	recvDataSize[i] = size;
    }

384
385

    /// Is used to specify from which rank data will be received.
386
    template<class T>
387
    void recv(map<int, T> &fromRanks)
388
389
390
391
392
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

393
      for (typename map<int, T>::iterator it = fromRanks.begin();
394
	   it != fromRanks.end(); ++it)
395
396
	recvDataSize[it->first] = 
	  (exchangeDataSize ? -1 : StdMpiHelper<SendT>::getBufferSize(it->second));
397
398
    }

399
400

    /// Returns received data, see \ref recvData.
401
    map<int, RecvT>& getRecvData()
402
403
404
405
    {
      return recvData;
    }

406
407

    /// Returns received data from a specific rank, see \ref recvData.
408
409
    RecvT& getRecvData(int rank)
    {
410
411
      FUNCNAME("StdMpi::getRecvData()");
      TEST_EXIT_DBG(recvData.count(rank))("No recv data from rank %d\n", rank);
412
413
414
      return recvData[rank];
    }

415
416
417

    /// If data sizes should be exchanged before the data itself is communicated,
    /// this will be done in this function.
418
419
    void commDataSize()
    {
420
421
      FUNCNAME("StdMpi::commDataSize()");

422
      MPI::Request* request= new MPI::Request[sendData.size() + recvDataSize.size()];
Thomas Witkowski's avatar
Thomas Witkowski committed
423
424
425
      vector<int> sendBuffers;
      sendBuffers.resize(sendDataSize.size());

426
427
      int requestCounter = 0;

Thomas Witkowski's avatar
Thomas Witkowski committed
428
      for (map<int, int>::iterator sendIt = sendDataSize.begin();
429
	   sendIt != sendDataSize.end(); ++sendIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
430
	sendBuffers[requestCounter] = sendIt->second;
431
432
433
434
 	request[requestCounter] = 
 	  mpiComm.Isend(&(sendBuffers[requestCounter]), 1, 
			MPI_INT, sendIt->first, 0);
	requestCounter++;
435
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
436
      
437
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
438
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
439
440
        request[requestCounter++] = 
          mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0);
441
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
442
443
 
      MPI::Request::Waitall(requestCounter, request);
444
445
446
447
448

#if (DEBUG != 0)
      bool testall = MPI::Request::Testall(requestCounter, request);
      TEST_EXIT(testall)("Should not happen!\n");
#endif
449
450

      delete[] request;
451
452
    }

453
454
455
456

    /// Main function of the class, makes the communication. When the function
    /// returns, all data is send and received.
    void startCommunication()
457
458
459
460
461
    {
      FUNCNAME("StdMpi::startCommunication()");

      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

462
463
464
      typedef typename StdMpiHelper<SendT>::cppDataType cppDataType;
      MPI_Datatype mpiDataType = StdMpiHelper<SendT>::mpiDataType;

465
466
467
      if (exchangeDataSize)
	commDataSize();

Thomas Witkowski's avatar
Thomas Witkowski committed
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      // === Remove empty data communication. ===
      
      {
	map<int, int>::iterator it = sendDataSize.begin();
	while (it != sendDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    sendData.erase(it->first);
	    sendDataSize.erase(it++);
	  } else
	    ++it;
	}
      }

      {
	map<int, int>::iterator it = recvDataSize.begin();
	while (it != recvDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    recvData.erase(it->first);
	    recvDataSize.erase(it++);
	  } else
	    ++it;
	}
      }


      // === Start communication. ===

499
      MPI::Request *request = new MPI::Request[sendData.size() + recvDataSize.size()];
500
      int requestCounter = 0;
501
      vector<cppDataType*> sendBuffers, recvBuffers;
502

503
      for (typename map<int, SendT>::iterator sendIt = sendData.begin();
504
	   sendIt != sendData.end(); ++sendIt) {
505
	int bufferSize = sendDataSize[sendIt->first];
Thomas Witkowski's avatar
Thomas Witkowski committed
506
507
508
509
510

	// Ommit sending empty buffers
	if (bufferSize == 0)
	  continue;

511
512
	cppDataType* buf = new cppDataType[bufferSize];
	StdMpiHelper<SendT>::createBuffer(sendIt->second, buf);
513
514

  	request[requestCounter++] = 
515
  	  mpiComm.Isend(buf, bufferSize, mpiDataType, sendIt->first, 0);
516
517
518
519

	sendBuffers.push_back(buf);
      }

520
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
521
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
522
523
524
	// Ommit receiving empty buffers
	if (recvIt->second == 0)
	  continue;
525

526
	cppDataType* buf = new cppDataType[recvIt->second];
527
528

  	request[requestCounter++] =
529
  	  mpiComm.Irecv(buf, recvIt->second, mpiDataType, recvIt->first, 0);
530
	
531
532
533
534
535
	recvBuffers.push_back(buf);	
      }

      MPI::Request::Waitall(requestCounter, request);

Thomas Witkowski's avatar
Thomas Witkowski committed
536
      for (unsigned int i = 0; i < sendBuffers.size(); i++)
537
538
539
540
	delete [] sendBuffers[i];
      sendBuffers.clear();

      int i = 0;
541
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
542
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
543
544
	if (recvIt->second == 0)
	  continue;
545

546
547
548
	StdMpiHelper<SendT>::makeFromBuffer(recvData[recvIt->first], 
					    recvBuffers[i], 
					    recvIt->second);
549
550
551
	delete [] recvBuffers[i];
	i++;
      }      
552
      delete[] request;
553
554
555
      commPrepared = false;
    }

556
557
558
559
560
  protected:
    ///
    MPI::Intracomm mpiComm;

    ///
561
    map<int, SendT> sendData;
562
563

    ///
564
    map<int, RecvT> recvData;
565

566
    map<int, int> sendDataSize;
567

568
    map<int, int> recvDataSize;
569
570
571
572

    bool commPrepared;

    bool exchangeDataSize;
573
574

    static int ccc;
575
576
  };

577
} }
578
579

#endif