StdMpi.h 14 KB
Newer Older
1
2
3
4
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
5
// ==  http://www.amdis-fem.org                                              ==
6
7
// ==                                                                        ==
// ============================================================================
8
9
10
11
12
13
14
15
16
17
18
19
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


20
21
22
23
24
25
26

/** \file StdMpi.h */

#ifndef AMDIS_STDMPI_H
#define AMDIS_STDMPI_H

#include <map>
27
#include <stdint.h>
28
#include <mpi.h>
29
#include "MeshStructure.h"
30
#include "parallel/InteriorBoundary.h"
31
32

namespace AMDiS {
33
34

  using namespace std;
35
36
   
  /** \brief
Thomas Witkowski's avatar
Thomas Witkowski committed
37
38
39
40
   * The class StdMpiHelper defines for a type a set of variables, types and
   * functions  that makes it able to transfer objects of the original type to
   * a buffer which is suitable to be send and received using MPI 
   * communication.
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
   *
   *    mpiDataType     Specifies the MPI data type that should be used for 
   *                    communication.
   *    cppDataType     Specifies the C++ data type of the buffer.
   *    getBufferSize   This functions calculates the size of the buffer for a
   *                    specific object.
   *    createBuffer    Create the buffer for a specific object.
   *    makeFromBuffer  Create an object of a buffer.
   */
  template<typename T>
  struct StdMpiHelper {};

  template<>
  struct StdMpiHelper<int> {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(int &data);

    static void createBuffer(int &data, int *buf);
    
    static void makeFromBuffer(int &data, int *buf, int bufSize);
  };


  template<>
68
  struct StdMpiHelper<vector<int> > {
69
    static MPI_Datatype mpiDataType;
70

71
    typedef int cppDataType;
72

73
74
75
76
77
78
79
80
81
    static int getBufferSize(vector<int> &data);

    static int getBufferSize(vector<const int*> &data);

    static void createBuffer(vector<int> &data, int *buf);
    
    static void makeFromBuffer(vector<int> &data, int *buf, int bufSize);
  };

82

83
84
85
86
87
88
89
  template<>
  struct StdMpiHelper<std::set<int> > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(std::set<int> &data);
90

91
    static void createBuffer(std::set<int> &data, int *buf);
92
    
93
    static void makeFromBuffer(std::set<int> &data, int *buf, int bufSize);
94
  };
95

96

97
98
99
100
101
102
103
104
105
106
107
108
109
110
  template<>
  struct StdMpiHelper<vector<std::set<int> > > {
    static MPI_Datatype mpiDataType;

    typedef int cppDataType;

    static int getBufferSize(vector<std::set<int> > &data);

    static void createBuffer(vector<std::set<int> > &data, int *buf);
    
    static void makeFromBuffer(vector<std::set<int> > &data, 
			       int *buf, int bufSize);
  };

111

112
  template<>
113
  struct StdMpiHelper<vector<double> > {
114
115
116
117
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

118
    static int getBufferSize(vector<double> &data);
119

120
    static void createBuffer(vector<double> &data, double *buf);
121

122
    static void makeFromBuffer(vector<double> &data, double *buf, int bufSize);
123
  };
124

Thomas Witkowski's avatar
Thomas Witkowski committed
125

126
  template<>
127
  struct StdMpiHelper<vector<vector<double> > > {
128
129
130
131
    static MPI_Datatype mpiDataType;

    typedef double cppDataType;

132
    static int getBufferSize(vector<vector<double> > &data);
133

134
    static void createBuffer(vector<vector<double> > &data, double *buf);
135

136
    static void makeFromBuffer(vector<vector<double> > &data, double *buf, int bufSize);
137
138
  };

Thomas Witkowski's avatar
Thomas Witkowski committed
139
  
140
141
  // MeshDistributor::MeshCodeVec
  template<>
142
  struct StdMpiHelper<vector<MeshStructure> > {
143
144
145
146
    static MPI_Datatype mpiDataType;

    typedef uint64_t cppDataType;

147
    static int getBufferSize(vector<MeshStructure> &data);
148

149
    static void createBuffer(vector<MeshStructure> &data, uint64_t *buf);
150
    
151
    static void makeFromBuffer(vector<MeshStructure> &data, 
152
153
			       uint64_t *buf, int bufSize);
  };
154

155

156
  template<>
157
  struct StdMpiHelper<vector<AtomicBoundary> > {
158
    static MPI_Datatype mpiDataType;
159

160
    typedef int cppDataType;
161

162
    static int getBufferSize(vector<AtomicBoundary> &data);
163

164
    static void createBuffer(vector<AtomicBoundary> &data, int *buf);
165

166
    static void makeFromBuffer(vector<AtomicBoundary> &data, int *buf, int bufSize);
167
  };
168
169


170
  // PeriodicMap::PeriodicDofMap
171
  template<>
172
  struct StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > > {
173
    static MPI_Datatype mpiDataType;
174

175
    typedef int cppDataType;
176

177
    static int getBufferSize(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data);
178

179
    static void createBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf);
180
    
181
    static void makeFromBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize);
182
  };
183

184

185
186
  // PetscSolver::createPetscNnzStructure::MatrixNnzEntry
  template<>
187
  struct StdMpiHelper<vector<std::pair<int, int> > > {
188
    static MPI_Datatype mpiDataType;
189

190
    typedef int cppDataType;
191

192
    static int getBufferSize(vector<std::pair<int, int> > &data);
193

194
    static void createBuffer(vector<std::pair<int, int> > &data, int *buf);
195

196
    static void makeFromBuffer(vector<std::pair<int, int> > &data, 
197
198
			       int *buf, int bufSize);
  };
199
200


201
202
  // ParallelDebug::CoordsVec
  template<>
203
  struct StdMpiHelper<vector<WorldVector<double> > > {
204
    static MPI_Datatype mpiDataType;
Thomas Witkowski's avatar
Thomas Witkowski committed
205

206
207
    typedef double cppDataType;
    
208
    static int getBufferSize(vector<WorldVector<double> > &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
209

210
    static void createBuffer(vector<WorldVector<double> > &data, double *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
211

212
    static void makeFromBuffer(vector<WorldVector<double> > &data, double *buf, int bufSize);
213
  };
Thomas Witkowski's avatar
Thomas Witkowski committed
214
215


216
217
  // ParallelDebug::testGlobalIndexByCoords::CoordsIndexMap
  template<>
218
  struct StdMpiHelper<map<WorldVector<double>, int> > {
219
    static MPI_Datatype mpiDataType;
220

221
    typedef double cppDataType;
222

223
    static int getBufferSize(map<WorldVector<double>, int> &data);
224

225
    static void createBuffer(map<WorldVector<double>, int> &data, double* buf);
226

227
    static void makeFromBuffer(map<WorldVector<double>, int> &data, double* buf, int bufSize);
228
  };
229

230
231
232
233

  /** \brief
   * This class is used to easily send and receive STL containers using MPI.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
234
  template<typename SendT, typename RecvT=SendT>
235
236
237
  class StdMpi 
  {
  public:
238
239
240
241
242
243
244
245
246
247
    /** \brief
     * Constructor of the class.
     *
     * \param[in]  comm   Reference to the MPI communicator that should be used.
     * \param[in]  b      If true, the size of the data that will be communicated
     *                    between the ranks will be communicated before to achieve
     *                    correct communication. If the value is false, the data size
     *                    must be set by hand and is assumed to be correct on all
     *                    sending and receiving ranks.
     */
248
    StdMpi(MPI::Intracomm &comm, bool b = true) :
249
      mpiComm(comm),
250
251
      commPrepared(true),
      exchangeDataSize(b)
252
253
    {}

254
    /// Reset all variables.
255
    void clear()
256
257
258
    {
      sendData.clear();
      recvData.clear();
259
260
261
      sendDataSize.clear();
      recvDataSize.clear();

262
263
264
      commPrepared = true;
    }

265
266
267
268
269
270
271

    /** \brief
     * Send data to rank.
     *
     * \param[in]  toRank   Rank number to which the data is send to.
     * \param[in]  data     Data to be send.
     */
272
273
274
275
276
277
278
    void send(int toRank, SendT& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      sendData[toRank] = data;
279
      sendDataSize[toRank] = StdMpiHelper<SendT>::getBufferSize(data);
280
281
    }

282
283
284
285
286
287
288

    /** \brief
     * Send data to rank.
     *
     * \param[in]  data    Maps rank number to the data that should be send to
     *                     these ranks.
     */
289
    void send(map<int, SendT>& data)
290
291
292
293
294
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

295
      for (typename map<int, SendT>::iterator it = data.begin(); 
296
297
	   it != data.end(); ++it) {
	sendData[it->first] = it->second;
298
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
299
300
301
      }
    }

302
303

    /// Returns sending data, see \ref sendData.
304
    map<int, SendT>& getSendData()
305
306
307
308
    {
      return sendData;
    }

309
310

    /// Returns the data that should be send to a specific rank, see \ref sendData.
311
312
313
314
315
    SendT& getSendData(int rank)
    {
      return sendData[rank];
    }

316
317

    /// Updates the buffer sizes for all sending data.
318
319
    void updateSendDataSize()
    {
320
      for (typename map<int, SendT>::iterator it = sendData.begin(); 
321
	   it != sendData.end(); ++it)
322
	sendDataSize[it->first] = StdMpiHelper<SendT>::getBufferSize(it->second);
323
324
    }

325
326
327
328
329
330
331

    /** \brief
     * Is used to specify from which rank data should be received.
     * 
     * \param[in]  fromRank   Rank number.
     * \param[in]  size       
     */
332
333
334
335
336
337
338
339
340
    void recv(int fromRank, int size = -1)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      recvDataSize[fromRank] = size;
    }

341
342

    /// Is used to specify that data is received from all ranks.
343
344
345
346
347
348
349
350
351
352
    void recvFromAll(int size = -1)
    {
      FUNCNAME("StdMpi::recvFromAll()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is no prepared!\n");

      for (int i = 0; i < mpiComm.Get_size(); i++)
	recvDataSize[i] = size;
    }

353
354

    /// Is used to specify from which rank data will be received.
355
    template<class T>
356
    void recv(map<int, T> &fromRanks)
357
358
359
360
361
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

362
      for (typename map<int, T>::iterator it = fromRanks.begin();
363
	   it != fromRanks.end(); ++it)
364
365
	recvDataSize[it->first] = 
	  (exchangeDataSize ? -1 : StdMpiHelper<SendT>::getBufferSize(it->second));
366
367
    }

368
369

    /// Returns received data, see \ref recvData.
370
    map<int, RecvT>& getRecvData()
371
372
373
374
    {
      return recvData;
    }

375
376

    /// Returns received data from a specific rank, see \ref recvData.
377
378
379
380
381
    RecvT& getRecvData(int rank)
    {
      return recvData[rank];
    }

382
383
384

    /// If data sizes should be exchanged before the data itself is communicated,
    /// this will be done in this function.
385
386
    void commDataSize()
    {
387
388
      FUNCNAME("StdMpi::commDataSize()");

389
      MPI::Request request[sendData.size() + recvDataSize.size()];
Thomas Witkowski's avatar
Thomas Witkowski committed
390
391
392
      vector<int> sendBuffers;
      sendBuffers.resize(sendDataSize.size());

393
394
      int requestCounter = 0;

Thomas Witkowski's avatar
Thomas Witkowski committed
395
      for (map<int, int>::iterator sendIt = sendDataSize.begin();
396
	   sendIt != sendDataSize.end(); ++sendIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
397
398
	sendBuffers[requestCounter] = sendIt->second;

399
400
401
402
 	request[requestCounter] = 
 	  mpiComm.Isend(&(sendBuffers[requestCounter]), 1, 
			MPI_INT, sendIt->first, 0);
	requestCounter++;
403
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
404
      
405
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
406
407
408
409
410
	   recvIt != recvDataSize.end(); ++recvIt)
        request[requestCounter++] = 
          mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0);
 
      MPI::Request::Waitall(requestCounter, request);
411
412
    }

413
414
415
416

    /// Main function of the class, makes the communication. When the function
    /// returns, all data is send and received.
    void startCommunication()
417
418
419
420
421
    {
      FUNCNAME("StdMpi::startCommunication()");

      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

422
423
424
      typedef typename StdMpiHelper<SendT>::cppDataType cppDataType;
      MPI_Datatype mpiDataType = StdMpiHelper<SendT>::mpiDataType;

425
426
427
      if (exchangeDataSize)
	commDataSize();

Thomas Witkowski's avatar
Thomas Witkowski committed
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459

      // === Remove empty data communication. ===
      
      {
	map<int, int>::iterator it = sendDataSize.begin();
	while (it != sendDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    sendData.erase(it->first);
	    sendDataSize.erase(it++);
	  } else
	    ++it;
	}
      }

      {
	map<int, int>::iterator it = recvDataSize.begin();
	while (it != recvDataSize.end()) {
	  TEST_EXIT_DBG(it->second >= 0)("Should not happen!\n");

	  if (it->second == 0) {
	    recvData.erase(it->first);
	    recvDataSize.erase(it++);
	  } else
	    ++it;
	}
      }


      // === Start communication. ===

460
461
      MPI::Request request[sendData.size() + recvDataSize.size()];
      int requestCounter = 0;
462
      vector<cppDataType*> sendBuffers, recvBuffers;
463

464
      for (typename map<int, SendT>::iterator sendIt = sendData.begin();
465
	   sendIt != sendData.end(); ++sendIt) {
466
	int bufferSize = sendDataSize[sendIt->first];
Thomas Witkowski's avatar
Thomas Witkowski committed
467
468
469
470
471

	// Ommit sending empty buffers
	if (bufferSize == 0)
	  continue;

472
473
	cppDataType* buf = new cppDataType[bufferSize];
	StdMpiHelper<SendT>::createBuffer(sendIt->second, buf);
474
475

  	request[requestCounter++] = 
476
  	  mpiComm.Isend(buf, bufferSize, mpiDataType, sendIt->first, 0);
477
478
479
480

	sendBuffers.push_back(buf);
      }

481
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
482
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
483
484
485
	// Ommit receiving empty buffers
	if (recvIt->second == 0)
	  continue;
486

487
	cppDataType* buf = new cppDataType[recvIt->second];
488
489

  	request[requestCounter++] =
490
  	  mpiComm.Irecv(buf, recvIt->second, mpiDataType, recvIt->first, 0);
491
	
492
493
494
495
496
	recvBuffers.push_back(buf);	
      }

      MPI::Request::Waitall(requestCounter, request);

Thomas Witkowski's avatar
Thomas Witkowski committed
497
      for (unsigned int i = 0; i < sendBuffers.size(); i++)
498
499
500
501
	delete [] sendBuffers[i];
      sendBuffers.clear();

      int i = 0;
502
      for (map<int, int>::iterator recvIt = recvDataSize.begin();
503
	   recvIt != recvDataSize.end(); ++recvIt) {
Thomas Witkowski's avatar
Thomas Witkowski committed
504
505
	if (recvIt->second == 0)
	  continue;
506

507
508
509
	StdMpiHelper<SendT>::makeFromBuffer(recvData[recvIt->first], 
					    recvBuffers[i], 
					    recvIt->second);
510
511
512
	delete [] recvBuffers[i];
	i++;
      }      
513
514
515
516

      commPrepared = false;
    }

517
518
519
520
521
  protected:
    ///
    MPI::Intracomm mpiComm;

    ///
522
    map<int, SendT> sendData;
523
524

    ///
525
    map<int, RecvT> recvData;
526

527
    map<int, int> sendDataSize;
528

529
    map<int, int> recvDataSize;
530
531
532
533
534
535
536
537
538

    bool commPrepared;

    bool exchangeDataSize;
  };

}

#endif