StdMpi.h 8.45 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  TU Dresden                                                            ==
// ==                                                                        ==
// ==  Institut fr Wissenschaftliches Rechnen                               ==
// ==  Zellescher Weg 12-14                                                  ==
// ==  01069 Dresden                                                         ==
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  https://gforge.zih.tu-dresden.de/projects/amdis/                      ==
// ==                                                                        ==
// ============================================================================

/** \file StdMpi.h */

#ifndef AMDIS_STDMPI_H
#define AMDIS_STDMPI_H

#include <map>
26
#include <stdint.h>
27
#include <mpi.h>
28
#include "MeshStructure.h"
29
#include "parallel/InteriorBoundary.h"
30
31
32

namespace AMDiS {

33
34
  int intSizeOf(int &data);

35
  int intSizeOf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data);
36

37
38
  int intSizeOf(std::map<WorldVector<double>, int> &data);

39
  int intSizeOf(std::vector<MeshStructure> &data);
40

41
  int intSizeOf(std::vector<WorldVector<double> > &data);
42

43
  int intSizeOf(std::vector<int> &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
44

45
  int intSizeOf(std::vector<double> &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
46
  
47
  int intSizeOf(std::vector<std::pair<int, int> > &data);
48

49
  int intSizeOf(std::vector<AtomicBoundary> &data);
50

51
  int intSizeOf(std::vector<BoundaryObject> &data);
52

53
  int intSizeOf(std::vector<const DegreeOfFreedom*> &data);
54

55
56
  int intSizeOf(std::vector<std::vector<double> > &data);

57
58
59
60
  void makeBuf(int &data, int *buf);

  void makeFromBuf(int &data, int *buf, int bufSize);

61
  void makeBuf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data, int* buf);
62

63
  void makeFromBuf(std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > &data, 
64
		   int *buf, int bufSize);
65

66
67
68
69
  void makeBuf(std::map<WorldVector<double>, int> &data, double* buf);

  void makeFromBuf(std::map<WorldVector<double>, int> &data, double* buf, int bufSize);

70
  void makeBuf(std::vector<MeshStructure> &data, uint64_t *buf);
71

72
  void makeFromBuf(std::vector<MeshStructure> &data, 
73
		   uint64_t *buf, int bufSize);
74

75
  void makeBuf(std::vector<WorldVector<double> > &data, double *buf);
76

77
  void makeFromBuf(std::vector<WorldVector<double> > &data, double *buf, int bufSize);
78

79
  void makeBuf(std::vector<int> &data, int *buf);
80

81
  void makeFromBuf(std::vector<int> &data, int *buf, int bufSize);
82

83
  void makeBuf(std::vector<double> &data, double *buf);
84

85
  void makeFromBuf(std::vector<double> &data, double *buf, int bufSize);
Thomas Witkowski's avatar
Thomas Witkowski committed
86

87
  void makeBuf(std::vector<std::pair<int, int> > &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
88

89
  void makeBuf(std::vector<AtomicBoundary> &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
90

91
  void makeFromBuf(std::vector<AtomicBoundary> &data, int *buf, int bufSize);
Thomas Witkowski's avatar
Thomas Witkowski committed
92

93
  void makeBuf(std::vector<BoundaryObject> &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
94

95
  void makeFromBuf(std::vector<BoundaryObject> &data, int *buf, int bufSize);
96

97
98
99
100
  void makeBuf(std::vector<std::vector<double> > &data, double *buf);

  void makeFromBuf(std::vector<std::vector<double> > &data, double *buf, int bufSize);

101
102


Thomas Witkowski's avatar
Thomas Witkowski committed
103
  template<typename SendT, typename RecvT=SendT>
104
105
106
  class StdMpi 
  {
  public:
107
    StdMpi(MPI::Intracomm &comm, bool b = true) :
108
      mpiComm(comm),
109
110
      commPrepared(true),
      exchangeDataSize(b)
111
112
    {}

113
    void clear()
114
115
116
    {
      sendData.clear();
      recvData.clear();
117
118
119
      sendDataSize.clear();
      recvDataSize.clear();

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
      commPrepared = true;
    }

    void send(int toRank, SendT& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      sendData[toRank] = data;
      sendDataSize[toRank] = intSizeOf(data);
    }

    void send(std::map<int, SendT>& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      for (typename std::map<int, SendT>::iterator it = data.begin(); 
	   it != data.end(); ++it) {
	sendData[it->first] = it->second;
	sendDataSize[it->first] = intSizeOf(it->second);
      }
    }

146
147
148
149
150
151
152
153
154
155
    std::map<int, SendT>& getSendData()
    {
      return sendData;
    }

    SendT& getSendData(int rank)
    {
      return sendData[rank];
    }

156
157
158
159
160
161
162
    void updateSendDataSize()
    {
      for (typename std::map<int, SendT>::iterator it = sendData.begin(); 
	   it != sendData.end(); ++it)
	sendDataSize[it->first] = intSizeOf(it->second);
    }

163
164
165
166
167
168
169
170
171
    void recv(int fromRank, int size = -1)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      recvDataSize[fromRank] = size;
    }

172
173
174
175
176
177
178
179
180
181
    void recvFromAll(int size = -1)
    {
      FUNCNAME("StdMpi::recvFromAll()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is no prepared!\n");

      for (int i = 0; i < mpiComm.Get_size(); i++)
	recvDataSize[i] = size;
    }

182
183
184
185
186
187
188
189
190
    template<class T>
    void recv(std::map<int, T> &fromRanks)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      for (typename std::map<int, T>::iterator it = fromRanks.begin();
	   it != fromRanks.end(); ++it)
191
	recvDataSize[it->first] = (exchangeDataSize ? -1 : intSizeOf(it->second));
192
193
    }

194
195
196
197
198
    std::map<int, RecvT>& getRecvData()
    {
      return recvData;
    }

199
200
201
202
203
    RecvT& getRecvData(int rank)
    {
      return recvData[rank];
    }

204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    void commDataSize()
    {
      MPI::Request request[sendData.size() + recvDataSize.size()];
      std::vector<int> sendBuffers, recvBuffers;
      int requestCounter = 0;

      int i = 0;
      for (typename std::map<int, int>::iterator sendIt = sendDataSize.begin();
	   sendIt != sendDataSize.end(); ++sendIt) {
	sendBuffers.push_back(sendIt->second);
	request[requestCounter++] =
	  mpiComm.Isend(&(sendBuffers[i]), 1, MPI_INT, sendIt->first, 0);
	i++;
      }

      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt)
	request[requestCounter++] =
	  mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0);
      
      MPI::Request::Waitall(requestCounter, request);      
    }

    template<class T>
228
    void startCommunication(MPI_Datatype m)
229
230
231
232
233
    {
      FUNCNAME("StdMpi::startCommunication()");

      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

234
235
236
      if (exchangeDataSize)
	commDataSize();

237
238
      MPI::Request request[sendData.size() + recvDataSize.size()];
      int requestCounter = 0;
239
      std::vector<T*> sendBuffers, recvBuffers;
240
241
242

      for (typename std::map<int, SendT>::iterator sendIt = sendData.begin();
	   sendIt != sendData.end(); ++sendIt) {
243
244
245
	int bufferSize = sendDataSize[sendIt->first];
	T* buf = new T[bufferSize];
	makeBuf(sendIt->second, buf);
246
247

  	request[requestCounter++] = 
248
  	  mpiComm.Isend(buf, bufferSize, m, sendIt->first, 0);
249
250
251
252
253
254

	sendBuffers.push_back(buf);
      }

      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt) {
255
	T* buf = new T[recvIt->second];
256
257

  	request[requestCounter++] =
258
259
  	  mpiComm.Irecv(buf, recvIt->second, m, recvIt->first, 0);
	
260
261
262
263
264
265
266
267
268
269
270
271
	recvBuffers.push_back(buf);	
      }

      MPI::Request::Waitall(requestCounter, request);

      for (int i = 0; i < static_cast<int>(sendBuffers.size()); i++)
	delete [] sendBuffers[i];
      sendBuffers.clear();

      int i = 0;
      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt) {
272

273
	makeFromBuf(recvData[recvIt->first], recvBuffers[i], recvIt->second);
274
275
276
	delete [] recvBuffers[i];
	i++;
      }      
277
278
279
280

      commPrepared = false;
    }

281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
  protected:
    ///
    MPI::Intracomm mpiComm;

    ///
    std::map<int, SendT> sendData;

    ///
    std::map<int, RecvT> recvData;

    std::map<int, int> sendDataSize;

    std::map<int, int> recvDataSize;

    bool commPrepared;

    bool exchangeDataSize;
  };

}

#endif