StdMpi.h 8.24 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  TU Dresden                                                            ==
// ==                                                                        ==
// ==  Institut fr Wissenschaftliches Rechnen                               ==
// ==  Zellescher Weg 12-14                                                  ==
// ==  01069 Dresden                                                         ==
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  https://gforge.zih.tu-dresden.de/projects/amdis/                      ==
// ==                                                                        ==
// ============================================================================

/** \file StdMpi.h */

#ifndef AMDIS_STDMPI_H
#define AMDIS_STDMPI_H

#include <map>
26
#include <mpi.h>
27
#include "MeshStructure.h"
28
#include "parallel/InteriorBoundary.h"
29
30
31

namespace AMDiS {

32
33
  int intSizeOf(int &data);

34
  int intSizeOf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data);
35

36
37
  int intSizeOf(std::map<WorldVector<double>, int> &data);

38
  int intSizeOf(std::vector<MeshStructure> &data);
39

40
  int intSizeOf(std::vector<WorldVector<double> > &data);
41

42
  int intSizeOf(std::vector<int> &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
43

44
  int intSizeOf(std::vector<double> &data);
Thomas Witkowski's avatar
Thomas Witkowski committed
45
  
46
  int intSizeOf(std::vector<std::pair<int, int> > &data);
47

48
  int intSizeOf(std::vector<AtomicBoundary> &data);
49

50
  int intSizeOf(std::vector<BoundaryObject> &data);
51

52
  int intSizeOf(std::vector<const DegreeOfFreedom*> &data);
53

54
55
56
57
  void makeBuf(int &data, int *buf);

  void makeFromBuf(int &data, int *buf, int bufSize);

58
  void makeBuf(std::map<const DegreeOfFreedom*, DegreeOfFreedom> &data, int* buf);
59

60
  void makeFromBuf(std::vector<std::pair<DegreeOfFreedom, DegreeOfFreedom> > &data, 
61
		   int *buf, int bufSize);
62

63
64
65
66
  void makeBuf(std::map<WorldVector<double>, int> &data, double* buf);

  void makeFromBuf(std::map<WorldVector<double>, int> &data, double* buf, int bufSize);

67
  void makeBuf(std::vector<MeshStructure> &data, unsigned long int *buf);
68

69
  void makeFromBuf(std::vector<MeshStructure> &data, 
70
		   unsigned long int *buf, int bufSize);
71

72
  void makeBuf(std::vector<WorldVector<double> > &data, double *buf);
73

74
  void makeFromBuf(std::vector<WorldVector<double> > &data, double *buf, int bufSize);
75

76
  void makeBuf(std::vector<int> &data, int *buf);
77

78
  void makeFromBuf(std::vector<int> &data, int *buf, int bufSize);
79

80
  void makeBuf(std::vector<double> &data, double *buf);
81

82
  void makeFromBuf(std::vector<double> &data, double *buf, int bufSize);
Thomas Witkowski's avatar
Thomas Witkowski committed
83

84
  void makeBuf(std::vector<std::pair<int, int> > &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
85

86
  void makeBuf(std::vector<AtomicBoundary> &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
87

88
  void makeFromBuf(std::vector<AtomicBoundary> &data, int *buf, int bufSize);
Thomas Witkowski's avatar
Thomas Witkowski committed
89

90
  void makeBuf(std::vector<BoundaryObject> &data, int *buf);
Thomas Witkowski's avatar
Thomas Witkowski committed
91

92
  void makeFromBuf(std::vector<BoundaryObject> &data, int *buf, int bufSize);
93

94
95


Thomas Witkowski's avatar
Thomas Witkowski committed
96
  template<typename SendT, typename RecvT=SendT>
97
98
99
  class StdMpi 
  {
  public:
100
    StdMpi(MPI::Intracomm &comm, bool b = true) :
101
      mpiComm(comm),
102
103
      commPrepared(true),
      exchangeDataSize(b)
104
105
    {}

106
    void clear()
107
108
109
    {
      sendData.clear();
      recvData.clear();
110
111
112
      sendDataSize.clear();
      recvDataSize.clear();

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      commPrepared = true;
    }

    void send(int toRank, SendT& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      sendData[toRank] = data;
      sendDataSize[toRank] = intSizeOf(data);
    }

    void send(std::map<int, SendT>& data)
    {
      FUNCNAME("StdMpi::send()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      for (typename std::map<int, SendT>::iterator it = data.begin(); 
	   it != data.end(); ++it) {
	sendData[it->first] = it->second;
	sendDataSize[it->first] = intSizeOf(it->second);
      }
    }

139
140
141
142
143
144
145
146
147
148
    std::map<int, SendT>& getSendData()
    {
      return sendData;
    }

    SendT& getSendData(int rank)
    {
      return sendData[rank];
    }

149
150
151
152
153
154
155
    void updateSendDataSize()
    {
      for (typename std::map<int, SendT>::iterator it = sendData.begin(); 
	   it != sendData.end(); ++it)
	sendDataSize[it->first] = intSizeOf(it->second);
    }

156
157
158
159
160
161
162
163
164
    void recv(int fromRank, int size = -1)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      recvDataSize[fromRank] = size;
    }

165
166
167
168
169
170
171
172
173
174
    void recvFromAll(int size = -1)
    {
      FUNCNAME("StdMpi::recvFromAll()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is no prepared!\n");

      for (int i = 0; i < mpiComm.Get_size(); i++)
	recvDataSize[i] = size;
    }

175
176
177
178
179
180
181
182
183
    template<class T>
    void recv(std::map<int, T> &fromRanks)
    {
      FUNCNAME("StdMpi::recv()");
      
      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

      for (typename std::map<int, T>::iterator it = fromRanks.begin();
	   it != fromRanks.end(); ++it)
184
	recvDataSize[it->first] = (exchangeDataSize ? -1 : intSizeOf(it->second));
185
186
    }

187
188
189
190
191
    std::map<int, RecvT>& getRecvData()
    {
      return recvData;
    }

192
193
194
195
196
    RecvT& getRecvData(int rank)
    {
      return recvData[rank];
    }

197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
    void commDataSize()
    {
      MPI::Request request[sendData.size() + recvDataSize.size()];
      std::vector<int> sendBuffers, recvBuffers;
      int requestCounter = 0;

      int i = 0;
      for (typename std::map<int, int>::iterator sendIt = sendDataSize.begin();
	   sendIt != sendDataSize.end(); ++sendIt) {
	sendBuffers.push_back(sendIt->second);
	request[requestCounter++] =
	  mpiComm.Isend(&(sendBuffers[i]), 1, MPI_INT, sendIt->first, 0);
	i++;
      }

      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt)
	request[requestCounter++] =
	  mpiComm.Irecv(&(recvIt->second), 1, MPI_INT, recvIt->first, 0);
      
      MPI::Request::Waitall(requestCounter, request);      
    }

    template<class T>
221
    void startCommunication(MPI_Datatype m)
222
223
224
225
226
    {
      FUNCNAME("StdMpi::startCommunication()");

      TEST_EXIT_DBG(commPrepared)("Communication is not prepared!\n");

227
228
229
      if (exchangeDataSize)
	commDataSize();

230
231
      MPI::Request request[sendData.size() + recvDataSize.size()];
      int requestCounter = 0;
232
      std::vector<T*> sendBuffers, recvBuffers;
233
234
235

      for (typename std::map<int, SendT>::iterator sendIt = sendData.begin();
	   sendIt != sendData.end(); ++sendIt) {
236
237
238
	int bufferSize = sendDataSize[sendIt->first];
	T* buf = new T[bufferSize];
	makeBuf(sendIt->second, buf);
239
240

  	request[requestCounter++] = 
241
  	  mpiComm.Isend(buf, bufferSize, m, sendIt->first, 0);
242
243
244
245
246
247

	sendBuffers.push_back(buf);
      }

      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt) {
248
	T* buf = new T[recvIt->second];
249
250

  	request[requestCounter++] =
251
252
  	  mpiComm.Irecv(buf, recvIt->second, m, recvIt->first, 0);
	
253
254
255
256
257
258
259
260
261
262
263
264
	recvBuffers.push_back(buf);	
      }

      MPI::Request::Waitall(requestCounter, request);

      for (int i = 0; i < static_cast<int>(sendBuffers.size()); i++)
	delete [] sendBuffers[i];
      sendBuffers.clear();

      int i = 0;
      for (std::map<int, int>::iterator recvIt = recvDataSize.begin();
	   recvIt != recvDataSize.end(); ++recvIt) {
265

266
	makeFromBuf(recvData[recvIt->first], recvBuffers[i], recvIt->second);
267
268
269
	delete [] recvBuffers[i];
	i++;
      }      
270
271
272
273

      commPrepared = false;
    }

274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
  protected:
    ///
    MPI::Intracomm mpiComm;

    ///
    std::map<int, SendT> sendData;

    ///
    std::map<int, RecvT> recvData;

    std::map<int, int> sendDataSize;

    std::map<int, int> recvDataSize;

    bool commPrepared;

    bool exchangeDataSize;
  };

}

#endif