StdMpi.cc 15.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/******************************************************************************
 *
 * AMDiS - Adaptive multidimensional simulations
 *
 * Copyright (C) 2013 Dresden University of Technology. All Rights Reserved.
 * Web: https://fusionforge.zih.tu-dresden.de/projects/amdis
 *
 * Authors: 
 * Simon Vey, Thomas Witkowski, Andreas Naumann, Simon Praetorius, et al.
 *
 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *
 * This file is part of AMDiS
 *
 * See also license.opensource.txt in the distribution.
 * 
 ******************************************************************************/
20
21


22
23
#include "StdMpi.h"

24
25
using namespace std;

26
namespace AMDiS { namespace Parallel {
27

28
  MPI_Datatype StdMpiHelper<int>::mpiDataType = MPI_INT;
29
30
31
32
33
  MPI_Datatype StdMpiHelper<vector<int> >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<std::set<int> >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<vector<std::set<int> > >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<vector<double> >::mpiDataType = MPI_DOUBLE;
  MPI_Datatype StdMpiHelper<vector<vector<double> > >::mpiDataType = MPI_DOUBLE;
34
  MPI_Datatype StdMpiHelper<vector<MeshStructure> >::mpiDataType = MPI_UNSIGNED_LONG_LONG;
35
36
37
38
39
  MPI_Datatype StdMpiHelper<vector<AtomicBoundary> >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<vector<std::pair<int, int> > >::mpiDataType = MPI_INT;
  MPI_Datatype StdMpiHelper<vector<WorldVector<double> > >::mpiDataType = MPI_DOUBLE;
  MPI_Datatype StdMpiHelper<map<WorldVector<double>, int> >::mpiDataType = MPI_DOUBLE;
40
  MPI_Datatype StdMpiHelper<map<int, WorldVector<double> > >::mpiDataType = MPI_DOUBLE;
41
  MPI_Datatype StdMpiHelper<vector<vector<WorldVector<double> > > >::mpiDataType = MPI_DOUBLE;
42
43
44
45

  // T = int

  int StdMpiHelper<int>::getBufferSize(int &data)
46
47
48
49
  {
    return 1;
  }

50
  void StdMpiHelper<int>::createBuffer(int &data, int *buf)
51
  {
52
    buf[0] = data;
53
54
  }

55
  void StdMpiHelper<int>::makeFromBuffer(int &data, int *buf, int bufSize)
56
  {
57
    data = buf[0];
58
59
  }

60
61


62
  // T = vector<int>
63

64
  int StdMpiHelper<vector<int> >::getBufferSize(vector<int> &data)
65
  {
66
    return data.size();
67
68
  }

69
  int StdMpiHelper<vector<int> >::getBufferSize(vector<const int*> &data)
70
71
72
73
  {
    return data.size();
  }

74
  void StdMpiHelper<vector<int> >::createBuffer(vector<int> &data, int *buf)
75
  {
76
77
    for (unsigned int i = 0; i < data.size(); i++)
      buf[i] = data[i];
78
  }
79

80
  void StdMpiHelper<vector<int> >::makeFromBuffer(vector<int> &data, int *buf, int bufSize)
81
  {
82
83
84
85
    data.resize(bufSize);

    for (int i = 0; i < bufSize; i++)
      data[i] = buf[i];
86
87
  }

88
89


90
  // T = std::set<int>
91

92
  int StdMpiHelper<std::set<int> >::getBufferSize(std::set<int> &data)
93
  {
94
    return data.size();
95
96
  }

97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
  void StdMpiHelper<std::set<int> >::createBuffer(std::set<int> &data, int *buf)
  {
    int i = 0;
    for (std::set<int>::iterator it = data.begin(); it != data.end(); ++it)
      buf[i++] = *it;
  }

  void StdMpiHelper<std::set<int> >::makeFromBuffer(std::set<int> &data, int *buf, int bufSize)
  {
    data.clear();

    for (int i = 0; i < bufSize; i++)
      data.insert(buf[i]);
  }


  // T = vector<std::set<int> >

  int StdMpiHelper<vector<std::set<int> > >::getBufferSize(vector<std::set<int> > &data)
  {
    int dataSize = 1;
    for (unsigned int i = 0; i < data.size(); i++)
      dataSize += data[i].size() + 1;

    return dataSize;
  }

124
125
  void StdMpiHelper<vector<std::set<int> > >::createBuffer(vector<std::set<int> > &data, 
							   int *buf)
126
127
128
129
130
131
132
133
134
135
136
  {
    int counter = 1;
    buf[0] = data.size();

    for (unsigned int i = 0; i < data.size(); i++) {
      buf[counter++] = data[i].size();
      for (std::set<int>::iterator it = data[i].begin(); it != data[i].end(); ++it)
	buf[counter++] = *it;
    }
  }

137
138
139
  void StdMpiHelper<vector<std::set<int> > >::makeFromBuffer(vector<std::set<int> > &data, 
							     int *buf, 
							     int bufSize)
140
  {
141
    FUNCNAME("StdMpiHelper<vector<set<int> > >::makeFromBuffer()");
142
143
144
145
146
147
148

    int counter = 1;
    data.resize(buf[0]);

    for (int i = 0; i < buf[0]; i++) {
      data[i].clear();
      int setSize = buf[counter++];
149
150
151
      TEST_EXIT_DBG(setSize <= bufSize - counter)
	("Should not happen: %d %d %d\n", setSize, bufSize, counter);

152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
      for (int j = 0; j < setSize; j++)
	data[i].insert(buf[counter++]);
    }

    TEST_EXIT(counter == bufSize)("Counter is %d, but buffer size is %d!\n",
				  counter, bufSize);
  }



  // T = vector<double>

  int StdMpiHelper<vector<double> >::getBufferSize(vector<double> &data)
  {
    return data.size();
  }

169
170
  void StdMpiHelper<vector<double> >::createBuffer(vector<double> &data, 
						   double *buf)
171
  {
172
173
    for (unsigned int i = 0; i < data.size(); i++)
      buf[i] = data[i];
174
175
  }

176
  void StdMpiHelper<vector<double> >::makeFromBuffer(vector<double> &data, double *buf, int bufSize)
177
  {
178
179
180
181
    data.resize(bufSize);

    for (int i = 0; i < bufSize; i++)
      data[i] = buf[i];
182
183
  }

184
185


186
  // T = vector<vector<double> >
187

188
  int StdMpiHelper<vector<vector<double> > >::getBufferSize(vector<vector<double> > &data)
189
190
191
192
193
194
195
196
197
  {
    int size = 1;

    for (unsigned int i = 0; i < data.size(); i++)
      size += data[i].size() + 1;
    
    return size;
  }

198
199
  void StdMpiHelper<vector<vector<double> > >::createBuffer(vector<vector<double> > &data, 
							    double *buf)
200
  {
201
202
    buf[0] = data.size();
    int counter = 1;
203

204
205
206
207
    for (unsigned int i = 0; i < data.size(); i++) {
      buf[counter++] = data[i].size();
      for (unsigned int j = 0; j < data[i].size(); j++)
	buf[counter++] = data[i][j];
208
209
210
    }
  }

211
212
213
  void StdMpiHelper<vector<vector<double> > >::makeFromBuffer(vector<vector<double> > &data, 
							      double *buf, 
							      int bufSize)
214
  {
215
216
    data.resize(static_cast<unsigned int>(buf[0]));
    int counter = 1;
217

218
219
220
221
222
    for (unsigned int i = 0; i < data.size(); i++) {
      data[i].resize(static_cast<unsigned int>(buf[counter++]));
      
      for (unsigned int j = 0; j < data[i].size(); j++)
	data[i][j] = buf[counter++];
223
224
225
226
227
    }
  }



228
  // T = vector<MeshStructure>
229

230
  int StdMpiHelper<vector<MeshStructure> >::getBufferSize(vector<MeshStructure> &data)
231
  {
232
233
    FUNCNAME("StdMpiHelper<vector<MeshStructure> >::getBufferSize()");

234
235
236
    int size = 0;
    for (unsigned int i = 0; i < data.size(); i++)
      size += data[i].getCode().size() + 2;
237

238
    return size;
239
240
  }

241
  void StdMpiHelper<vector<MeshStructure> >::createBuffer(vector<MeshStructure> &data, uint64_t *buf)
242
  {    
243
244
    FUNCNAME("StdMpiHelper<vector<MeshStructure> >::createBuffer()");

245
246
247
248
249
250
251
252
253
254
    int pos = 0;
    for (unsigned int i = 0; i < data.size(); i++) {
      buf[pos++] = data[i].getCode().size();
      buf[pos++] = data[i].getNumElements();

      for (unsigned int j = 0; j < data[i].getCode().size(); j++)
	buf[pos++] = data[i].getCode()[j];
    }
  }

255
  void StdMpiHelper<vector<MeshStructure> >::makeFromBuffer(vector<MeshStructure> &data, 
256
							    uint64_t *buf, int bufSize)
257
  {
258
259
    FUNCNAME("StdMpiHelper<vector<MeshStructure> >::makeFromBuffer()");

260
261
    int pos = 0;

262
    while (pos < bufSize) {
263
264
      int codeSize = buf[pos++];
      int nElements = buf[pos++];
265
      vector<uint64_t> code;
266
267
268
269
270
271
272
273
274
275
276
277
278
      code.resize(codeSize);
      for (int i = 0; i < codeSize; i++)
	code[i] = buf[pos++];

      MeshStructure meshCode;
      meshCode.init(code, nElements);

      data.push_back(meshCode);
    }	
  }



279
  // T = vector<AtomicBoundary>
280

281
  int StdMpiHelper<vector<AtomicBoundary> >::getBufferSize(vector<AtomicBoundary> &data)
282
  {
283
    return data.size() * 6;
284
285
  }

286
  void StdMpiHelper<vector<AtomicBoundary> >::createBuffer(vector<AtomicBoundary> &data, int *buf)
287
288
  {
    for (unsigned int i = 0; i < data.size(); i++) {
289
290
291
292
293
294
      buf[i * 6] = data[i].rankObj.elIndex;
      buf[i * 6 + 1] = data[i].rankObj.subObj;
      buf[i * 6 + 2] = data[i].rankObj.ithObj;
      buf[i * 6 + 3] = data[i].neighObj.elIndex;
      buf[i * 6 + 4] = data[i].neighObj.subObj;
      buf[i * 6 + 5] = data[i].neighObj.ithObj;
295
296
297
    }
  }

298
  void StdMpiHelper<vector<AtomicBoundary> >::makeFromBuffer(vector<AtomicBoundary> &data, int *buf, int bufSize)
299
300
301
302
  {
    if (bufSize == 0)
      return;

303
    TEST_EXIT(bufSize % 6 == 0)("This should not happen!\n");    
304

305
306
307
308
309
310
311
312
    data.resize(bufSize / 6);
    for (int i = 0; i < bufSize / 6; i++) {
      data[i].rankObj.elIndex = buf[i * 6];
      data[i].rankObj.subObj = static_cast<GeoIndex>(buf[i * 6 + 1]);
      data[i].rankObj.ithObj = buf[i * 6 + 2];
      data[i].neighObj.elIndex = buf[i * 6 + 3];
      data[i].neighObj.subObj = static_cast<GeoIndex>(buf[i * 6 + 4]);
      data[i].neighObj.ithObj = buf[i * 6 + 5];
313
314
315
316
317
    }
  }



318
  // T = map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> >
319

320
  int StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > >::getBufferSize(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data)
321
  {
322
    int size = 1;
323

324
    for (map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
325
326
	 it != data.end(); ++it) {
      size += 2 + it->second.size() * 2;
327
328
    }

329
    return size;
330
331
  }

332
  void StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > >::createBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf)
333
334
335
336
  {
    buf[0] = data.size();
    int counter = 1;

337
    for (map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
338
339
340
341
	 it != data.end(); ++it) {
      buf[counter++] = it->first;
      buf[counter++] = it->second.size();

342
      for (map<DegreeOfFreedom, DegreeOfFreedom>::iterator it2 = it->second.begin();
343
344
345
346
347
348
349
	   it2 != it->second.end(); ++it2) {
	buf[counter++] = it2->first;
	buf[counter++] = it2->second;
      }
    }
  }

350
  void StdMpiHelper<map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > >::makeFromBuffer(map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize)
351
352
353
354
355
356
357
358
359
360
  {
    data.clear();

    if (bufSize == 0)
      return;

    int counter = 1;
    
    for (int i = 0; i < buf[0]; i++) {
      BoundaryType bound = buf[counter++];
361
      map<DegreeOfFreedom, DegreeOfFreedom> dofs;
362
363
364
365
366
367
368
369
370
371
372
373
374
375

      int nDofs = buf[counter++];
      for (int j = 0; j < nDofs; j++) {
	DegreeOfFreedom dof0, dof1;
	dof0 = buf[counter++];
	dof1 = buf[counter++];
	dofs[dof0] = dof1;
      }

      data[bound] = dofs;
    }

    TEST_EXIT(bufSize == counter)("Should not happen!\n");
  }
376
377
378



379
  // T = vector<std::pair<int, int> >
380

381
  int StdMpiHelper<vector<std::pair<int, int> > >::getBufferSize(vector<std::pair<int, int> > &data)
382
383
384
385
  {
    return data.size() * 2;
  }

386
  void StdMpiHelper<vector<std::pair<int, int> > >::createBuffer(vector<std::pair<int, int> > &data, int *buf)
387
388
389
390
391
392
393
  {
    for (unsigned int i = 0; i < data.size(); i++) {
      buf[i * 2] = data[i].first;
      buf[i * 2 + 1] = data[i].second;
    }
  }

394
  void StdMpiHelper<vector<std::pair<int, int> > >::makeFromBuffer(vector<std::pair<int, int> > &data, 
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
									int *buf, int bufSize)
  {    
    if (bufSize == 0)
      return;

    TEST_EXIT(bufSize % 2 == 0)("This should not happen!\n");

    data.clear();
    data.reserve(bufSize / 2);

    for (int i = 0; i < (bufSize / 2); i++)
      data.push_back(std::make_pair(buf[i * 2], buf[i * 2 + 1]));
  }



411
  // T = vector<WorldVector<double> >
412

413
  int StdMpiHelper<vector<WorldVector<double> > >::getBufferSize(vector<WorldVector<double> > &data)
414
415
416
417
  {
    return data.size() * Global::getGeo(WORLD);
  }

418
  void StdMpiHelper<vector<WorldVector<double> > >::createBuffer(vector<WorldVector<double> > &data, double *buf)
419
420
421
422
423
424
425
426
  {
    int dimOfWorld = Global::getGeo(WORLD);
    int pos = 0;
    for (unsigned int i = 0; i < data.size(); i++)
      for (int j = 0; j < dimOfWorld; j++)
	buf[pos++] = data[i][j];
  }

427
  void StdMpiHelper<vector<WorldVector<double> > >::makeFromBuffer(vector<WorldVector<double> > &data, double *buf, int bufSize)
428
429
430
431
432
433
434
435
436
437
438
439
440
  {
    int dimOfWorld = Global::getGeo(WORLD);
    TEST_EXIT(bufSize % Global::getGeo(WORLD) == 0)("This should not happen!\n");

    int pos = 0;
    data.resize(bufSize / Global::getGeo(WORLD));
    for (unsigned int i = 0; i < data.size(); i++)
      for (int j = 0; j < dimOfWorld; j++)
	data[i][j] = buf[pos++];
  }



441
  // T = map<WorldVector<double>, int>
442

443
  int StdMpiHelper<map<WorldVector<double>, int> >::getBufferSize(map<WorldVector<double>, int> &data)
444
445
446
447
  {
    return data.size() * (Global::getGeo(WORLD) + 1);
  }

448
  void StdMpiHelper<map<WorldVector<double>, int> >::createBuffer(map<WorldVector<double>, int> &data, double* buf)
449
450
  {
    int i = 0;
451
    for (map<WorldVector<double>, int>::iterator it = data.begin();
452
453
454
455
456
457
458
	 it != data.end(); ++it) {
      for (int j = 0; j < Global::getGeo(WORLD); j++)
	buf[i++] = it->first[j];
      buf[i++] = static_cast<double>(it->second);      
    }
  }

459
  void StdMpiHelper<map<WorldVector<double>, int> >::makeFromBuffer(map<WorldVector<double>, int> &data, double* buf, int bufSize)
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
  {
    if (bufSize == 0)
      return;

    int oneEntrySize = Global::getGeo(WORLD) + 1;
    int nEntry = bufSize / oneEntrySize;

    TEST_EXIT(bufSize % oneEntrySize == 0)("This should not happen!\n");

    data.clear();
    int i = 0;
    WorldVector<double> coords;

    for (int j = 0; j < nEntry; j++) {
      for (int k = 0; k < Global::getGeo(WORLD); k++)
	coords[k] = buf[i++];
      data[coords] = static_cast<int>(buf[i++]);
    }
  }

480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
  // T = map<WorldVector<double>, int>

  int StdMpiHelper<map<int, WorldVector<double> > >::getBufferSize(map<int, WorldVector<double> > &data)
  {
    return data.size() * (Global::getGeo(WORLD) + 1);
  }

  void StdMpiHelper<map<int, WorldVector<double> > >::createBuffer(map<int, WorldVector<double> > &data, double* buf)
  {
    int i = 0;
    for (map<int, WorldVector<double> >::iterator it = data.begin();
	 it != data.end(); ++it) {
      for (int j = 0; j < Global::getGeo(WORLD); j++)
	buf[i++] = it->second[j];
      buf[i++] = static_cast<double>(it->first);      
    }
  }

  void StdMpiHelper<map<int, WorldVector<double> > >::makeFromBuffer(map<int, WorldVector<double> > &data, double* buf, int bufSize)
  {
    if (bufSize == 0)
      return;

    int oneEntrySize = Global::getGeo(WORLD) + 1;
    int nEntry = bufSize / oneEntrySize;

    TEST_EXIT(bufSize % oneEntrySize == 0)("This should not happen!\n");

    data.clear();
    int i = 0;
    WorldVector<double> coords;

    for (int j = 0; j < nEntry; j++) {
      for (int k = 0; k < Global::getGeo(WORLD); k++)
	coords[k] = buf[i++];
      data[static_cast<int>(buf[i++])] = coords;
    }
  }
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540

  // T = vector<vector<WorldVector<double> > >

  int StdMpiHelper<vector<vector<WorldVector<double> > > >::getBufferSize(vector<vector<WorldVector<double> > > &data)
  {
    int result = 1;

    for (unsigned int i = 0; i < data.size(); i++)
      result += 1 + (data[i].size() * Global::getGeo(WORLD));

    return result;
  }

  void StdMpiHelper<vector<vector<WorldVector<double> > > >::createBuffer(vector<vector<WorldVector<double> > > &data, 
									  double* buf)
  {
    int counter = 0;

    buf[counter++] = static_cast<double>(data.size());

    for (unsigned int i = 0; i < data.size(); i++) {
      buf[counter++] = data[i].size();
      for (unsigned int j = 0; j < data[i].size(); j++)
541
	for (int k = 0; k < Global::getGeo(WORLD); k++) 
542
543
544
545
546
547
548
549
550
551
552
553
554
	  buf[counter++] = data[i][j][k];
    }
  }

  void StdMpiHelper<vector<vector<WorldVector<double> > > >::makeFromBuffer(vector<vector<WorldVector<double> > > &data, 
									    double* buf, int bufSize)
  {
    int counter = 0;
    data.resize(static_cast<int>(buf[counter++]));

    for (unsigned int i = 0; i < data.size(); i++) {
      data[i].resize(static_cast<int>(buf[counter++]));
      for (unsigned int j = 0; j < data[i].size(); j++) 
555
	for (int k = 0; k < Global::getGeo(WORLD); k++) 
556
557
558
559
560
561
	  data[i][j][k] = buf[counter++];
    }

    TEST_EXIT_DBG(counter == bufSize)("There is something very wrong!\n");
  }

562
} }