ParallelDofMapping.cc 12.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

  void DofToMatIndex::getReverse(int rowIndex, int &component, int &dofIndex)
  {
    FUNCNAME("DofToMatIndex::getReverse()");

    for (map<int, map<DegreeOfFreedom, int> >::iterator it0 = data.begin();
	 it0 != data.end(); ++it0)
      for (map<DegreeOfFreedom, int>::iterator it1 = it0->second.begin();
	   it1 != it0->second.end(); ++it1)
	if (it1->second == rowIndex) {
	  component = it0->first;
	  dofIndex = it1->first;
	  return;
	}
  
    component = -1;
    dofIndex = -1;
  }


40
  void FeSpaceDofMap::clear()
41
42
  {
    dofMap.clear();
43

44
    nonRankDofs.clear();
45

46
47
48
49
    nRankDofs = 0;
    nLocalDofs = 0;
    nOverallDofs = 0;
    rStartDofs = 0;
50
51
52
  }


53
  void FeSpaceDofMap::update()
54
  {
55
    FUNCNAME("FeSpaceDofMap::update()");
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    // === Compute local indices for all rank owned DOFs. ===
    
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
    
    // === Compute number of local and global DOFs in the mapping. ===
    
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
    // === If required, compute also the global indices. ===
    
    if (needGlobalMapping) {
      computeGlobalMapping();
73
      
74
75
      if (hasNonLocalDofs)
	computeNonLocalIndices();
76
    }
77
78
79
  }


80
  void FeSpaceDofMap::computeGlobalMapping()
81
  {
82
83
    FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");

84
85
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      it->second.global = it->second.local + rStartDofs;
86
87
  }

88

89
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
90
  {
91
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
92

93
94
    TEST_EXIT_DBG(dofComm)("No DOF communicator defined!\n");

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
95
96
    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

97
98
99
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

100
    StdMpi<vector<int> > stdMpi(mpiComm);
101

102
    for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpace); 
103
104
105
106
	 !it.end(); it.nextRank()) {
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);
107
      
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
108
      for (; !it.endDofIter(); it.nextDof())
109
110
	if (dofMap.count(it.getDofIndex()) && 
	    !nonRankDofs.count(it.getDofIndex()))
111
	  stdMpi.getSendData(rank).
112
	    push_back(dofMap[it.getDofIndex()].global);
113
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
114
115
116

    stdMpi.updateSendDataSize();

117
118
119

    // === Check from which ranks this rank must receive some data. ===

120
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace); 
121
	 !it.end(); it.nextRank()) {
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
122
123
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
124
	if (nonRankDofs.count(it.getDofIndex())) {
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
125
126
127
128
129
	  recvFromRank = true;
	  break;
	}
      }

130
131
132
133
134
135
136
      if (recvFromRank) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
137
138
    }

139
140
141

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
142
143
    stdMpi.startCommunication();

144
145
146

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
147
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
148
	 !it.end(); it.nextRank()) {
149
150
151
152
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
153
154
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
155
156
	if (nonRankDofs.count(it.getDofIndex()))
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
157
158
159
160
    }
  }


161
  void ParallelDofMapping::init(MeshLevelData &ldata,
162
				vector<const FiniteElemSpace*> &fe,
163
				vector<const FiniteElemSpace*> &uniqueFe,
164
165
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
166
  {
167
168
    FUNCNAME("ParallelDofMapping::init()");

169
    levelData = &ldata;
170
    feSpaces = fe;
171
    feSpacesUnique = uniqueFe;
172
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
173
    
174
175
176

    // === Init the mapping for all different FE spaces. ===

177
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
178
179
180
181
182
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
183
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
184
185


186
187
188
189
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

190
191
    TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");

192
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
193
194
195
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

196
197
198
199
200
    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;
    dofToMatIndex.clear();
201
202
  }

203
204
205
206
207
208
209
210
211
212
213
  
  void ParallelDofMapping::setMpiComm(MPI::Intracomm &m, int l)
  {
    mpiComm = m;
    meshLevel = l;

    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].setMpiComm(m, l);
  }

214

215
  void ParallelDofMapping::setDofComm(DofComm &dc)
216
217
218
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

219
    dofComm = &dc;
220
221

    // Add the DOF communicator also to all FE space DOF mappings.
222
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
223
	 it != feSpacesUnique.end(); ++it)
224
      data[*it].setDofComm(dc);
225
  }
226

227

Thomas Witkowski's avatar
Thomas Witkowski committed
228
229
230
231
232
233
234
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
235
      data.insert(make_pair(feSpace, FeSpaceDofMap(levelData)));
Thomas Witkowski's avatar
Thomas Witkowski committed
236
237
238
239
240
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


241
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
242
  {
243
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
244
245
    
    int result = 0;
246
247
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
248
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
249
250
251
252
253
254
    }
    
    return result;
  }


255
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
256
  {
257
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
258
259
    
    int result = 0;
260
261
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
262
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
263
264
265
266
267
268
    }
    
    return result;
  }


269
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
270
  {
271
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
272
273
274
275

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
276
      result += data.find(feSpaces[i])->second.nOverallDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
277
278
279
280
281
282
    }
    
    return result;
  }


283
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
284
  {
285
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
286
287
288
289

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
290
      result += data.find(feSpaces[i])->second.rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
291
292
293
294
295
296
297
298
299
300
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

301
    // First, update all FE space DOF mappings.
302
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
303
304
305
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
306
307
308
309
310
311
312
313
314
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
    
    // And finally, compute the matrix indices.
    if (needMatIndex)
      computeMatIndex(needMatIndexFromGlobal);
Thomas Witkowski's avatar
Thomas Witkowski committed
315
316
  }
  
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331

  void ParallelDofMapping::update(vector<const FiniteElemSpace*>& fe)
  {
    FUNCNAME("ParallelDofMapping::update()");

    for (vector<const FiniteElemSpace*>::iterator it = fe.begin();
	 it != fe.end(); ++it)
      if (find(feSpacesUnique.begin(), feSpacesUnique.end(), *it) == 
	  feSpacesUnique.end())
	ERROR_EXIT("Wrong FE space!\n");

    feSpaces = fe;
    update();
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
332
  
333
  void ParallelDofMapping::computeMatIndex(bool globalIndex)
Thomas Witkowski's avatar
Thomas Witkowski committed
334
335
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
336

337
    dofToMatIndex.clear();
Thomas Witkowski's avatar
Thomas Witkowski committed
338
    
339
340
341
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
342
    int offset = rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
343
    
344
345
346

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
347
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
348
349
350

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
351
      DofMap& dofMap = data[feSpaces[i]].getMap();
352
      for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
353
	if (data[feSpaces[i]].isRankDof(it->first)) {
Thomas Witkowski's avatar
Thomas Witkowski committed
354
	  int globalMatIndex = it->second.local + offset;
355
	  if (globalIndex)
356
	    dofToMatIndex.add(i, it->second.global, globalMatIndex);
357
	  else
358
	    dofToMatIndex.add(i, it->first, globalMatIndex);
Thomas Witkowski's avatar
Thomas Witkowski committed
359
360
361
	}
      }
      
362
363
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
364
      offset += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
365
	
366
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
367
368
369
      if (!hasNonLocalDofs)
	continue;
      
370
      TEST_EXIT_DBG(dofComm != NULL)("No communicator given!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
371
      
372
373
374
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

375
      StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
376
      for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpaces[i]); 
Thomas Witkowski's avatar
Thomas Witkowski committed
377
378
379
380
381
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
382
	    if (globalIndex)
383
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
384
	    else
385
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
Thomas Witkowski's avatar
Thomas Witkowski committed
386
	
387
388
389
390
391
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	stdMpi.send(rank, sendGlobalDofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
392
393
      }
      
394
      for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
395
396
397
398
399
400
401
	   !it.end(); it.nextRank()) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
402
403
404
      
      stdMpi.startCommunication();
      
405
406
407
408
409
410
411
412
413
414
415
416
417
418
      for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	int counter = 0;
	for (; !it.endDofIter(); it.nextDof()) {
	  if (dofMap.count(it.getDofIndex())) {
	    DegreeOfFreedom d = stdMpi.getRecvData(rank)[counter++];
	    if (globalIndex)
	      dofToMatIndex.add(i, dofMap[it.getDofIndex()].global, d);
	    else
	      dofToMatIndex.add(i, it.getDofIndex(), d);
Thomas Witkowski's avatar
Thomas Witkowski committed
419
420
421
	  }
	}
      }
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472


      // === Communicate DOFs on periodic boundaries. ===

      stdMpi.clear();

      for (DofComm::Iterator it(dofComm->getPeriodicDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex())) {
	    if (globalIndex) {
	      sendGlobalDofs.push_back(dofMap[it.getDofIndex()].global);
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
	    } else {
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
	    }
	  }
	
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	stdMpi.send(rank, sendGlobalDofs);
	stdMpi.recv(rank);
      }

      stdMpi.startCommunication();

      for (DofComm::Iterator it(dofComm->getPeriodicDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {

	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	int counter = 0;

	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex())) {
	    if (globalIndex) {
	      dofToMatIndex.add(i, 
				stdMpi.getRecvData(it.getRank())[counter++],
				stdMpi.getRecvData(it.getRank())[counter++]);
	    } else {
	      dofToMatIndex.add(i, it.getDofIndex(),
				stdMpi.getRecvData(it.getRank())[counter++]);
	    }
	  }
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
473
474
475
    }
  }
  
476
}