ParallelDofMapping.cc 12 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
  {
    dofMap.clear();
23

24
    nonRankDofs.clear();
25

26
27
28
29
    nRankDofs = 0;
    nLocalDofs = 0;
    nOverallDofs = 0;
    rStartDofs = 0;
30
31
32
  }


33
  void FeSpaceDofMap::update()
34
  {
35
    FUNCNAME("FeSpaceDofMap::update()");
36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    // === Compute local indices for all rank owned DOFs. ===
    
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
    
    // === Compute number of local and global DOFs in the mapping. ===
    
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
    // === If required, compute also the global indices. ===
    
    if (needGlobalMapping) {
      computeGlobalMapping();
53
      
54
55
      if (hasNonLocalDofs)
	computeNonLocalIndices();
56
    }
57
58
59
  }


60
  void FeSpaceDofMap::computeGlobalMapping()
61
  {
62
63
    FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");

64
65
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      it->second.global = it->second.local + rStartDofs;
66
67
  }

68

69
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
70
  {
71
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
72

73
74
    TEST_EXIT_DBG(dofComm)("No DOF communicator defined!\n");

Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
75
76
    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

77
78
79
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

80
    StdMpi<vector<int> > stdMpi(mpiComm);
81

82
    for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpace); 
83
84
85
86
	 !it.end(); it.nextRank()) {
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);
87
      
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
88
      for (; !it.endDofIter(); it.nextDof())
89
90
	if (dofMap.count(it.getDofIndex()) && 
	    !nonRankDofs.count(it.getDofIndex()))
91
	  stdMpi.getSendData(rank).
92
	    push_back(dofMap[it.getDofIndex()].global);
93
    }
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
94
95
96

    stdMpi.updateSendDataSize();

97
98
99

    // === Check from which ranks this rank must receive some data. ===

100
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace); 
101
	 !it.end(); it.nextRank()) {
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
102
103
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
104
	if (nonRankDofs.count(it.getDofIndex())) {
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
105
106
107
108
109
	  recvFromRank = true;
	  break;
	}
      }

110
111
112
113
114
115
116
      if (recvFromRank) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
117
118
    }

119
120
121

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
122
123
    stdMpi.startCommunication();

124
125
126

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
127
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
128
	 !it.end(); it.nextRank()) {
129
130
131
132
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);

Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
133
134
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
135
136
	if (nonRankDofs.count(it.getDofIndex()))
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
137
138
139
140
    }
  }


141
  void ParallelDofMapping::init(MeshLevelData &ldata,
142
				vector<const FiniteElemSpace*> &fe,
143
				vector<const FiniteElemSpace*> &uniqueFe,
144
145
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
146
  {
147
148
    FUNCNAME("ParallelDofMapping::init()");

149
    levelData = &ldata;
150
    feSpaces = fe;
151
    feSpacesUnique = uniqueFe;
152
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
153
    
154
155
156

    // === Init the mapping for all different FE spaces. ===

157
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
158
159
160
161
162
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
163
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
164
165


166
167
168
169
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

170
171
    TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");

172
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
173
174
175
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

176
177
178
179
180
    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;
    dofToMatIndex.clear();
181
182
  }

183
184
185
186
187
188
189
190
191
192
193
  
  void ParallelDofMapping::setMpiComm(MPI::Intracomm &m, int l)
  {
    mpiComm = m;
    meshLevel = l;

    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].setMpiComm(m, l);
  }

194

195
  void ParallelDofMapping::setDofComm(DofComm &dc)
196
197
198
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

199
    dofComm = &dc;
200
201

    // Add the DOF communicator also to all FE space DOF mappings.
202
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
203
	 it != feSpacesUnique.end(); ++it)
204
      data[*it].setDofComm(dc);
205
  }
206

207

Thomas Witkowski's avatar
Thomas Witkowski committed
208
209
210
211
212
213
214
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
215
      data.insert(make_pair(feSpace, FeSpaceDofMap(levelData)));
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217
218
219
220
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


221
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
222
  {
223
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
224
225
    
    int result = 0;
226
227
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
228
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
229
230
231
232
233
234
    }
    
    return result;
  }


235
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
236
  {
237
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
238
239
    
    int result = 0;
240
241
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
242
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
243
244
245
246
247
248
    }
    
    return result;
  }


249
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
250
  {
251
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
252
253
254
255

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
256
      result += data.find(feSpaces[i])->second.nOverallDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
257
258
259
260
261
262
    }
    
    return result;
  }


263
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
264
  {
265
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
266
267
268
269

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
270
      result += data.find(feSpaces[i])->second.rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
271
272
273
274
275
276
277
278
279
280
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

281
    // First, update all FE space DOF mappings.
282
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
283
284
285
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
286
287
288
289
290
291
292
293
294
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
    
    // And finally, compute the matrix indices.
    if (needMatIndex)
      computeMatIndex(needMatIndexFromGlobal);
Thomas Witkowski's avatar
Thomas Witkowski committed
295
296
  }
  
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311

  void ParallelDofMapping::update(vector<const FiniteElemSpace*>& fe)
  {
    FUNCNAME("ParallelDofMapping::update()");

    for (vector<const FiniteElemSpace*>::iterator it = fe.begin();
	 it != fe.end(); ++it)
      if (find(feSpacesUnique.begin(), feSpacesUnique.end(), *it) == 
	  feSpacesUnique.end())
	ERROR_EXIT("Wrong FE space!\n");

    feSpaces = fe;
    update();
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
312
  
313
  void ParallelDofMapping::computeMatIndex(bool globalIndex)
Thomas Witkowski's avatar
Thomas Witkowski committed
314
315
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
316

317
    dofToMatIndex.clear();
Thomas Witkowski's avatar
Thomas Witkowski committed
318
    
319
320
321
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
322
    int offset = rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
323
    
324
325
326

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
327
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
328
329
330

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
331
      DofMap& dofMap = data[feSpaces[i]].getMap();
332
      for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
333
	if (data[feSpaces[i]].isRankDof(it->first)) {
Thomas Witkowski's avatar
Thomas Witkowski committed
334
	  int globalMatIndex = it->second.local + offset;
335
	  if (globalIndex)
336
	    dofToMatIndex.add(i, it->second.global, globalMatIndex);
337
	  else
338
	    dofToMatIndex.add(i, it->first, globalMatIndex);
Thomas Witkowski's avatar
Thomas Witkowski committed
339
340
341
	}
      }
      
342
343
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
344
      offset += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
345
	
346
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
347
348
349
      if (!hasNonLocalDofs)
	continue;
      
350
      TEST_EXIT_DBG(dofComm != NULL)("No communicator given!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
351
      
352
353
354
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

355
      StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
356
      for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpaces[i]); 
Thomas Witkowski's avatar
Thomas Witkowski committed
357
358
359
360
361
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
362
	    if (globalIndex)
363
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
364
	    else
365
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
Thomas Witkowski's avatar
Thomas Witkowski committed
366
	
367
368
369
370
371
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	stdMpi.send(rank, sendGlobalDofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
372
373
      }
      
374
      for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
375
376
377
378
379
380
381
	   !it.end(); it.nextRank()) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
382
383
384
      
      stdMpi.startCommunication();
      
385
386
387
388
389
390
391
392
393
394
395
396
397
398
      for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	int counter = 0;
	for (; !it.endDofIter(); it.nextDof()) {
	  if (dofMap.count(it.getDofIndex())) {
	    DegreeOfFreedom d = stdMpi.getRecvData(rank)[counter++];
	    if (globalIndex)
	      dofToMatIndex.add(i, dofMap[it.getDofIndex()].global, d);
	    else
	      dofToMatIndex.add(i, it.getDofIndex(), d);
Thomas Witkowski's avatar
Thomas Witkowski committed
399
400
401
	  }
	}
      }
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452


      // === Communicate DOFs on periodic boundaries. ===

      stdMpi.clear();

      for (DofComm::Iterator it(dofComm->getPeriodicDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex())) {
	    if (globalIndex) {
	      sendGlobalDofs.push_back(dofMap[it.getDofIndex()].global);
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
	    } else {
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
	    }
	  }
	
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	stdMpi.send(rank, sendGlobalDofs);
	stdMpi.recv(rank);
      }

      stdMpi.startCommunication();

      for (DofComm::Iterator it(dofComm->getPeriodicDofs(), 0, feSpaces[i]); 
	   !it.end(); it.nextRank()) {

	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	int counter = 0;

	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex())) {
	    if (globalIndex) {
	      dofToMatIndex.add(i, 
				stdMpi.getRecvData(it.getRank())[counter++],
				stdMpi.getRecvData(it.getRank())[counter++]);
	    } else {
	      dofToMatIndex.add(i, it.getDofIndex(),
				stdMpi.getRecvData(it.getRank())[counter++]);
	    }
	  }
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
453
454
455
    }
  }
  
456
}