ParallelDofMapping.cc 10.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
  {
    dofMap.clear();
23

24
    nonRankDofs.clear();
25

26
27
28
29
    nRankDofs = 0;
    nLocalDofs = 0;
    nOverallDofs = 0;
    rStartDofs = 0;
30
31
32
  }


33
  void FeSpaceDofMap::update()
34
  {
35
    FUNCNAME("FeSpaceDofMap::update()");
36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    // === Compute local indices for all rank owned DOFs. ===
    
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
    
    // === Compute number of local and global DOFs in the mapping. ===
    
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
    // === If required, compute also the global indices. ===
    
    if (needGlobalMapping) {
      computeGlobalMapping();
53
      
54
55
      if (hasNonLocalDofs)
	computeNonLocalIndices();
56
    }
57
58
59
  }


60
  void FeSpaceDofMap::computeGlobalMapping()
61
  {
62
63
    FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");

64
65
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      it->second.global = it->second.local + rStartDofs;
66
67
  }

68

69
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
70
  {
71
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
72

73
74
    TEST_EXIT_DBG(dofComm)("No DOF communicator defined!\n");

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
75
76
    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

77
78
79
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

80
    StdMpi<vector<int> > stdMpi(mpiComm);
81

82
    for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpace); 
83
84
85
86
	 !it.end(); it.nextRank()) {
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);
87
      
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
88
      for (; !it.endDofIter(); it.nextDof())
89
90
	if (dofMap.count(it.getDofIndex()) && 
	    !nonRankDofs.count(it.getDofIndex()))
91
	  stdMpi.getSendData(rank).
92
	    push_back(dofMap[it.getDofIndex()].global);
93
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
94
95
96

    stdMpi.updateSendDataSize();

97
98
99

    // === Check from which ranks this rank must receive some data. ===

100
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace); 
101
	 !it.end(); it.nextRank()) {
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
102
103
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
104
	if (nonRankDofs.count(it.getDofIndex())) {
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
105
106
107
108
109
	  recvFromRank = true;
	  break;
	}
      }

110
111
112
113
114
115
116
      if (recvFromRank) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
117
118
    }

119
120
121

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
122
123
    stdMpi.startCommunication();

124
125
126

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
127
    for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
128
	 !it.end(); it.nextRank()) {
129
130
131
132
      int rank = it.getRank();
      if (meshLevel > 0)
	rank = levelData->mapRank(rank, 0, meshLevel);

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
133
134
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
135
136
	if (nonRankDofs.count(it.getDofIndex()))
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
137
138
139
140
    }
  }


141
  void ParallelDofMapping::init(MeshLevelData &ldata,
142
				vector<const FiniteElemSpace*> &fe,
143
				vector<const FiniteElemSpace*> &uniqueFe,
144
145
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
146
  {
147
148
    FUNCNAME("ParallelDofMapping::init()");

149
    levelData = &ldata;
150
    feSpaces = fe;
151
    feSpacesUnique = uniqueFe;
152
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
153
    
154
155
156

    // === Init the mapping for all different FE spaces. ===

157
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
158
159
160
161
162
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
163
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
164
165


166
167
168
169
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

170
171
    TEST_EXIT_DBG(levelData)("No mesh level data object defined!\n");

172
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
173
174
175
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

176
177
178
179
180
    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;
    dofToMatIndex.clear();
181
182
  }

183
184
185
186
187
188
189
190
191
192
193
  
  void ParallelDofMapping::setMpiComm(MPI::Intracomm &m, int l)
  {
    mpiComm = m;
    meshLevel = l;

    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].setMpiComm(m, l);
  }

194

195
  void ParallelDofMapping::setDofComm(DofComm &dc)
196
197
198
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

199
    dofComm = &dc;
200
201

    // Add the DOF communicator also to all FE space DOF mappings.
202
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
203
	 it != feSpacesUnique.end(); ++it)
204
      data[*it].setDofComm(dc);
205
  }
206

207

Thomas Witkowski's avatar
Thomas Witkowski committed
208
209
210
211
212
213
214
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
215
      data.insert(make_pair(feSpace, FeSpaceDofMap(levelData)));
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217
218
219
220
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


221
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
222
  {
223
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
224
225
    
    int result = 0;
226
227
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
228
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
229
230
231
232
233
234
    }
    
    return result;
  }


235
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
236
  {
237
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
238
239
    
    int result = 0;
240
241
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
242
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
243
244
245
246
247
248
    }
    
    return result;
  }


249
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
250
  {
251
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
252
253
254
255

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
256
      result += data.find(feSpaces[i])->second.nOverallDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
257
258
259
260
261
262
    }
    
    return result;
  }


263
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
264
  {
265
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
266
267
268
269

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
270
      result += data.find(feSpaces[i])->second.rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
271
272
273
274
275
276
277
278
279
280
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

281
    // First, update all FE space DOF mappings.
282
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
283
284
285
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
286
287
288
289
290
291
292
293
294
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
    
    // And finally, compute the matrix indices.
    if (needMatIndex)
      computeMatIndex(needMatIndexFromGlobal);
Thomas Witkowski's avatar
Thomas Witkowski committed
295
296
  }
  
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311

  void ParallelDofMapping::update(vector<const FiniteElemSpace*>& fe)
  {
    FUNCNAME("ParallelDofMapping::update()");

    for (vector<const FiniteElemSpace*>::iterator it = fe.begin();
	 it != fe.end(); ++it)
      if (find(feSpacesUnique.begin(), feSpacesUnique.end(), *it) == 
	  feSpacesUnique.end())
	ERROR_EXIT("Wrong FE space!\n");

    feSpaces = fe;
    update();
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
312
  
313
  void ParallelDofMapping::computeMatIndex(bool globalIndex)
Thomas Witkowski's avatar
Thomas Witkowski committed
314
315
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
316

317
    dofToMatIndex.clear();
Thomas Witkowski's avatar
Thomas Witkowski committed
318
    
319
320
321
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
322
    int offset = rStartDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
323
    
324
325
326

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
327
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
328
329
330

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
331
      DofMap& dofMap = data[feSpaces[i]].getMap();
332
      for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
333
	if (data[feSpaces[i]].isRankDof(it->first)) {
Thomas Witkowski's avatar
Thomas Witkowski committed
334
	  int globalMatIndex = it->second.local + offset;
335
	  if (globalIndex)
336
	    dofToMatIndex.add(i, it->second.global, globalMatIndex);
337
	  else
338
	    dofToMatIndex.add(i, it->first, globalMatIndex);
Thomas Witkowski's avatar
Thomas Witkowski committed
339
340
341
	}
      }
      
342
343
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
344
      offset += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
345
	
346
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
347
348
349
      if (!hasNonLocalDofs)
	continue;
      
350
      TEST_EXIT_DBG(dofComm != NULL)("No communicator given!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
351
      
352
353
354
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

355
      StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
356
      for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpaces[i]); 
Thomas Witkowski's avatar
Thomas Witkowski committed
357
358
359
360
361
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
362
	    if (globalIndex)
363
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
364
	    else
365
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
Thomas Witkowski's avatar
Thomas Witkowski committed
366
	
367
368
369
370
371
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);
	
	stdMpi.send(rank, sendGlobalDofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
372
373
      }
      
374
      for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
375
376
377
378
379
380
381
	   !it.end(); it.nextRank()) {
	int rank = it.getRank();
	if (meshLevel > 0)
	  rank = levelData->mapRank(rank, 0, meshLevel);

	stdMpi.recv(rank);
      }
Thomas Witkowski's avatar
Thomas Witkowski committed
382
383
384
385
      
      stdMpi.startCommunication();
      
      {
386
	for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]); 
Thomas Witkowski's avatar
Thomas Witkowski committed
387
	     !it.end(); it.nextRank()) {
388
389
390
391
	  int rank = it.getRank();
	  if (meshLevel > 0)
	    rank = levelData->mapRank(rank, 0, meshLevel);
	  
Thomas Witkowski's avatar
Thomas Witkowski committed
392
393
394
	  int counter = 0;
	  for (; !it.endDofIter(); it.nextDof()) {
	    if (dofMap.count(it.getDofIndex())) {
395
	      DegreeOfFreedom d = stdMpi.getRecvData(rank)[counter++];
396
	      if (globalIndex)
397
		dofToMatIndex.add(i, dofMap[it.getDofIndex()].global, d);
398
	      else
399
		dofToMatIndex.add(i, it.getDofIndex(), d);
Thomas Witkowski's avatar
Thomas Witkowski committed
400
401
402
403
404
405
406
	    }
	  }
	}
      }
    }
  }
  
407
}