ParallelDofMapping.cc 9.52 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
  {
    dofMap.clear();
23
    nonRankDofs.clear();
24
    nRankDofs = 0;
25
    nLocalDofs = 0;
26
27
28
29
30
    nOverallDofs = 0;
    rStartDofs = 0;
  }


31
  void FeSpaceDofMap::update()
32
  {
33
34
    // === Compute local indices for all rank owned DOFs. ===

35
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
Thomas Witkowski's avatar
Thomas Witkowski committed
36
37
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
38
39
40
        
    // === Compute number of local and global DOFs in the mapping. ===

41
42
    nOverallDofs = 0;
    rStartDofs = 0;
43
    mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
44
    
45
    // === If required, compute also the global indices. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
46

47
48
49
50
51
52
    if (needGlobalMapping) {
      computeGlobalMapping();
      
      if (hasNonLocalDofs)
	computeNonLocalIndices();
    }
53
54
55
  }


56
  void FeSpaceDofMap::computeGlobalMapping()
57
  {
58
    for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
59
      it->second.global = it->second.local + rStartDofs;
60
61
  }

62

63
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
64
  {
65
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
66
67
68

    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

69
70
71
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

72
    StdMpi<vector<int> > stdMpi(mpiComm);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
73
74
75
    for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
76
	  stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
77
78
79

    stdMpi.updateSendDataSize();

80
81
82

    // === Check from which ranks this rank must receive some data. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
83
84
85
86
87
88
89
90
91
92
93
94
95
    for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
	if (nonRankDofs.count(it.getDofIndex())) {
	  recvFromRank = true;
	  break;
	}
      }

      if (recvFromRank) 
	stdMpi.recv(it.getRank());
    }

96
97
98

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
99
100
    stdMpi.startCommunication();

101
102
103

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
104
105
106
107
108
    for (DofComm::Iterator it(*recvDofs, feSpace);
	 !it.end(); it.nextRank()) {
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
	if (nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
109
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
110
111
112
113
    }
  }


114
  void ParallelDofMapping::init(MPI::Intracomm m,
115
				vector<const FiniteElemSpace*> &fe,
116
				vector<const FiniteElemSpace*> &uniqueFe,
117
118
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
119
  {
120
121
122
123
    FUNCNAME("ParallelDofMapping::init()");

    mpiComm = m;
    feSpaces = fe;
124
    feSpacesUnique = uniqueFe;
125
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
126
    
127
128
129

    // === Init the mapping for all different FE spaces. ===

130
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
131
132
133
134
135
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
136
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
137
138


139
140
141
142
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

143
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
144
145
146
147
148
149
150
151
152
153
154
155
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;

    dofToMatIndex.clear();
  }


156
157
158
159
160
161
162
163
  void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

    sendDofs = &pSend;
    recvDofs = &pRecv;

    // Add the DOF communicator also to all FE space DOF mappings.
164
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
165
166
167
168
169
	 it != feSpacesUnique.end(); ++it)
      data[*it].setDofComm(pSend, pRecv);
  }
  

Thomas Witkowski's avatar
Thomas Witkowski committed
170
171
172
173
174
175
176
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
177
      data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
Thomas Witkowski's avatar
Thomas Witkowski committed
178
179
180
181
182
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


183
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
184
  {
185
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
186
187
    
    int result = 0;
188
189
190
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
191
192
193
194
195
196
    }
    
    return result;
  }


197
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
198
  {
199
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
200
201
    
    int result = 0;
202
203
204
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
205
206
207
208
209
210
    }
    
    return result;
  }


211
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
212
  {
213
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
214
215
216
217
218
219
220
221
222
223
224

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.nOverallDofs;
    }
    
    return result;
  }


225
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
226
  {
227
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.rStartDofs;
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

243
    // First, update all FE space DOF mappings.
244
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
245
246
247
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
248
249
250
251
252
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
Thomas Witkowski's avatar
Thomas Witkowski committed
253
    
254
    // And finally, compute the matrix indices.
255
256
    if (needMatIndex)
      computeMatIndex(needMatIndexFromGlobal);
Thomas Witkowski's avatar
Thomas Witkowski committed
257
258
  }
  
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273

  void ParallelDofMapping::update(vector<const FiniteElemSpace*>& fe)
  {
    FUNCNAME("ParallelDofMapping::update()");

    for (vector<const FiniteElemSpace*>::iterator it = fe.begin();
	 it != fe.end(); ++it)
      if (find(feSpacesUnique.begin(), feSpacesUnique.end(), *it) == 
	  feSpacesUnique.end())
	ERROR_EXIT("Wrong FE space!\n");

    feSpaces = fe;
    update();
  }

Thomas Witkowski's avatar
Thomas Witkowski committed
274
  
275
  void ParallelDofMapping::computeMatIndex(bool globalIndex)
Thomas Witkowski's avatar
Thomas Witkowski committed
276
277
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
278

Thomas Witkowski's avatar
Thomas Witkowski committed
279
280
    dofToMatIndex.clear();
    
281
282
283
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
284
285
    int offset = rStartDofs;
    
286
287
288

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
289
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
290
291
292

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
293
294
      DofMap& dofMap = data[feSpaces[i]].getMap();
      for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
Thomas Witkowski's avatar
Thomas Witkowski committed
295
296
	if (data[feSpaces[i]].isRankDof(it->first)) {
	  int globalMatIndex = it->second.local + offset;
297
298
299
300
	  if (globalIndex)
	    dofToMatIndex.add(i, it->second.global, globalMatIndex);
	  else
	    dofToMatIndex.add(i, it->first, globalMatIndex);
Thomas Witkowski's avatar
Thomas Witkowski committed
301
302
303
	}
      }
      
304
305
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
306
307
      offset += data[feSpaces[i]].nRankDofs;
	
308
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
309
310
311
312
313
314
      if (!hasNonLocalDofs)
	continue;
      
      TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
	("No communicator given!\n");
      
315
316
317
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

318
      StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
Thomas Witkowski's avatar
Thomas Witkowski committed
319
320
321
322
323
324
      for (DofComm::Iterator it(*sendDofs, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
325
326
327
328
	    if (globalIndex)
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
	    else
	      sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
Thomas Witkowski's avatar
Thomas Witkowski committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
	
	stdMpi.send(it.getRank(), sendGlobalDofs);
      }
      
      for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	   !it.end(); it.nextRank())
	stdMpi.recv(it.getRank());
      
      stdMpi.startCommunication();
      
      {
	for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	     !it.end(); it.nextRank()) {
	  int counter = 0;
	  for (; !it.endDofIter(); it.nextDof()) {
	    if (dofMap.count(it.getDofIndex())) {
	      DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++];
346
347
348
349
	      if (globalIndex)
		dofToMatIndex.add(i, dofMap[it.getDofIndex()].global, d);
	      else
		dofToMatIndex.add(i, it.getDofIndex(), d);
Thomas Witkowski's avatar
Thomas Witkowski committed
350
351
352
353
354
355
356
	    }
	  }
	}
      }
    }
  }
  
357
}