ParallelDofMapping.cc 8.93 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
  {
    dofMap.clear();
23
    nonRankDofs.clear();
24
    nRankDofs = 0;
25
    nLocalDofs = 0;
26
27
28
29
30
    nOverallDofs = 0;
    rStartDofs = 0;
  }


31
  void FeSpaceDofMap::update()
32
  {
33
34
    // === Compute local indices for all rank owned DOFs. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
35
36
37
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
38
39
40
        
    // === Compute number of local and global DOFs in the mapping. ===

41
42
43
44
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(*mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
45
    // === If required, compute also the global indices. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
46

47
48
49
50
51
52
    if (needGlobalMapping) {
      computeGlobalMapping();
      
      if (hasNonLocalDofs)
	computeNonLocalIndices();
    }
53
54
55
  }


56
  void FeSpaceDofMap::computeGlobalMapping()
57
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
58
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
59
      it->second.global = it->second.local + rStartDofs;
60
61
  }

62

63
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
64
  {
65
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
66
67
68

    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

69
70
71
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
72
73
74
75
    StdMpi<vector<int> > stdMpi(*mpiComm);
    for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
76
	  stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
77
78
79

    stdMpi.updateSendDataSize();

80
81
82

    // === Check from which ranks this rank must receive some data. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
83
84
85
86
87
88
89
90
91
92
93
94
95
    for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
	if (nonRankDofs.count(it.getDofIndex())) {
	  recvFromRank = true;
	  break;
	}
      }

      if (recvFromRank) 
	stdMpi.recv(it.getRank());
    }

96
97
98

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
99
100
    stdMpi.startCommunication();

101
102
103

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
104
105
106
107
108
    for (DofComm::Iterator it(*recvDofs, feSpace);
	 !it.end(); it.nextRank()) {
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
	if (nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
109
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
110
111
112
113
    }
  }


114
115
  void ParallelDofMapping::init(MPI::Intracomm *m,
				vector<const FiniteElemSpace*> &fe,
116
				vector<const FiniteElemSpace*> &uniqueFe,
117
118
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
119
  {
120
121
122
123
    FUNCNAME("ParallelDofMapping::init()");

    mpiComm = m;
    feSpaces = fe;
124
    feSpacesUnique = uniqueFe;
125
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
126
    
127
128
129

    // === Init the mapping for all different FE spaces. ===

130
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
131
132
133
134
135
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
136
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
137
138


139
140
141
142
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

143
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
144
145
146
147
148
149
150
151
152
153
154
155
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;

    dofToMatIndex.clear();
  }


156
157
158
159
160
161
162
163
  void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

    sendDofs = &pSend;
    recvDofs = &pRecv;

    // Add the DOF communicator also to all FE space DOF mappings.
164
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
165
166
167
168
169
	 it != feSpacesUnique.end(); ++it)
      data[*it].setDofComm(pSend, pRecv);
  }
  

Thomas Witkowski's avatar
Thomas Witkowski committed
170
171
172
173
174
175
176
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
177
      data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
Thomas Witkowski's avatar
Thomas Witkowski committed
178
179
180
181
182
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


183
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
184
  {
185
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
186
187
    
    int result = 0;
188
189
190
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
191
192
193
194
195
196
    }
    
    return result;
  }


197
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
198
  {
199
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
200
201
    
    int result = 0;
202
203
204
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
205
206
207
208
209
210
    }
    
    return result;
  }


211
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
212
  {
213
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
214
215
216
217
218
219
220
221
222
223
224

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.nOverallDofs;
    }
    
    return result;
  }


225
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
226
  {
227
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.rStartDofs;
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

243
    // First, update all FE space DOF mappings.
244
    for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
245
246
247
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
248
249
250
251
252
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
Thomas Witkowski's avatar
Thomas Witkowski committed
253
    
254
    // And finally, compute the matrix indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
255
256
257
258
259
260
261
262
263
264
    computeMatIndex();
  }
  
  
  void ParallelDofMapping::computeMatIndex()
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
    
    dofToMatIndex.clear();
    
265
266
267
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
268
269
    int offset = rStartDofs;
    
270
271
272

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
273
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
274
275
276

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
Thomas Witkowski's avatar
Thomas Witkowski committed
277
278
279
280
281
282
283
284
285
      map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap();
      typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType;
      for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) {
	if (data[feSpaces[i]].isRankDof(it->first)) {
	  int globalMatIndex = it->second.local + offset;
	  dofToMatIndex.add(i, it->first, globalMatIndex);
	}
      }
      
286
287
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
288
289
      offset += data[feSpaces[i]].nRankDofs;
	
290
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
291
292
293
294
295
296
      if (!hasNonLocalDofs)
	continue;
      
      TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
	("No communicator given!\n");
      
297
298
299
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

Thomas Witkowski's avatar
Thomas Witkowski committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
      StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
      for (DofComm::Iterator it(*sendDofs, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
	    sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
	
	stdMpi.send(it.getRank(), sendGlobalDofs);
      }
      
      for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	   !it.end(); it.nextRank())
	stdMpi.recv(it.getRank());
      
      stdMpi.startCommunication();
      
      {
	for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	     !it.end(); it.nextRank()) {
	  int counter = 0;
	  for (; !it.endDofIter(); it.nextDof()) {
	    if (dofMap.count(it.getDofIndex())) {
	      DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++];
	      dofToMatIndex.add(i, it.getDofIndex(), d);
	    }
	  }
	}
      }
    }
  }
  
333
}