ParallelDofMapping.cc 9.01 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
  {
    dofMap.clear();
23
    nonRankDofs.clear();
24
    nRankDofs = 0;
25
    nLocalDofs = 0;
26
27
28
29
30
    nOverallDofs = 0;
    rStartDofs = 0;
  }


31
  void FeSpaceDofMap::update()
32
  {
33
34
    // === Compute local indices for all rank owned DOFs. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
35
36
37
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
38
39
40
        
    // === Compute number of local and global DOFs in the mapping. ===

41
42
43
44
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(*mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
45
    // === If required, compute also the global indices. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
46

47
48
49
50
51
52
    if (needGlobalMapping) {
      computeGlobalMapping();
      
      if (hasNonLocalDofs)
	computeNonLocalIndices();
    }
53
54
55
  }


56
  void FeSpaceDofMap::computeGlobalMapping()
57
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
58
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
59
      it->second.global = it->second.local + rStartDofs;
60
61
  }

62

63
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
64
  {
65
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
66
67
68

    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

69
70
71
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
72
73
74
75
    StdMpi<vector<int> > stdMpi(*mpiComm);
    for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
76
	  stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
77
78
79

    stdMpi.updateSendDataSize();

80
81
82

    // === Check from which ranks this rank must receive some data. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
83
84
85
86
87
88
89
90
91
92
93
94
95
    for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
	if (nonRankDofs.count(it.getDofIndex())) {
	  recvFromRank = true;
	  break;
	}
      }

      if (recvFromRank) 
	stdMpi.recv(it.getRank());
    }

96
97
98

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
99
100
    stdMpi.startCommunication();

101
102
103

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
104
105
106
107
108
    for (DofComm::Iterator it(*recvDofs, feSpace);
	 !it.end(); it.nextRank()) {
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
	if (nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
109
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
110
111
112
113
    }
  }


114
115
116
117
  void ParallelDofMapping::init(MPI::Intracomm *m,
				vector<const FiniteElemSpace*> &fe,
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
118
  {
119
120
121
122
123
    FUNCNAME("ParallelDofMapping::init()");

    mpiComm = m;
    feSpaces = fe;
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
124
    
125
    // === Create a set of unique FE spaces. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
126

127
128
129
130
131
132
133
134
135
136
137
138
    for (unsigned int i = 0; i < feSpaces.size(); i++)
      feSpacesUnique.insert(feSpaces[i]);


    // === Init the mapping for all different FE spaces. ===

    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
139
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
140
141


142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  void ParallelDofMapping::clear()
  {
    FUNCNAME("ParallelDofMapping::clear()");

    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].clear();

    nRankDofs = -1;
    nLocalDofs = -1;
    nOverallDofs = -1;
    rStartDofs = -1;

    dofToMatIndex.clear();
  }


159
160
161
162
163
164
165
166
167
168
169
170
171
172
  void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

    sendDofs = &pSend;
    recvDofs = &pRecv;

    // Add the DOF communicator also to all FE space DOF mappings.
    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].setDofComm(pSend, pRecv);
  }
  

Thomas Witkowski's avatar
Thomas Witkowski committed
173
174
175
176
177
178
179
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
180
      data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
Thomas Witkowski's avatar
Thomas Witkowski committed
181
182
183
184
185
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


186
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
187
  {
188
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
189
190
    
    int result = 0;
191
192
193
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
194
195
196
197
198
199
    }
    
    return result;
  }


200
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
201
  {
202
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
203
204
    
    int result = 0;
205
206
207
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
208
209
210
211
212
213
    }
    
    return result;
  }


214
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
215
  {
216
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
217
218
219
220
221
222
223
224
225
226
227

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.nOverallDofs;
    }
    
    return result;
  }


228
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
229
  {
230
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.rStartDofs;
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

246
    // First, update all FE space DOF mappings.
Thomas Witkowski's avatar
Thomas Witkowski committed
247
248
249
250
    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
251
252
253
254
255
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
Thomas Witkowski's avatar
Thomas Witkowski committed
256
    
257
    // And finally, compute the matrix indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
258
259
260
261
262
263
264
265
266
267
    computeMatIndex();
  }
  
  
  void ParallelDofMapping::computeMatIndex()
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
    
    dofToMatIndex.clear();
    
268
269
270
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
271
272
    int offset = rStartDofs;
    
273
274
275

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
276
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
277
278
279

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
Thomas Witkowski's avatar
Thomas Witkowski committed
280
281
282
283
284
285
286
287
288
      map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap();
      typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType;
      for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) {
	if (data[feSpaces[i]].isRankDof(it->first)) {
	  int globalMatIndex = it->second.local + offset;
	  dofToMatIndex.add(i, it->first, globalMatIndex);
	}
      }
      
289
290
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
291
292
      offset += data[feSpaces[i]].nRankDofs;
	
293
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
294
295
296
297
298
299
      if (!hasNonLocalDofs)
	continue;
      
      TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
	("No communicator given!\n");
      
300
301
302
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

Thomas Witkowski's avatar
Thomas Witkowski committed
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
      StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
      for (DofComm::Iterator it(*sendDofs, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
	    sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
	
	stdMpi.send(it.getRank(), sendGlobalDofs);
      }
      
      for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	   !it.end(); it.nextRank())
	stdMpi.recv(it.getRank());
      
      stdMpi.startCommunication();
      
      {
	for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	     !it.end(); it.nextRank()) {
	  int counter = 0;
	  for (; !it.endDofIter(); it.nextDof()) {
	    if (dofMap.count(it.getDofIndex())) {
	      DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++];
	      dofToMatIndex.add(i, it.getDofIndex(), d);
	    }
	  }
	}
      }
    }
  }
  
336
}