ParallelDofMapping.cc 8.63 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


13
#include "parallel/ParallelDofMapping.h"
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
14
#include "parallel/StdMpi.h"
15
16
17
18
19

namespace AMDiS {

  using namespace std;

20
  void FeSpaceDofMap::clear()
21
22
23
24
25
26
27
28
  {
    dofMap.clear();
    nRankDofs = 0;
    nOverallDofs = 0;
    rStartDofs = 0;
  }


29
  void FeSpaceDofMap::update()
30
  {
31
32
    // === Compute local indices for all rank owned DOFs. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
33
34
35
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
      if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
	it->second.local = nRankDofs++;
36
37
38
        
    // === Compute number of local and global DOFs in the mapping. ===

39
40
41
42
    nOverallDofs = 0;
    rStartDofs = 0;
    mpi::getDofNumbering(*mpiComm, nRankDofs, rStartDofs, nOverallDofs);
    
43
    // === If required, compute also the global indices. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
44

45
46
47
48
49
50
    if (needGlobalMapping) {
      computeGlobalMapping();
      
      if (hasNonLocalDofs)
	computeNonLocalIndices();
    }
51
52
53
  }


54
  void FeSpaceDofMap::computeGlobalMapping()
55
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
56
    for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
57
      it->second.global = it->second.local + rStartDofs;
58
59
  }

60

61
  void FeSpaceDofMap::computeNonLocalIndices()
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
62
  {
63
    FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
64
65
66

    typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;

67
68
69
    // === Send all global indices of DOFs that are owned by the rank to all ===
    // === other ranks that also include this DOF.                           ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
70
71
72
73
    StdMpi<vector<int> > stdMpi(*mpiComm);
    for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
      for (; !it.endDofIter(); it.nextDof())
	if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
74
	  stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
75
76
77

    stdMpi.updateSendDataSize();

78
79
80

    // === Check from which ranks this rank must receive some data. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
81
82
83
84
85
86
87
88
89
90
91
92
93
    for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
      bool recvFromRank = false;
      for (; !it.endDofIter(); it.nextDof()) {
	if (nonRankDofs.count(it.getDofIndex())) {
	  recvFromRank = true;
	  break;
	}
      }

      if (recvFromRank) 
	stdMpi.recv(it.getRank());
    }

94
95
96

    // === Start communication to exchange global indices. ===

Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
97
98
    stdMpi.startCommunication();

99
100
101

    // === And set the global indices for all DOFs that are not owned by rank. ===
    
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
102
103
104
105
106
    for (DofComm::Iterator it(*recvDofs, feSpace);
	 !it.end(); it.nextRank()) {
      int i = 0;
      for (; !it.endDofIter(); it.nextDof())
	if (nonRankDofs.count(it.getDofIndex()))
Thomas Witkowski's avatar
Thomas Witkowski committed
107
	  dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
108
109
110
111
    }
  }


112
113
114
115
  void ParallelDofMapping::init(MPI::Intracomm *m,
				vector<const FiniteElemSpace*> &fe,
				bool needGlobalMapping,
				bool bNonLocalDofs)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
116
  {
117
118
119
120
121
    FUNCNAME("ParallelDofMapping::init()");

    mpiComm = m;
    feSpaces = fe;
    hasNonLocalDofs = bNonLocalDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
122
    
123
    // === Create a set of unique FE spaces. ===
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
124

125
126
127
128
129
130
131
132
133
134
135
136
    for (unsigned int i = 0; i < feSpaces.size(); i++)
      feSpacesUnique.insert(feSpaces[i]);


    // === Init the mapping for all different FE spaces. ===

    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it) {
      addFeSpace(*it);
      data[*it].setNeedGlobalMapping(needGlobalMapping);
      data[*it].setNonLocalDofs(hasNonLocalDofs);
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
137
  }
Thomas Witkowski's avatar
Thomas Witkowski committed
138
139


140
141
142
143
144
145
146
147
148
149
150
151
152
153
  void ParallelDofMapping::setDofComm(DofComm &pSend, DofComm &pRecv)
  {
    FUNCNAME("ParallelDofMapping::setDofComm()");

    sendDofs = &pSend;
    recvDofs = &pRecv;

    // Add the DOF communicator also to all FE space DOF mappings.
    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].setDofComm(pSend, pRecv);
  }
  

Thomas Witkowski's avatar
Thomas Witkowski committed
154
155
156
157
158
159
160
  void ParallelDofMapping::addFeSpace(const FiniteElemSpace* feSpace)
  {
    FUNCNAME("ParallelDofMapping::addFeSpace()");
    
    if (data.count(feSpace))
      data.find(feSpace)->second.clear();
    else
161
      data.insert(make_pair(feSpace, FeSpaceDofMap(mpiComm)));
Thomas Witkowski's avatar
Thomas Witkowski committed
162
163
164
165
166
    
    data.find(feSpace)->second.setFeSpace(feSpace);
  }    


167
  int ParallelDofMapping::computeRankDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
168
  {
169
    FUNCNAME("ParallelDofMapping::computeRankDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
170
171
    
    int result = 0;
172
173
174
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nRankDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
175
176
177
178
179
180
    }
    
    return result;
  }


181
  int ParallelDofMapping::computeLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
182
  {
183
    FUNCNAME("ParallelDofMapping::computeLocalDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
184
185
    
    int result = 0;
186
187
188
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data[feSpaces[i]].nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
189
190
191
192
193
194
    }
    
    return result;
  }


195
  int ParallelDofMapping::computeOverallDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
196
  {
197
    FUNCNAME("ParallelDofMapping::computeOverallDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
198
199
200
201
202
203
204
205
206
207
208

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.nOverallDofs;
    }
    
    return result;
  }


209
  int ParallelDofMapping::computeStartDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
210
  {
211
    FUNCNAME("ParallelDofMapping::computeStartDofs()");
Thomas Witkowski's avatar
Thomas Witkowski committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226

    int result = 0;
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
      TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
      result += data.find(feSpaces[i])->second.rStartDofs;
    }
    
    return result;
  }


  void ParallelDofMapping::update()
  {
    FUNCNAME("ParallelDofMapping::update()");

227
    // First, update all FE space DOF mappings.
Thomas Witkowski's avatar
Thomas Witkowski committed
228
229
230
231
    for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
	 it != feSpacesUnique.end(); ++it)
      data[*it].update();
    
232
233
234
235
236
    // Compute all numbers from this mappings.
    nRankDofs = computeRankDofs();
    nLocalDofs = computeLocalDofs();
    nOverallDofs = computeOverallDofs();
    rStartDofs = computeStartDofs();
Thomas Witkowski's avatar
Thomas Witkowski committed
237
    
238
    // And finally, compute the matrix indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
239
240
241
242
243
244
245
246
247
248
    computeMatIndex();
  }
  
  
  void ParallelDofMapping::computeMatIndex()
  {
    FUNCNAME("ParallelDofMapping::computeMatIndex()");
    
    dofToMatIndex.clear();
    
249
250
251
    // The offset is always added to the local matrix index. The offset for the
    // DOFs in the first FE spaces is the smalled global index of a DOF that is
    // owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
252
253
    int offset = rStartDofs;
    
254
255
256

    // === Create the matrix indices for all component FE spaces. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
257
    for (unsigned int i = 0; i < feSpaces.size(); i++) {
258
259
260

      // Traverse all DOFs of the FE space and create for all rank owned DOFs
      // a matrix index.
Thomas Witkowski's avatar
Thomas Witkowski committed
261
262
263
264
265
266
267
268
269
      map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap();
      typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType;
      for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) {
	if (data[feSpaces[i]].isRankDof(it->first)) {
	  int globalMatIndex = it->second.local + offset;
	  dofToMatIndex.add(i, it->first, globalMatIndex);
	}
      }
      
270
271
      // Increase the offset for the next FE space by the number of DOFs owned 
      // by the rank in the current FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
272
273
      offset += data[feSpaces[i]].nRankDofs;
	
274
      // If there are no non local DOFs, continue with the next FE space.
Thomas Witkowski's avatar
Thomas Witkowski committed
275
276
277
278
279
280
      if (!hasNonLocalDofs)
	continue;
      
      TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
	("No communicator given!\n");
      
281
282
283
      // === Communicate the matrix indices for all DOFs that are on some ===
      // === interior boundaries.                                         ===

Thomas Witkowski's avatar
Thomas Witkowski committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
      StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
      for (DofComm::Iterator it(*sendDofs, feSpaces[i]); 
	   !it.end(); it.nextRank()) {
	vector<DegreeOfFreedom> sendGlobalDofs;
	
	for (; !it.endDofIter(); it.nextDof())
	  if (dofMap.count(it.getDofIndex()))
	    sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
	
	stdMpi.send(it.getRank(), sendGlobalDofs);
      }
      
      for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	   !it.end(); it.nextRank())
	stdMpi.recv(it.getRank());
      
      stdMpi.startCommunication();
      
      {
	for (DofComm::Iterator it(*recvDofs, feSpaces[i]); 
	     !it.end(); it.nextRank()) {
	  int counter = 0;
	  for (; !it.endDofIter(); it.nextDof()) {
	    if (dofMap.count(it.getDofIndex())) {
	      DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++];
	      dofToMatIndex.add(i, it.getDofIndex(), d);
	    }
	  }
	}
      }
    }
  }
  
317
}