ParallelDofMapping.h 12.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
35

36
37
  using namespace std;

38
39
  /// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
  /// and a global one.
Thomas Witkowski's avatar
Thomas Witkowski committed
40
41
42
43
44
  struct MultiIndex
  {
    int local, global;
  };

45

46
  /** \brief
47
48
49
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
50
51
52
53
54
55
56
57
58
59
60
61
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

62
63
64
65
66
67
68
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
69
70
71
72
73
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

74
    /// Maps a global DOF index to the global matrix index for a specific 
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


97
98
99
100
101
102
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
103
104
  {
  public:
105
106
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
107
    FeSpaceDofMap() 
108
109
110
    {
      ERROR_EXIT("Should not be called!\n");
    }
111
112
     
    /// This is the only valid constructur to be used. 
113
    FeSpaceDofMap(MPI::Intracomm m)
114
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
115
116
	sendDofs(NULL),
	recvDofs(NULL),
117
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
118
	needGlobalMapping(false),
119
	hasNonLocalDofs(false),
120
	nRankDofs(0),
Thomas Witkowski's avatar
Thomas Witkowski committed
121
	nLocalDofs(0),
122
	nOverallDofs(0),
123
	rStartDofs(0)
124
    {}
125
126

    /// Clears all data of the mapping.
127
    void clear();
128
    
129
130
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
131
    MultiIndex& operator[](DegreeOfFreedom d)
132
133
134
135
136
137
    {
      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");

      return dofMap[d];
    }
    
138
139
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
140
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
141
    {
142
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
143
144
145
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
146
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
147
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
148
149
      if (dof1 != -1)
	nRankDofs++;
150
151
    }
    
152
153
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
154
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
155
    {
156
      FUNCNAME("FeSpaceDofMap::insert()");
157
158
159
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
160
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
161
      nLocalDofs++;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
162
      nonRankDofs.insert(dof0);
163
164
    }
    
165
    /// Checks if a given DOF is in the DOF mapping.
166
167
168
169
    bool isSet(DegreeOfFreedom dof)
    {
      return static_cast<bool>(dofMap.count(dof));
    }
170

171
172
173
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
174
175
176
177
    bool isRankDof(DegreeOfFreedom dof)
    {
      return !(static_cast<bool>(nonRankDofs.count(dof)));
    }
178
    
179
    /// Returns number of DOFs in the mapping.
180
181
182
183
184
    unsigned int size()
    {
      return dofMap.size();
    }
    
185
    /// Returns the raw data of the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
186
    map<DegreeOfFreedom, MultiIndex>& getMap()
187
188
189
    {
      return dofMap;
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
190

191
192
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
193

194
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
195
196
197
198
199
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

200
201
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
202
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
203
    {
204
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
205
206
    }

207
    /// Informs the mapping whether a global index must be computed.
208
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
209
    {
210
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
211
212
    }

213
    /// Sets the DOF communicators.
214
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
215
    {
216
217
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
218
219
    }

220
  private:
221
222
223
224
225
226
227
228
229
    /// Computes a global mapping from the local one.
    void computeGlobalMapping();

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
    void computeNonLocalIndices();

  private:
    /// MPI communicator object;
230
    MPI::Intracomm mpiComm;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
231

232
233
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
234

235
236
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
237
238
    const FiniteElemSpace *feSpace;

239
    /// Mapping data from DOF indices to local and global indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
240
    map<DegreeOfFreedom, MultiIndex> dofMap;
241

242
    /// Set of all DOFs that are in mapping but are not owned by the rank.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
243
244
    std::set<DegreeOfFreedom> nonRankDofs;

245
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
246
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
247

248
249
250
251
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
252
253
254
  public:
    /// 
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
255
256
  };
  
257
258
259
260
261
262
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
263
  class ParallelDofMapping
264
265
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
266
    ParallelDofMapping() 
267
      : mpiComm(NULL),
268
269
	sendDofs(NULL),
	recvDofs(NULL),
270
	hasNonLocalDofs(false),
271
272
	feSpaces(0),
	nRankDofs(-1),
273
	nLocalDofs(-1),
274
275
276
	nOverallDofs(-1),
	rStartDofs(-1)
    {} 
277

278
279
280
281
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
282
283
284
285
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
286
287
288
289
290
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
291
    void init(MPI::Intracomm m,
292
	      vector<const FiniteElemSpace*> &fe,
293
	      vector<const FiniteElemSpace*> &uniqueFe,
294
295
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
296
297
298
    
    /// Clear all data.
    void clear();
299
300
301
302
303
304
305

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
306
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
307
      FUNCNAME("ParallelDofMapping::operator[]()");
308

309
310
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
311
312
313
314

      return data.find(feSpace)->second;
    }

315
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
316
317
318
319
320
321
322
    inline int getRankDofs()
    {
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");

      return nRankDofs;
    }

323
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
Thomas Witkowski's avatar
Thomas Witkowski committed
324
325
326
327
328
329
330
    inline int getLocalDofs()
    {
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");

      return nLocalDofs;
    }

331
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
332
333
334
335
336
337
338
    inline int getOverallDofs()
    {
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");

      return nOverallDofs;
    }

339
340
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
341
    inline int getStartDofs()
342
343
344
345
346
347
    {
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");

      return rStartDofs;
    }

348
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
349
    void update();
350

351
352
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
353
354
355
356
357
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

358
359
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
360
361
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
362
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
363
364
365
366
367
368
369

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
370
371
372
373
374
375
376
377
378
379
380
381
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
    int computeRankDofs();

    /// Compute \ref nLocalDofs.
    int computeLocalDofs();

    /// Compute \ref nOverallDofs.
    int computeOverallDofs();

    /// Compute \ref rStartDofs.
    int computeStartDofs();

    /// Compute local and global matrix indices.
    void computeMatIndex();

401
  private:
402
    /// MPI communicator object;
403
    MPI::Intracomm mpiComm;
404

405
406
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
407

408
409
410
411
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
412

413
414
415
416
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
417
418
    vector<const FiniteElemSpace*> feSpaces;

419
420
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
421
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
422

423
424
425
426
427
428
429
430
431
432
433
    /// Number of DOFs owned by rank.
    int nRankDofs;

    /// Number of DOFs in rank's subdomain.
    int nLocalDofs;

    /// Number of global DOFs (this value is thus the same on all ranks).
    int nOverallDofs;

    /// Smallest global index of a DOF owned by the rank.
    int rStartDofs;
434
435
436
437

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
438
439
440
441
442
  };

}

#endif