ParallelDofMapping.h 12 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34
35
36

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
  using namespace std;

37
38
  /// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
  /// and a global one.
Thomas Witkowski's avatar
Thomas Witkowski committed
39
40
41
42
43
  struct MultiIndex
  {
    int local, global;
  };

44

45
  /** \brief
46
47
48
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
49
50
51
52
53
54
55
56
57
58
59
60
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

61
62
63
64
65
66
67
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
68
69
70
71
72
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

73
    /// Maps a global DOF index to the global matrix index for a specific 
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


96
97
98
99
100
101
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
102
103
  {
  public:
104
105
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
106
    FeSpaceDofMap() 
107
108
109
    {
      ERROR_EXIT("Should not be called!\n");
    }
110
111
112
     
    /// This is the only valid constructur to be used. 
    FeSpaceDofMap(MPI::Intracomm* m)
113
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
114
115
	sendDofs(NULL),
	recvDofs(NULL),
116
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
117
	needGlobalMapping(false),
118
	hasNonLocalDofs(false),
119
	nRankDofs(0),
Thomas Witkowski's avatar
Thomas Witkowski committed
120
	nLocalDofs(0),
121
	nOverallDofs(0),
122
123
124
125
126
127
128
129
	rStartDofs(0)
    {
      FUNCNAME("FeSpaceDofMap::FeSpaceDofMap()");

      TEST_EXIT(mpiComm)("No MPI Communicator specified!\n");
    }

    /// Clears all data of the mapping.
130
    void clear();
131
    
132
133
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
134
    MultiIndex& operator[](DegreeOfFreedom d)
135
136
137
138
139
140
    {
      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");

      return dofMap[d];
    }
    
141
142
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
143
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
144
    {
145
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
146
147
148
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
149
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
150
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
151
152
      if (dof1 != -1)
	nRankDofs++;
153
154
    }
    
155
156
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
157
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
158
    {
159
      FUNCNAME("FeSpaceDofMap::insert()");
160
161
162
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
163
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
164
      nLocalDofs++;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
165
      nonRankDofs.insert(dof0);
166
167
    }
    
168
    /// Checks if a given DOF is in the DOF mapping.
169
170
171
172
    bool isSet(DegreeOfFreedom dof)
    {
      return static_cast<bool>(dofMap.count(dof));
    }
173

174
175
176
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
177
178
179
180
    bool isRankDof(DegreeOfFreedom dof)
    {
      return !(static_cast<bool>(nonRankDofs.count(dof)));
    }
181
    
182
    /// Returns number of DOFs in the mapping.
183
184
185
186
187
    unsigned int size()
    {
      return dofMap.size();
    }
    
188
    /// Returns the raw data of the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
189
    map<DegreeOfFreedom, MultiIndex>& getMap()
190
191
192
    {
      return dofMap;
    }
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
193

194
195
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
196

197
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
198
199
200
201
202
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

203
204
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
205
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
206
    {
207
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
208
209
    }

210
    /// Informs the mapping whether a global index must be computed.
211
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
212
    {
213
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
214
215
    }

216
    /// Sets the DOF communicators.
217
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
218
    {
219
220
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
221
222
    }

223
  private:
224
225
226
227
228
229
230
231
232
    /// Computes a global mapping from the local one.
    void computeGlobalMapping();

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
    void computeNonLocalIndices();

  private:
    /// MPI communicator object;
233
    MPI::Intracomm* mpiComm;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
234

235
236
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
237

238
239
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
240
241
    const FiniteElemSpace *feSpace;

242
    /// Mapping data from DOF indices to local and global indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
243
    map<DegreeOfFreedom, MultiIndex> dofMap;
244

245
    /// Set of all DOFs that are in mapping but are not owned by the rank.
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
246
247
    std::set<DegreeOfFreedom> nonRankDofs;

248
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
249
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP    
Thomas Witkowski committed
250

251
252
253
254
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
255
256
257
  public:
    /// 
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
258
259
  };
  
260
261
262
263
264
265
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
266
  class ParallelDofMapping
267
268
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
269
    ParallelDofMapping() 
270
      : mpiComm(NULL),
271
272
	sendDofs(NULL),
	recvDofs(NULL),
273
	hasNonLocalDofs(false),
274
275
276
277
278
	feSpaces(0),
	nRankDofs(-1),
	nOverallDofs(-1),
	rStartDofs(-1)
    {} 
279

280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
     *                                PDE to be solved. 
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
    void init(MPI::Intracomm *m,
	      vector<const FiniteElemSpace*> &fe,
	      bool needGlobalMapping,
	      bool bNonLocalDofs);

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
301
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
302
      FUNCNAME("ParallelDofMapping::operator[]()");
303
304
305
306
307
308

      TEST_EXIT_DBG(data.count(feSpace))("Should not happen!\n");

      return data.find(feSpace)->second;
    }

309
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
310
311
312
313
314
315
316
    inline int getRankDofs()
    {
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");

      return nRankDofs;
    }

317
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
Thomas Witkowski's avatar
Thomas Witkowski committed
318
319
320
321
322
323
324
    inline int getLocalDofs()
    {
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");

      return nLocalDofs;
    }

325
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
326
327
328
329
330
331
332
    inline int getOverallDofs()
    {
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");

      return nOverallDofs;
    }

333
334
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
335
    inline int getStartDofs()
336
337
338
339
340
341
    {
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");

      return rStartDofs;
    }

342
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
343
    void update();
344

345
346
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
347
348
349
350
351
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

352
353
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
354
355
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
356
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
357
358
359
360
361
362
363

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
    }

364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
    int computeRankDofs();

    /// Compute \ref nLocalDofs.
    int computeLocalDofs();

    /// Compute \ref nOverallDofs.
    int computeOverallDofs();

    /// Compute \ref rStartDofs.
    int computeStartDofs();

    /// Compute local and global matrix indices.
    void computeMatIndex();

383
  private:
384
    /// MPI communicator object;
385
    MPI::Intracomm* mpiComm;
386

387
388
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
389

390
391
392
393
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
394

395
396
397
398
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
399
400
    vector<const FiniteElemSpace*> feSpaces;

401
402
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
Thomas Witkowski's avatar
Thomas Witkowski committed
403
404
    std::set<const FiniteElemSpace*> feSpacesUnique;

405
406
407
408
409
410
411
412
413
414
415
    /// Number of DOFs owned by rank.
    int nRankDofs;

    /// Number of DOFs in rank's subdomain.
    int nLocalDofs;

    /// Number of global DOFs (this value is thus the same on all ranks).
    int nOverallDofs;

    /// Smallest global index of a DOF owned by the rank.
    int rStartDofs;
416
417
418
419

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
420
421
422
423
424
  };

}

#endif