ParallelDofMapping.h 12.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34
35
36

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
  using namespace std;

37
38
  /// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
  /// and a global one.
Thomas Witkowski's avatar
Thomas Witkowski committed
39
40
41
42
43
  struct MultiIndex
  {
    int local, global;
  };

44

45
  /** \brief
46
47
48
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
49
50
51
52
53
54
55
56
57
58
59
60
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

61
62
63
64
65
66
67
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
68
69
70
71
72
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

73
    /// Maps a global DOF index to the global matrix index for a specific 
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


96
97
98
99
100
101
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
102
103
  {
  public:
104
105
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
106
    FeSpaceDofMap() 
107
108
109
    {
      ERROR_EXIT("Should not be called!\n");
    }
110
111
112
     
    /// This is the only valid constructur to be used. 
    FeSpaceDofMap(MPI::Intracomm* m)
113
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
114
115
	sendDofs(NULL),
	recvDofs(NULL),
116
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
117
	needGlobalMapping(false),
118
	hasNonLocalDofs(false),
119
	nRankDofs(0),
Thomas Witkowski's avatar
Thomas Witkowski committed
120
	nLocalDofs(0),
121
	nOverallDofs(0),
122
123
124
125
126
127
128
129
	rStartDofs(0)
    {
      FUNCNAME("FeSpaceDofMap::FeSpaceDofMap()");

      TEST_EXIT(mpiComm)("No MPI Communicator specified!\n");
    }

    /// Clears all data of the mapping.
130
    void clear();
131
    
132
133
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
134
    MultiIndex& operator[](DegreeOfFreedom d)
135
136
137
138
139
140
    {
      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");

      return dofMap[d];
    }
    
141
142
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
143
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
144
    {
145
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
146
147
148
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
149
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
150
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
151
152
      if (dof1 != -1)
	nRankDofs++;
153
154
    }
    
155
156
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
157
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
158
    {
159
      FUNCNAME("FeSpaceDofMap::insert()");
160
161
162
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
163
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
164
      nLocalDofs++;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
165
      nonRankDofs.insert(dof0);
166
167
    }
    
168
    /// Checks if a given DOF is in the DOF mapping.
169
170
171
172
    bool isSet(DegreeOfFreedom dof)
    {
      return static_cast<bool>(dofMap.count(dof));
    }
173

174
175
176
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
177
178
179
180
    bool isRankDof(DegreeOfFreedom dof)
    {
      return !(static_cast<bool>(nonRankDofs.count(dof)));
    }
181
    
182
    /// Returns number of DOFs in the mapping.
183
184
185
186
187
    unsigned int size()
    {
      return dofMap.size();
    }
    
188
    /// Returns the raw data of the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
189
    map<DegreeOfFreedom, MultiIndex>& getMap()
190
191
192
    {
      return dofMap;
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
193

194
195
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
196

197
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
198
199
200
201
202
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

203
204
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
205
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
206
    {
207
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
208
209
    }

210
    /// Informs the mapping whether a global index must be computed.
211
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
212
    {
213
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
214
215
    }

216
    /// Sets the DOF communicators.
217
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
218
    {
219
220
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
221
222
    }

223
  private:
224
225
226
227
228
229
230
231
232
    /// Computes a global mapping from the local one.
    void computeGlobalMapping();

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
    void computeNonLocalIndices();

  private:
    /// MPI communicator object;
233
    MPI::Intracomm* mpiComm;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
234

235
236
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
237

238
239
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
240
241
    const FiniteElemSpace *feSpace;

242
    /// Mapping data from DOF indices to local and global indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
243
    map<DegreeOfFreedom, MultiIndex> dofMap;
244

245
    /// Set of all DOFs that are in mapping but are not owned by the rank.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
246
247
    std::set<DegreeOfFreedom> nonRankDofs;

248
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
249
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
250

251
252
253
254
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
255
256
257
  public:
    /// 
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
258
259
  };
  
260
261
262
263
264
265
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
266
  class ParallelDofMapping
267
268
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
269
    ParallelDofMapping() 
270
      : mpiComm(NULL),
271
272
	sendDofs(NULL),
	recvDofs(NULL),
273
	hasNonLocalDofs(false),
274
275
	feSpaces(0),
	nRankDofs(-1),
276
	nLocalDofs(-1),
277
278
279
	nOverallDofs(-1),
	rStartDofs(-1)
    {} 
280

281
282
283
284
285
286
287
288
289
290
291
292
293
294
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
     *                                PDE to be solved. 
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
    void init(MPI::Intracomm *m,
	      vector<const FiniteElemSpace*> &fe,
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
295
296
297
    
    /// Clear all data.
    void clear();
298
299
300
301
302
303
304

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
305
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
306
      FUNCNAME("ParallelDofMapping::operator[]()");
307
308
309
310
311
312

      TEST_EXIT_DBG(data.count(feSpace))("Should not happen!\n");

      return data.find(feSpace)->second;
    }

313
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
314
315
316
317
318
319
320
    inline int getRankDofs()
    {
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");

      return nRankDofs;
    }

321
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
Thomas Witkowski's avatar
Thomas Witkowski committed
322
323
324
325
326
327
328
    inline int getLocalDofs()
    {
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");

      return nLocalDofs;
    }

329
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
330
331
332
333
334
335
336
    inline int getOverallDofs()
    {
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");

      return nOverallDofs;
    }

337
338
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
339
    inline int getStartDofs()
340
341
342
343
344
345
    {
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");

      return rStartDofs;
    }

346
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
347
    void update();
348

349
350
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
351
352
353
354
355
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

356
357
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
358
359
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
360
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
361
362
363
364
365
366
367

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
    }

368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
    int computeRankDofs();

    /// Compute \ref nLocalDofs.
    int computeLocalDofs();

    /// Compute \ref nOverallDofs.
    int computeOverallDofs();

    /// Compute \ref rStartDofs.
    int computeStartDofs();

    /// Compute local and global matrix indices.
    void computeMatIndex();

387
  private:
388
    /// MPI communicator object;
389
    MPI::Intracomm* mpiComm;
390

391
392
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
393

394
395
396
397
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
398

399
400
401
402
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
403
404
    vector<const FiniteElemSpace*> feSpaces;

405
406
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
Thomas Witkowski's avatar
Thomas Witkowski committed
407
408
    std::set<const FiniteElemSpace*> feSpacesUnique;

409
410
411
412
413
414
415
416
417
418
419
    /// Number of DOFs owned by rank.
    int nRankDofs;

    /// Number of DOFs in rank's subdomain.
    int nLocalDofs;

    /// Number of global DOFs (this value is thus the same on all ranks).
    int nOverallDofs;

    /// Smallest global index of a DOF owned by the rank.
    int rStartDofs;
420
421
422
423

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
424
425
426
427
428
  };

}

#endif