ParallelDofMapping.h 12.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34
35
36

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
  using namespace std;

37
38
  /// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
  /// and a global one.
Thomas Witkowski's avatar
Thomas Witkowski committed
39
40
41
42
43
  struct MultiIndex
  {
    int local, global;
  };

44

45
  /** \brief
46
47
48
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
49
50
51
52
53
54
55
56
57
58
59
60
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

61
62
63
64
65
66
67
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
68
69
70
71
72
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

73
    /// Maps a global DOF index to the global matrix index for a specific 
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


96
97
98
99
100
101
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
102
103
  {
  public:
104
105
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
106
    FeSpaceDofMap() 
107
108
109
    {
      ERROR_EXIT("Should not be called!\n");
    }
110
111
     
    /// This is the only valid constructur to be used. 
112
    FeSpaceDofMap(MPI::Intracomm m)
113
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
114
115
	sendDofs(NULL),
	recvDofs(NULL),
116
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
117
	needGlobalMapping(false),
118
	hasNonLocalDofs(false),
119
	nRankDofs(0),
Thomas Witkowski's avatar
Thomas Witkowski committed
120
	nLocalDofs(0),
121
	nOverallDofs(0),
122
	rStartDofs(0)
123
    {}
124
125

    /// Clears all data of the mapping.
126
    void clear();
127
    
128
129
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
130
    MultiIndex& operator[](DegreeOfFreedom d)
131
132
133
134
135
136
    {
      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");

      return dofMap[d];
    }
    
137
138
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
139
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
140
    {
141
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
142
143
144
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
145
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
146
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
147
148
      if (dof1 != -1)
	nRankDofs++;
149
150
    }
    
151
152
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
153
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
154
    {
155
      FUNCNAME("FeSpaceDofMap::insert()");
156
157
158
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
159
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
160
      nLocalDofs++;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
161
      nonRankDofs.insert(dof0);
162
163
    }
    
164
    /// Checks if a given DOF is in the DOF mapping.
165
166
167
168
    bool isSet(DegreeOfFreedom dof)
    {
      return static_cast<bool>(dofMap.count(dof));
    }
169

170
171
172
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
173
174
175
176
    bool isRankDof(DegreeOfFreedom dof)
    {
      return !(static_cast<bool>(nonRankDofs.count(dof)));
    }
177
    
178
    /// Returns number of DOFs in the mapping.
179
180
181
182
183
    unsigned int size()
    {
      return dofMap.size();
    }
    
184
    /// Returns the raw data of the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
185
    map<DegreeOfFreedom, MultiIndex>& getMap()
186
187
188
    {
      return dofMap;
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
189

190
191
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
192

193
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
194
195
196
197
198
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

199
200
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
201
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
202
    {
203
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
204
205
    }

206
    /// Informs the mapping whether a global index must be computed.
207
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
208
    {
209
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
210
211
    }

212
    /// Sets the DOF communicators.
213
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
214
    {
215
216
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
217
218
    }

219
  private:
220
221
222
223
224
225
226
227
228
    /// Computes a global mapping from the local one.
    void computeGlobalMapping();

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
    void computeNonLocalIndices();

  private:
    /// MPI communicator object;
229
    MPI::Intracomm mpiComm;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
230

231
232
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
233

234
235
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
236
237
    const FiniteElemSpace *feSpace;

238
    /// Mapping data from DOF indices to local and global indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
239
    map<DegreeOfFreedom, MultiIndex> dofMap;
240

241
    /// Set of all DOFs that are in mapping but are not owned by the rank.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
242
243
    std::set<DegreeOfFreedom> nonRankDofs;

244
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
245
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
246

247
248
249
250
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
251
252
253
  public:
    /// 
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
254
255
  };
  
256
257
258
259
260
261
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
262
  class ParallelDofMapping
263
264
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
265
    ParallelDofMapping() 
266
      : mpiComm(NULL),
267
268
	sendDofs(NULL),
	recvDofs(NULL),
269
	hasNonLocalDofs(false),
270
271
	feSpaces(0),
	nRankDofs(-1),
272
	nLocalDofs(-1),
273
274
275
	nOverallDofs(-1),
	rStartDofs(-1)
    {} 
276

277
278
279
280
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
281
282
283
284
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
285
286
287
288
289
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
290
    void init(MPI::Intracomm m,
291
	      vector<const FiniteElemSpace*> &fe,
292
	      vector<const FiniteElemSpace*> &uniqueFe,
293
294
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
295
296
297
    
    /// Clear all data.
    void clear();
298
299
300
301
302
303
304

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
305
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
306
      FUNCNAME("ParallelDofMapping::operator[]()");
307
308
309
310
311
312

      TEST_EXIT_DBG(data.count(feSpace))("Should not happen!\n");

      return data.find(feSpace)->second;
    }

313
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
314
315
316
317
318
319
320
    inline int getRankDofs()
    {
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");

      return nRankDofs;
    }

321
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
Thomas Witkowski's avatar
Thomas Witkowski committed
322
323
324
325
326
327
328
    inline int getLocalDofs()
    {
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");

      return nLocalDofs;
    }

329
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
330
331
332
333
334
335
336
    inline int getOverallDofs()
    {
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");

      return nOverallDofs;
    }

337
338
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
339
    inline int getStartDofs()
340
341
342
343
344
345
    {
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");

      return rStartDofs;
    }

346
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
347
    void update();
348

349
350
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
351
352
353
354
355
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

356
357
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
358
359
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
360
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
361
362
363
364
365
366
367

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
    }

368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
    int computeRankDofs();

    /// Compute \ref nLocalDofs.
    int computeLocalDofs();

    /// Compute \ref nOverallDofs.
    int computeOverallDofs();

    /// Compute \ref rStartDofs.
    int computeStartDofs();

    /// Compute local and global matrix indices.
    void computeMatIndex();

387
  private:
388
    /// MPI communicator object;
389
    MPI::Intracomm mpiComm;
390

391
392
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
393

394
395
396
397
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
398

399
400
401
402
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
403
404
    vector<const FiniteElemSpace*> feSpaces;

405
406
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
407
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
408

409
410
411
412
413
414
415
416
417
418
419
    /// Number of DOFs owned by rank.
    int nRankDofs;

    /// Number of DOFs in rank's subdomain.
    int nLocalDofs;

    /// Number of global DOFs (this value is thus the same on all ranks).
    int nOverallDofs;

    /// Smallest global index of a DOF owned by the rank.
    int rStartDofs;
420
421
422
423

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
424
425
426
427
428
  };

}

#endif