ParallelDofMapping.h 13.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
#include "parallel/MeshLevelData.h"
28
29
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
30
#include "parallel/StdMpi.h"
31
32
33
34
35

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
36

37
38
  using namespace std;

39
  /** \brief
40
41
42
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
43
44
45
46
47
48
49
50
51
52
53
54
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

55
56
57
58
59
60
61
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
62
63
64
65
66
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

67
    /// Maps a global DOF index to the global matrix index for a specific 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


90
91
92
93
94
95
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
96
97
  {
  public:
98
99
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
100
    FeSpaceDofMap() 
101
102
103
    {
      ERROR_EXIT("Should not be called!\n");
    }
104
105
     
    /// This is the only valid constructur to be used. 
106
107
    FeSpaceDofMap(MeshLevelData* ld)
      : levelData(ld),
Thomas Witkowski's avatar
Thomas Witkowski committed
108
109
	sendDofs(NULL),
	recvDofs(NULL),
110
	feSpace(NULL),
111
	dofMap(1),
Thomas Witkowski's avatar
Thomas Witkowski committed
112
	needGlobalMapping(false),
113
	hasNonLocalDofs(false),
114
115
116
117
118
119
120
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
      clear();
    }
121
122

    /// Clears all data of the mapping.
123
    void clear();
124
125

#if 0    
126
127
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
128
    MultiIndex& operator[](DegreeOfFreedom d)
129
    {
130
      TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
131

132
      return dofMap[0][d];
133
    }
134
135
136
137
138
139
140
141
#else
    DofMap& operator[](int level)
    {
      TEST_EXIT_DBG(level < static_cast<int>(dofMap.size()))("Should not happen!\n");
      
      return dofMap[level];
    }
#endif
142
    
143
144
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
145
    void insertRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
146
    {
147
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
148
      
149
      TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
150
      
151
152
      dofMap[level][dof0].local = dof1;
      nLocalDofs[level]++;
Thomas Witkowski's avatar
Thomas Witkowski committed
153
      if (dof1 != -1)
154
	nRankDofs[level]++;
155
156
    }
    
157
158
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
159
    void insertNonRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
160
    {
161
      FUNCNAME("FeSpaceDofMap::insertNonRankDof()");
162
      
163
      TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
164
      
165
166
167
      dofMap[level][dof0].local = dof1;
      nLocalDofs[level]++;
      nonRankDofs[level].insert(dof0);
168
169
    }
    
170
    /// Checks if a given DOF is in the DOF mapping.
171
    bool isSet(DegreeOfFreedom dof, int level)
172
    {
173
      return static_cast<bool>(dofMap[level].count(dof));
174
    }
175

176
177
178
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
179
    bool isRankDof(DegreeOfFreedom dof, int level)
180
    {
181
      return !(static_cast<bool>(nonRankDofs[level].count(dof)));
182
    }
183
    
184
    /// Returns number of DOFs in the mapping.
185
    unsigned int size(int level)
186
    {
187
      return dofMap[level].size();
188
189
    }
    
190
    /// Returns the raw data of the mapping.
191
    DofMap& getMap(int level)
192
    {
193
      return dofMap[level];
194
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
195

196
197
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
198

199
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
200
201
202
203
204
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

205
206
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
207
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
208
    {
209
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
210
211
    }

212
    /// Informs the mapping whether a global index must be computed.
213
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
214
    {
215
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
216
217
    }

218
    /// Sets the DOF communicators.
219
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
220
    {
221
222
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
223
224
    }

225
  private:
226
    /// Computes a global mapping from the local one.
227
    void computeGlobalMapping(int level);
228
229
230

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
231
    void computeNonLocalIndices(int level);
232
233

  private:
234
    MeshLevelData *levelData;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
235

236
237
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
238

239
240
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
241
242
    const FiniteElemSpace *feSpace;

243
    /// Mapping data from DOF indices to local and global indices.
244
    vector<DofMap> dofMap;
245

246
    /// Set of all DOFs that are in mapping but are not owned by the rank.
247
    vector<std::set<DegreeOfFreedom> > nonRankDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
248

249
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
250
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
251

252
253
254
255
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
256

257
258
  public:
    /// 
259
    vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
260
261
  };
  
262
263
264
265
266
267
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
268
  class ParallelDofMapping
269
270
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
271
    ParallelDofMapping() 
272
273
      : levelData(NULL),
	sendDofs(NULL),
274
	recvDofs(NULL),
275
	hasNonLocalDofs(false),
276
277
	needMatIndex(false),
	needMatIndexFromGlobal(false),
278
	feSpaces(0),
279
280
281
282
283
284
285
286
287
288
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
      nRankDofs[0] = -1;
      nLocalDofs[0] = -1;
      nOverallDofs[0] = -1;
      rStartDofs[0] = -1;
    } 
289

290
291
292
293
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
294
295
296
297
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
298
299
300
301
302
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
303
    void init(MeshLevelData& mld,
304
	      vector<const FiniteElemSpace*> &fe,
305
	      vector<const FiniteElemSpace*> &uniqueFe,
306
307
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
308
309
310
    
    /// Clear all data.
    void clear();
311
312
313
314
315

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

316
317
318
319
320
321
    void setComputeMatIndex(bool b, bool global = false)
    {
      needMatIndex = b;
      needMatIndexFromGlobal = global;
    }

322
323
    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
324
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
325
      FUNCNAME("ParallelDofMapping::operator[]()");
326

327
328
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
329
330
331
332

      return data.find(feSpace)->second;
    }

333
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
334
    inline int getRankDofs(int level)
335
    {
336
      TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
337

338
      return nRankDofs[level];
339
340
    }

341
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
342
    inline int getLocalDofs(int level)
Thomas Witkowski's avatar
Thomas Witkowski committed
343
    {
344
      TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
345

346
      return nLocalDofs[level];
Thomas Witkowski's avatar
Thomas Witkowski committed
347
348
    }

349
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
350
    inline int getOverallDofs(int level)
351
    {
352
      TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
353

354
      return nOverallDofs[level];
355
356
    }

357
358
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
359
    inline int getStartDofs(int level)
360
    {
361
      TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
362

363
      return rStartDofs[level];
364
365
    }

366
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
367
    void update();
368

369
370
371
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

372
373
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
374
    inline int getMatIndex(int level, int ithComponent, DegreeOfFreedom d)
Thomas Witkowski's avatar
Thomas Witkowski committed
375
    {
376
      return dofToMatIndex[level].get(ithComponent, d);
Thomas Witkowski's avatar
Thomas Witkowski committed
377
378
    }

379
380
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
381
    inline int getLocalMatIndex(int level, int ithComponent, DegreeOfFreedom d)
382
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
383
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
384

385
      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d, level))
386
387
	("Should not happen!\n");

388
      return dofToMatIndex[level].get(ithComponent, d) - rStartDofs[level];
389
390
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
391
392
393
394
395
396
397
398
399
400
401
402
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

403
    /// Compute local and global matrix indices.
404
    void computeMatIndex(bool globalIndex, int level);
405

406
407
408
409
410
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
411
    int computeRankDofs(int level);
412
413

    /// Compute \ref nLocalDofs.
414
    int computeLocalDofs(int level);
415
416

    /// Compute \ref nOverallDofs.
417
    int computeOverallDofs(int level);
418
419

    /// Compute \ref rStartDofs.
420
    int computeStartDofs(int level);
421

422
  private:
423
    MeshLevelData *levelData;
424

425
426
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
427

428
429
430
431
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
432

433
434
435
436
437
438
439
440
441
442
    /// If true, matrix indeces for the stored DOFs are computed, see
    /// \ref computeMatIndex.
    bool needMatIndex;

    /// If matrix indices should be computed, this variable defines if the
    /// mapping from DOF indices to matrix row indices is defined on local
    /// or global DOF indices. If true, the mapping is to specify and to use
    /// on global ones, otherwise on local DOF indices.
    bool needMatIndexFromGlobal;

443
444
445
446
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
447
448
    vector<const FiniteElemSpace*> feSpaces;

449
450
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
451
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
452

453
    /// Number of DOFs owned by rank.
454
    vector<int> nRankDofs;
455
456

    /// Number of DOFs in rank's subdomain.
457
    vector<int> nLocalDofs;
458
459

    /// Number of global DOFs (this value is thus the same on all ranks).
460
    vector<int> nOverallDofs;
461
462

    /// Smallest global index of a DOF owned by the rank.
463
    vector<int> rStartDofs;
464
465
466

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
467
    vector<DofToMatIndex> dofToMatIndex;
468
469
470
471
472
  };

}

#endif