ParallelDofMapping.h 13.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
#include "parallel/MeshLevelData.h"
28
29
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
30
#include "parallel/StdMpi.h"
31
32
33
34
35

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
36

37
38
  using namespace std;

39
  /** \brief
40
41
42
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
43
44
45
46
47
48
49
50
51
52
53
54
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

55
56
57
58
59
60
61
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
62
63
64
65
66
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

67
    /// Maps a global DOF index to the global matrix index for a specific 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


90
91
92
93
94
95
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
96
97
  {
  public:
98
99
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
100
    FeSpaceDofMap() 
101
102
103
    {
      ERROR_EXIT("Should not be called!\n");
    }
104
105
     
    /// This is the only valid constructur to be used. 
106
107
    FeSpaceDofMap(MeshLevelData* ld)
      : levelData(ld),
108
	dofComm(NULL),
109
	feSpace(NULL),
110
	dofMap(1),
Thomas Witkowski's avatar
Thomas Witkowski committed
111
	needGlobalMapping(false),
112
	hasNonLocalDofs(false)
113
114
115
    {
      clear();
    }
116
117

    /// Clears all data of the mapping.
118
    void clear();
119
120

#if 0    
121
122
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
123
    MultiIndex& operator[](DegreeOfFreedom d)
124
    {
125
      TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
126

127
      return dofMap[0][d];
128
    }
129
130
131
132
133
134
135
136
#else
    DofMap& operator[](int level)
    {
      TEST_EXIT_DBG(level < static_cast<int>(dofMap.size()))("Should not happen!\n");
      
      return dofMap[level];
    }
#endif
137
    
138
139
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
140
    void insertRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
141
    {
142
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
143
      
144
      TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
145
      
146
147
      dofMap[level][dof0].local = dof1;
      nLocalDofs[level]++;
Thomas Witkowski's avatar
Thomas Witkowski committed
148
      if (dof1 != -1)
149
	nRankDofs[level]++;
150
151
    }
    
152
153
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
154
    void insertNonRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
155
    {
156
      FUNCNAME("FeSpaceDofMap::insertNonRankDof()");
157
      
158
      TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
159
      
160
161
162
      dofMap[level][dof0].local = dof1;
      nLocalDofs[level]++;
      nonRankDofs[level].insert(dof0);
163
164
    }
    
165
    /// Checks if a given DOF is in the DOF mapping.
166
    bool isSet(DegreeOfFreedom dof, int level)
167
    {
168
      return static_cast<bool>(dofMap[level].count(dof));
169
    }
170

171
172
173
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
174
    bool isRankDof(DegreeOfFreedom dof, int level)
175
    {
176
      return !(static_cast<bool>(nonRankDofs[level].count(dof)));
177
    }
178
    
179
    /// Returns number of DOFs in the mapping.
180
    unsigned int size(int level)
181
    {
182
      return dofMap[level].size();
183
184
    }
    
185
    /// Returns the raw data of the mapping.
186
    DofMap& getMap(int level)
187
    {
188
      return dofMap[level];
189
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
190

191
192
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
193

194
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
195
196
197
198
199
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

200
201
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
202
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
203
    {
204
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
205
206
    }

207
    /// Informs the mapping whether a global index must be computed.
208
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
209
    {
210
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
211
212
    }

213
    /// Sets the DOF communicators.
214
    void setDofComm(DofComm &dc)
Thomas Witkowski's avatar
Thomas Witkowski committed
215
    {
216
      dofComm = &dc;
Thomas Witkowski's avatar
Thomas Witkowski committed
217
218
    }

219
220
221
222
223
224
    void setMpiComm(MPI::Intracomm &m, int l)
    {
      mpiComm = m;
      meshLevel = l;
    }

225
  private:
226
    /// Computes a global mapping from the local one.
227
    void computeGlobalMapping(int level);
228
229
230

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
231
    void computeNonLocalIndices(int level);
232
233

  private:
234
    MeshLevelData *levelData;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
235

236
237
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
238

239
240
241
242
    MPI::Intracomm mpiComm;

    int meshLevel;

243
    /// The FE space this mapping belongs to. This is used only the get the
244
    /// correct DOF communicator in \ref dofComm.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
245
246
    const FiniteElemSpace *feSpace;

247
    /// Mapping data from DOF indices to local and global indices.
248
    vector<DofMap> dofMap;
249

250
    /// Set of all DOFs that are in mapping but are not owned by the rank.
251
    vector<std::set<DegreeOfFreedom> > nonRankDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
252

253
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
254
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
255

256
257
258
259
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
260

261
262
  public:
    /// 
263
    vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
264
265
  };
  
266
267
268
269
270
271
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
272
  class ParallelDofMapping
273
274
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
275
    ParallelDofMapping() 
276
      : levelData(NULL),
277
	hasNonLocalDofs(false),
278
279
	needMatIndex(false),
	needMatIndexFromGlobal(false),
280
	feSpaces(0),
281
282
283
284
285
286
287
288
289
290
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
      nRankDofs[0] = -1;
      nLocalDofs[0] = -1;
      nOverallDofs[0] = -1;
      rStartDofs[0] = -1;
    } 
291

292
293
294
295
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
296
297
298
299
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
300
301
302
303
304
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
305
    void init(MeshLevelData& mld,
306
	      vector<const FiniteElemSpace*> &fe,
307
	      vector<const FiniteElemSpace*> &uniqueFe,
308
309
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
310
311

    void setMpiComm(MPI::Intracomm &m, int l);
312
313
314
    
    /// Clear all data.
    void clear();
315
316
317

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
318
    void setDofComm(DofComm &dofComm);
319

320
321
322
323
324
325
    void setComputeMatIndex(bool b, bool global = false)
    {
      needMatIndex = b;
      needMatIndexFromGlobal = global;
    }

326
327
    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
328
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
329
      FUNCNAME("ParallelDofMapping::operator[]()");
330

331
332
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
333
334
335
336

      return data.find(feSpace)->second;
    }

337
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
338
    inline int getRankDofs(int level)
339
    {
340
      TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
341

342
      return nRankDofs[level];
343
344
    }

345
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
346
    inline int getLocalDofs(int level)
Thomas Witkowski's avatar
Thomas Witkowski committed
347
    {
348
      TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
349

350
      return nLocalDofs[level];
Thomas Witkowski's avatar
Thomas Witkowski committed
351
352
    }

353
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
354
    inline int getOverallDofs(int level)
355
    {
356
      TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
357

358
      return nOverallDofs[level];
359
360
    }

361
362
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
363
    inline int getStartDofs(int level)
364
    {
365
      TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
366

367
      return rStartDofs[level];
368
369
    }

370
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
371
    void update();
372

373
374
375
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

376
377
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
378
    inline int getMatIndex(int level, int ithComponent, DegreeOfFreedom d)
Thomas Witkowski's avatar
Thomas Witkowski committed
379
    {
380
      return dofToMatIndex[level].get(ithComponent, d);
Thomas Witkowski's avatar
Thomas Witkowski committed
381
382
    }

383
384
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
385
    inline int getLocalMatIndex(int level, int ithComponent, DegreeOfFreedom d)
386
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
387
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
388

389
      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d, level))
390
391
	("Should not happen!\n");

392
      return dofToMatIndex[level].get(ithComponent, d) - rStartDofs[level];
393
394
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
395
396
397
398
399
400
401
402
403
404
405
406
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

407
    /// Compute local and global matrix indices.
408
    void computeMatIndex(bool globalIndex, int level);
409

410
411
412
413
414
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
415
    int computeRankDofs(int level);
416
417

    /// Compute \ref nLocalDofs.
418
    int computeLocalDofs(int level);
419
420

    /// Compute \ref nOverallDofs.
421
    int computeOverallDofs(int level);
422
423

    /// Compute \ref rStartDofs.
424
    int computeStartDofs(int level);
425

426
  private:
427
428
429
430
    MPI::Intracomm mpiComm;

    int meshLevel;

431
    MeshLevelData *levelData;
432

433
434
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
435

436
437
438
439
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
440

441
442
443
444
445
446
447
448
449
450
    /// If true, matrix indeces for the stored DOFs are computed, see
    /// \ref computeMatIndex.
    bool needMatIndex;

    /// If matrix indices should be computed, this variable defines if the
    /// mapping from DOF indices to matrix row indices is defined on local
    /// or global DOF indices. If true, the mapping is to specify and to use
    /// on global ones, otherwise on local DOF indices.
    bool needMatIndexFromGlobal;

451
452
453
454
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
455
456
    vector<const FiniteElemSpace*> feSpaces;

457
458
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
459
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
460

461
    /// Number of DOFs owned by rank.
462
    vector<int> nRankDofs;
463
464

    /// Number of DOFs in rank's subdomain.
465
    vector<int> nLocalDofs;
466
467

    /// Number of global DOFs (this value is thus the same on all ranks).
468
    vector<int> nOverallDofs;
469
470

    /// Smallest global index of a DOF owned by the rank.
471
    vector<int> rStartDofs;
472
473
474

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
475
    vector<DofToMatIndex> dofToMatIndex;
476
477
478
479
480
  };

}

#endif